From 49ec05c8eeda8330af625b7e580a064e8f8c6dfa Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Mon, 10 Apr 2023 16:56:07 +0800
Subject: [PATCH 001/405] first commit

---
 cmake/external/flashattn.cmake                |   7 +-
 .../operators/fused/fused_gate_attention.h    | 362 ++++++++++++++----
 paddle/phi/backends/dynload/flashattn.h       |   3 +-
 3 files changed, 287 insertions(+), 85 deletions(-)

diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index eae35d90f50f0..53b11da6a6732 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -19,8 +19,11 @@ add_definitions(-DPADDLE_WITH_FLASHATTN)
 set(FLASHATTN_PREFIX_DIR ${THIRD_PARTY_PATH}/flashattn)
 set(FLASHATTN_SOURCE_SUBDIR csrc/flash_attn)
 set(FLASHATTN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flashattn)
-set(FLASHATTN_REPOSITORY ${GIT_URL}/PaddlePaddle/flash-attention.git)
-set(FLASHATTN_TAG f0edf243a813a65d05c75fcb331b2a95faf96bbc)
+set(FLASHATTN_REPOSITORY ${GIT_URL}/JamesLim-sy/flash-attention.git)
+set(FLASHATTN_TAG 05ecd140356cffe46444cd364944f6ec5067e724)
+
+#set(FLASHATTN_REPOSITORY ${GIT_URL}/PaddlePaddle/flash-attention.git)
+#set(FLASHATTN_TAG f0edf243a813a65d05c75fcb331b2a95faf96bbc)
 
 set(FLASHATTN_INCLUDE_DIR
     "${FLASHATTN_INSTALL_DIR}/include"
diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index c8c4733df2e2e..7f634e835cadf 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -14,6 +14,7 @@ limitations under the License. */
 
 #pragma once
 
+#include "paddle/phi/kernels/arange_kernel.h"
 #include "paddle/phi/kernels/funcs/broadcast_function.h"
 #include "paddle/phi/kernels/funcs/elementwise_base.h"
 #include "paddle/phi/kernels/funcs/elementwise_functor.h"
@@ -146,6 +147,22 @@ struct GateAttentionConfig {
     gate_out_dims = {batch_size, seq_len_m, seq_len_r, num_heads, head_dim};
   }
 
+  bool UseFlashAttn(const bool merge_qkv, const int64_t head_dim) {
+    if (merge_qkv) {
+      switch (head_dim) {
+        case 16:
+        case 32:
+        case 64:
+        case 128:
+          return true;
+        default:
+          return false;
+      }
+    } else {
+      return false;
+    }
+  }
+
   int64_t GetQuerySize() const {
     return batch_size * seq_len_m * seq_len_r * num_heads * head_dim;
   }
@@ -317,6 +334,15 @@ struct GateAttentionGradConfig : public GateAttentionConfig<T> {
   phi::DenseTensor qk_out_grad;
 };
 
+template <typename T, typename OUT_TYPE>
+__global__ void FlashAttRange(
+    int start, int step, int size, OUT_TYPE* out1, OUT_TYPE* out2) {
+  CUDA_KERNEL_LOOP(index, size) {
+    out1[index] = static_cast<OUT_TYPE>(start + step * index);
+    out2[index] = static_cast<OUT_TYPE>(start + step * index);
+  }
+}
+
 template <typename T>
 class FMHAGateRef {
  public:
@@ -336,106 +362,278 @@ class FMHAGateRef {
     T* q_ptr = nullptr;
     T* k_ptr = nullptr;
     T* v_ptr = nullptr;
-    if (merge_qkv_) {
-      // qkv_transpose_out = transpose(qkv_out)
+
+    if (config->UseFlashAttn(merge_qkv_, config->head_dim)) {
       PADDLE_ENFORCE_NOT_NULL(
           qkv_transpose_out,
           platform::errors::NotFound("The input qkv_transpose_out can not be "
                                      "nullptr when merge_qkv is true."));
-
       phi::DenseTensor* qkv_out = config->GetQKVOut();
       ComputeQKVTransposeForward(*qkv_out, qkv_transpose_out);
       config->ClearQKVOut();
 
+      // batch_dims = q.shape[:-3];
+      // n, no_heads, c = q.shape[-3:]
+      // 1. Dealing with qkv_out for flash_attn.
+      auto& qkv_dims = qkv_transpose_out.dims();
+      auto rank = qkv_dims.size();
+      auto n = qkv_dims[rank - 3];
+      auto no_heads = qkv_dims[rank - 2];
+      auto c = qkv_dims[rank - 1];
+
+      int64_t q_batch_size = 1;
+      for (int i = 0; i < (rank - 3); ++i) {
+        q_batch_size *= qkv_dims[i];
+      }
+      qkv_transpose_out.Resize({q_batch_size * n, no_heads, c});
       // q_size == k_size
       int64_t q_size = config->GetQuerySize();
       q_ptr = qkv_transpose_out->data<T>();
       k_ptr = q_ptr + q_size;
       v_ptr = k_ptr + q_size;
-    } else {
-      PADDLE_ENFORCE_NOT_NULL(
-          q_transpose_out,
-          platform::errors::NotFound("The input q_transpose_out can not be "
-                                     "nullptr when merge_qkv is false."));
-      PADDLE_ENFORCE_NOT_NULL(
-          k_transpose_out,
-          platform::errors::NotFound("The input k_transpose_out can not be "
-                                     "nullptr when merge_qkv is false."));
-      PADDLE_ENFORCE_NOT_NULL(
-          v_transpose_out,
-          platform::errors::NotFound("The input v_transpose_out can not be "
-                                     "nullptr when merge_qkv is false."));
-
-      phi::DenseTensor* query_out = config->GetQueryOut();
-      phi::DenseTensor* key_out = config->GetKeyOut();
-      phi::DenseTensor* value_out = config->GetValueOut();
-      ComputeQKVTransposeForward(*query_out,
-                                 *key_out,
-                                 *value_out,
-                                 q_transpose_out,
-                                 k_transpose_out,
-                                 v_transpose_out);
-
-      // q_size != k_size
-      q_ptr = q_transpose_out->data<T>();
-      k_ptr = k_transpose_out->data<T>();
-      v_ptr = v_transpose_out->data<T>();
-    }
-
-    // qk_out = BatchedGEMM(Q, K^T)
-    // [batch_size, seq_len_m, num_heads, seq_len_r, head_dim] *
-    //                [batch_size, seq_len_m, num_heads, m_size, head_dim]
-    // -> [batch_size, seq_len_m, num_heads, seq_len_r, m_size]
-    phi::DenseTensor* qk_out = config->GetQKOut(softmax_out);
-    T* qk_out_ptr = qk_out->data<T>();
-
-    int64_t gemm_batch_size =
-        config->batch_size * config->seq_len_m * config->num_heads;
-    int64_t gemm_m = config->seq_len_r;
-    int64_t gemm_n = config->m_size;
-    int64_t gemm_k = config->head_dim;
-
-    T alpha = static_cast<T>(1.0 / sqrt(config->head_dim));
-    ComputeBatchedGEMM(q_ptr,
-                       k_ptr,
-                       qk_out_ptr,
-                       false,
-                       true,
-                       gemm_m,
-                       gemm_n,
-                       gemm_k,
-                       gemm_batch_size,
-                       alpha);
 
-    // softmax_out = softmax(qk_out + nonbatched_bias + src_mask)
-    ComputeBiasMaskSoftmaxForward(
-        nonbatched_bias, src_mask, qk_out, softmax_out);
-    config->ClearQKOut();
+      // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
+      phi::DenseTensor cu_seq_q, cu_seq_k;
+      int64_t temp_size = (q_batch_size + 1) * n;
+      int64_t seq_size = 0;
+      int64_t start = 0, end = temp_size, step = n;
+      phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
+      cu_seq_q.Resize({seq_size});
+      cu_seq_k.Resize({seq_size});
+      AllocWithDebugInfo<int32_t>(dev_ctx_, "flash_attn: cu_seq_q", &cu_seq_q);
+      AllocWithDebugInfo<int32_t>(dev_ctx_, "flash_attn: cu_seq_k", &cu_seq_k);
+
+      int64_t block = std::min(seq_size, static_cast<int64_t>(256));
+      int64_t grid = (range_size + block - 1) / block;
+      FlashAttRange<T><<<grid, block, 0, dev_ctx_.stream()>>>(
+          start, step, end, ` cu_seq_q.data<T>(), cu_seq_k.data<T>());
+
+      // 1. Dealing with mask and bias for flash_attn.
+      phi::DenseTensor tmp_mask, tmp_bias;
+      if (src_mask) {
+        int64_t mask_first_dim = 0;
+        auto& mask_dim = src_mask->dims();
+        for (int i = 0; i < mask_dim.size() - 3; ++i) {
+          mask_first_dim *= mask_dim[i];
+        }
+        tmp_mask.ShareDataWith(*src_mask);
+        tmp_mask.Resize(
+            {mask_first_dim, mask_dim[-3], mask_dim[-2], mask_dim[-1]});
+      }
+      if (nonbatched_bias) {
+        int64_t bias_first_dim = 0;
+        auto& bias_dim = nonbatched_bias->dims();
+        for (int i = 0; i < bias_dim.size() - 3; ++i) {
+          bias_first_dim *= bias_dim[i];
+        }
+        tmp_bias.ShareDataWith(*nonbatched_bias);
+        tmp_bias.Resize(
+            {bias_first_dim, bias_dim[-3], bias_dim[-2], bias_dim[-1]});
+      }
 
-    // qktv_out = BatchedGEMM(softmax_out, V)
-    // [batch_size, seq_len_m, num_heads, seq_len_r, m_size] *
-    //               [batch_size, seq_len_m, num_heads, m_size, head_dim]
-    // -> [batch_size, seq_len_m, num_heads, seq_len_r, head_dim]
-    phi::DenseTensor* qktv_out = config->GetQKTVOut(gate_out);
-    T* qktv_out_ptr = qktv_out->data<T>();
+      // For flash_attn
+      bool is_bf16_ =
+          qkv_transpose_out.dtype() == DataType::BFLOAT16 ? true : false;
+      int64_t batch_size_ = cu_seq_q.numel() - 1;
+      int64_t total_q_ = qkv_dims[1];    // q.dims()[0]
+      int64_t total_k_ = qkv_dims[1];    // q.dims()[0]
+      int64_t num_heads_ = qkv_dims[2];  // q.dims()[1]
+      int64_t head_size_ = qkv_dims[3];  // q.dims()[2]
+      int max_seqlen_q_;
+      int max_seqlen_k_;
+      int num_splits = 0;  // 0 for an internal heuristic, which is optimal
+
+      phi::DenseTensor softmax_lse;
+      softmax_lse.Resize({batch_size_, num_heads_, max_seqlen_q_});
+      AllocWithDebugInfo<float>(
+          dev_ctx_, "flash_attn: softmax_lse", &softmax_lse);
+
+      auto gen = ctx.GetGenerator();
+      uint64_t inc = batch_size * num_heads * 32;
+      auto seed_offset_pair = gen->IncrementOffset(inc);
+
+      uint64_t seed = seed_offset_pair.first;
+      uint64_t offset = seed_offset_pair.second;
+
+      seed_offset->Resize({2});
+      auto* seed_offset_data = ctx.template HostAlloc<int64_t>(seed_offset);
+      seed_offset_data[0] = static_cast<int64_t>(seed);
+      seed_offset_data[1] = static_cast<int64_t>(offset);
+
+      uint64_t workspace_size;
+      bool succ = phi::dynload::flash_attn_fwd(
+          static_cast<const void*>(q_ptr),
+          static_cast<const void*>(k_ptr),
+          static_cast<const void*>(v_ptr),
+          nullptr,  // for calculation workspace size
+          static_cast<const void*>(cu_seq_q.data<T>()),
+          static_cast<const void*>(cu_seq_k.data<T>()),
+          total_q_,
+          total_k_,
+          batch_size_,
+          num_heads_,
+          head_size_,
+          max_seqlen_q_,
+          max_seqlen_k_,
+          /*p_dropout=*/0.f,
+          /*softmax_scale=*/static_cast<float>(1.0 / sqrt(config->head_size_));
+          /*zero_tensors=*/false,
+          /*is_causal=*/false,
+          is_bf16_,
+          num_splits,
+          softmax_lse.data();
+          softmax_out->data();
+          nullptr,
+          &workspace_size,
+          dev_ctx_.stream(),
+          seed,
+          offset,
+          src_mask ? src_mask->data() : nullptr,
+          nonbatched_bias ? nonbatched_bias->data() : nullptr,
+          src_mask->dims().Get(),
+          nonbatched_bias->dims().Get());
+      if (!succ) {
+        PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
+      }
+      DenseTensor workspace;
+      if (workspace_size > 0) {
+        workspace =
+            Empty<float>(ctx, {int64_t(workspace_size / sizeof(float))});
+      }
+      bool succ = phi::dynload::flash_attn_fwd(
+          static_cast<const void*>(q_ptr),
+          static_cast<const void*>(k_ptr),
+          static_cast<const void*>(v_ptr),
+          static_cast<void*>(fmha_out),  // for calculation workspace size
+          static_cast<const void*>(cu_seq_q.data<T>()),
+          static_cast<const void*>(cu_seq_k.data<T>()),
+          total_q_,
+          total_k_,
+          batch_size_,
+          num_heads_,
+          head_size_,
+          max_seqlen_q_,
+          max_seqlen_k_,
+          /*p_dropout=*/0.f,
+          /*softmax_scale=*/static_cast<float>(1.0 / sqrt(config->head_size_));
+          /*zero_tensors=*/false,
+          /*is_causal=*/false,
+          is_bf16_,
+          num_splits,
+          softmax_lse.data();
+          softmax_out->data();
+          workspace_size > 0 ? workspace.data() : nullptr,
+          &workspace_size,
+          dev_ctx_.stream(),
+          seed,
+          offset,
+          src_mask ? src_mask->data() : nullptr,
+          nonbatched_bias ? nonbatched_bias->data() : nullptr,
+          src_mask->dims().Get(),
+          nonbatched_bias->dims().Get());
+    } else {
+      if (merge_qkv_) {
+        // qkv_transpose_out = transpose(qkv_out)
+        PADDLE_ENFORCE_NOT_NULL(
+            qkv_transpose_out,
+            platform::errors::NotFound("The input qkv_transpose_out can not be "
+                                       "nullptr when merge_qkv is true."));
+
+        phi::DenseTensor* qkv_out = config->GetQKVOut();
+        ComputeQKVTransposeForward(*qkv_out, qkv_transpose_out);
+        config->ClearQKVOut();
+
+        // q_size == k_size
+        int64_t q_size = config->GetQuerySize();
+        q_ptr = qkv_transpose_out->data<T>();
+        k_ptr = q_ptr + q_size;
+        v_ptr = k_ptr + q_size;
+      } else {
+        PADDLE_ENFORCE_NOT_NULL(
+            q_transpose_out,
+            platform::errors::NotFound("The input q_transpose_out can not be "
+                                       "nullptr when merge_qkv is false."));
+        PADDLE_ENFORCE_NOT_NULL(
+            k_transpose_out,
+            platform::errors::NotFound("The input k_transpose_out can not be "
+                                       "nullptr when merge_qkv is false."));
+        PADDLE_ENFORCE_NOT_NULL(
+            v_transpose_out,
+            platform::errors::NotFound("The input v_transpose_out can not be "
+                                       "nullptr when merge_qkv is false."));
+
+        phi::DenseTensor* query_out = config->GetQueryOut();
+        phi::DenseTensor* key_out = config->GetKeyOut();
+        phi::DenseTensor* value_out = config->GetValueOut();
+        ComputeQKVTransposeForward(*query_out,
+                                   *key_out,
+                                   *value_out,
+                                   q_transpose_out,
+                                   k_transpose_out,
+                                   v_transpose_out);
+
+        // q_size != k_size
+        q_ptr = q_transpose_out->data<T>();
+        k_ptr = k_transpose_out->data<T>();
+        v_ptr = v_transpose_out->data<T>();
+      }
+      // qk_out = BatchedGEMM(Q, K^T)
+      // [batch_size, seq_len_m, num_heads, seq_len_r, head_dim] *
+      //                [batch_size, seq_len_m, num_heads, m_size, head_dim]
+      // -> [batch_size, seq_len_m, num_heads, seq_len_r, m_size]
+      phi::DenseTensor* qk_out = config->GetQKOut(softmax_out);
+      T* qk_out_ptr = qk_out->data<T>();
+
+      int64_t gemm_batch_size =
+          config->batch_size * config->seq_len_m * config->num_heads;
+      int64_t gemm_m = config->seq_len_r;
+      int64_t gemm_n = config->m_size;
+      int64_t gemm_k = config->head_dim;
+      // attn = torch.matmul(q, k.transpose(-1, -2))
+      T alpha = static_cast<T>(1.0 / sqrt(config->head_dim));
+      ComputeBatchedGEMM(q_ptr,
+                         k_ptr,
+                         qk_out_ptr,
+                         false,
+                         true,
+                         gemm_m,
+                         gemm_n,
+                         gemm_k,
+                         gemm_batch_size,
+                         alpha);
+      // attn = softmax_dropout(attn, 0, self.training, mask=mask, bias=bias)
+      // softmax_out = softmax(qk_out + nonbatched_bias + src_mask)
+      ComputeBiasMaskSoftmaxForward(
+          nonbatched_bias, src_mask, qk_out, softmax_out);
+      config->ClearQKOut();
+
+      // qktv_out = BatchedGEMM(softmax_out, V)
+      // [batch_size, seq_len_m, num_heads, seq_len_r, m_size] *
+      //               [batch_size, seq_len_m, num_heads, m_size, head_dim]
+      // -> [batch_size, seq_len_m, num_heads, seq_len_r, head_dim]
+      phi::DenseTensor* qktv_out = config->GetQKTVOut(gate_out);
+      T* qktv_out_ptr = qktv_out->data<T>();
 
-    gemm_m = config->seq_len_r;
-    gemm_n = config->head_dim;
-    gemm_k = config->m_size;
+      gemm_m = config->seq_len_r;
+      gemm_n = config->head_dim;
+      gemm_k = config->m_size;
 
-    T* softmax_out_ptr = softmax_out->data<T>();
-    ComputeBatchedGEMM(softmax_out_ptr,
-                       v_ptr,
-                       qktv_out_ptr,
-                       false,
-                       false,
-                       gemm_m,
-                       gemm_n,
-                       gemm_k,
-                       gemm_batch_size);
+      // o = torch.matmul(attn, v)
+      T* softmax_out_ptr = softmax_out->data<T>();
+      ComputeBatchedGEMM(softmax_out_ptr,
+                         v_ptr,
+                         qktv_out_ptr,
+                         false,
+                         false,
+                         gemm_m,
+                         gemm_n,
+                         gemm_k,
+                         gemm_batch_size);
+      // fmha_out = transpose(qktv_out)
+      // o = o.transpose(-2, -3).contiguous()
+      ComputeQKTVTransposeForward(*qktv_out, fmha_out);
+    }
 
-    // fmha_out = transpose(qktv_out)
-    ComputeQKTVTransposeForward(*qktv_out, fmha_out);
     config->ClearQKTVOut();
     if (config->has_gating) {
       gate_out->Resize(config->gate_out_dims);
diff --git a/paddle/phi/backends/dynload/flashattn.h b/paddle/phi/backends/dynload/flashattn.h
index ec443fd9f8e8e..87836a49a700d 100644
--- a/paddle/phi/backends/dynload/flashattn.h
+++ b/paddle/phi/backends/dynload/flashattn.h
@@ -46,7 +46,8 @@ extern void* flashattn_dso_handle;
 #define FLASHATTN_ROUTINE_EACH(__macro) \
   __macro(flash_attn_fwd);              \
   __macro(flash_attn_bwd);              \
-  __macro(flash_attn_error);
+  __macro(flash_attn_error);            \
+  __macro(flash_attn_fwd_with_bias_and_mask);
 
 FLASHATTN_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_FLASHATTN_WRAP);
 

From 76b87c2855b97116cef4f2b4404a8ab89eec6df3 Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Tue, 18 Apr 2023 18:43:05 +0800
Subject: [PATCH 002/405] fix some bugs

---
 cmake/external/flashattn.cmake                |   6 +-
 .../operators/fused/fused_gate_attention.h    | 291 ++++++++++++------
 paddle/phi/backends/dynload/flashattn.h       |   4 +-
 3 files changed, 205 insertions(+), 96 deletions(-)

diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index 53b11da6a6732..a37559c483421 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -20,10 +20,10 @@ set(FLASHATTN_PREFIX_DIR ${THIRD_PARTY_PATH}/flashattn)
 set(FLASHATTN_SOURCE_SUBDIR csrc/flash_attn)
 set(FLASHATTN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flashattn)
 set(FLASHATTN_REPOSITORY ${GIT_URL}/JamesLim-sy/flash-attention.git)
-set(FLASHATTN_TAG 05ecd140356cffe46444cd364944f6ec5067e724)
+set(FLASHATTN_TAG c7442255b553e8f8ddee21b48a4bca992678cb89)
 
-#set(FLASHATTN_REPOSITORY ${GIT_URL}/PaddlePaddle/flash-attention.git)
-#set(FLASHATTN_TAG f0edf243a813a65d05c75fcb331b2a95faf96bbc)
+# set(FLASHATTN_REPOSITORY ${GIT_URL}/PaddlePaddle/flash-attention.git)
+# set(FLASHATTN_TAG f0edf243a813a65d05c75fcb331b2a95faf96bbc)
 
 set(FLASHATTN_INCLUDE_DIR
     "${FLASHATTN_INSTALL_DIR}/include"
diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 7f634e835cadf..fcbebdda20a52 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -21,6 +21,13 @@ limitations under the License. */
 #include "paddle/phi/kernels/funcs/reduce_function.h"
 #include "paddle/phi/kernels/funcs/transpose_function.cu.h"
 #include "paddle/phi/kernels/gpudnn/softmax_gpudnn.h"
+#include "paddle/phi/kernels/funcs/range_function.h"
+#include "paddle/phi/kernels/empty_kernel.h"
+
+#ifdef PADDLE_WITH_FLASHATTN
+#include "paddle/phi/kernels/flash_attn_kernel.h"
+#include "paddle/phi/backends/dynload/flashattn.h"
+#endif
 
 namespace paddle {
 namespace operators {
@@ -147,7 +154,11 @@ struct GateAttentionConfig {
     gate_out_dims = {batch_size, seq_len_m, seq_len_r, num_heads, head_dim};
   }
 
-  bool UseFlashAttn(const bool merge_qkv, const int64_t head_dim) {
+  bool UseFlashAttn(const bool merge_qkv, const bool is_amp) {
+    if (!is_amp) { 
+      return false;
+    }
+
     if (merge_qkv) {
       switch (head_dim) {
         case 16:
@@ -334,15 +345,53 @@ struct GateAttentionGradConfig : public GateAttentionConfig<T> {
   phi::DenseTensor qk_out_grad;
 };
 
-template <typename T, typename OUT_TYPE>
+
+#define DEBUG_HERE printf("[%s, %d]: Run here!\n", __func__, __LINE__);
+#define DEBUG_DATA_INT(name, x)  do { \
+  printf("[%s, %d]: %s = %d\n", __func__, __LINE__, name, static_cast<int>(x)); \
+} whilie(0);
+
+#define DEBUG_DATA_FlOAT(name, x) do {\
+  printf("[%s, %d]: %s = %f\n", __func__, __LINE__, std::string(name), static_cast<float>(x)); \
+} whilie(0);
+
+#define DEBUG_DIMS(x) do { \
+  printf("[%s, %d]: dims is : [", __func__, __LINE__); \
+  for (int i = 0; i < x.size(); ++i) { \
+    printf("%d, ", x[i]);              \
+  }                                    \
+  printf(" ]\n");                      \
+} whilie(0);
+
+
+template <typename T>
 __global__ void FlashAttRange(
-    int start, int step, int size, OUT_TYPE* out1, OUT_TYPE* out2) {
+    int start, int step, int size, T* out1, T* out2) {
   CUDA_KERNEL_LOOP(index, size) {
-    out1[index] = static_cast<OUT_TYPE>(start + step * index);
-    out2[index] = static_cast<OUT_TYPE>(start + step * index);
+    out1[index] = static_cast<T>(start + step * index);
+    out2[index] = static_cast<T>(start + step * index);
   }
 }
 
+static void GetFlashAttnDimsString(const std::string& prefix,
+                                   const phi::DDim dim_val) {   
+  // if (VLOG_IS_ON(4)) {
+    std::ostringstream out_string;
+    out_string << "FlashAttn - " << prefix << ".dims() is [ ";
+    for (int i = 0; i < dim_val.size(); ++i) {
+        out_string << dim_val[i] << ", ";
+    }
+    out_string << "]\n";
+    VLOG(4) << out_string.str();
+    std::cout << out_string.str();
+  // }
+}
+
+#define DBG_WAIT  do {\
+  printf("[%s, %d] Run here.\n", __func__, __LINE__); \
+  dev_ctx_.Wait(); \
+} while(0);
+
 template <typename T>
 class FMHAGateRef {
  public:
@@ -362,30 +411,46 @@ class FMHAGateRef {
     T* q_ptr = nullptr;
     T* k_ptr = nullptr;
     T* v_ptr = nullptr;
+    bool is_bf16 = qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
+
+    if (std::is_same<T, phi::dtype::float16>::value) {
+      std::cout << "T is phi::dtype::float16. \n";
+    } else if (std::is_same<T, phi::dtype::bfloat16>::value) {
+      std::cout << "T is phi::dtype::bfloat16. \n";
+    } else if (std::is_same<T, float>::value) {
+      std::cout << "T is float. \n";
+    }
 
-    if (config->UseFlashAttn(merge_qkv_, config->head_dim)) {
+    if (config->UseFlashAttn(merge_qkv_, is_bf16)) {
       PADDLE_ENFORCE_NOT_NULL(
           qkv_transpose_out,
           platform::errors::NotFound("The input qkv_transpose_out can not be "
                                      "nullptr when merge_qkv is true."));
       phi::DenseTensor* qkv_out = config->GetQKVOut();
-      ComputeQKVTransposeForward(*qkv_out, qkv_transpose_out);
+      ComputeQKVTransposeForwardForFlashAttn(*qkv_out, qkv_transpose_out);
       config->ClearQKVOut();
+      
+      dev_ctx_.Wait();
+      qkv_transpose_out->Resize({3,
+          static_cast<int>(config->batch_size),
+          static_cast<int>(config->seq_len_m), 
+          static_cast<int>(config->seq_len_r),
+          static_cast<int>(config->num_heads),
+          static_cast<int>(config->head_dim)});
 
-      // batch_dims = q.shape[:-3];
-      // n, no_heads, c = q.shape[-3:]
       // 1. Dealing with qkv_out for flash_attn.
-      auto& qkv_dims = qkv_transpose_out.dims();
+      auto& qkv_dims = qkv_transpose_out->dims();
       auto rank = qkv_dims.size();
-      auto n = qkv_dims[rank - 3];
-      auto no_heads = qkv_dims[rank - 2];
-      auto c = qkv_dims[rank - 1];
-
       int64_t q_batch_size = 1;
-      for (int i = 0; i < (rank - 3); ++i) {
+      int64_t rest_dim = qkv_dims[rank - 3];
+      for (int i = 1; i < (rank - 3); ++i) {
         q_batch_size *= qkv_dims[i];
       }
-      qkv_transpose_out.Resize({q_batch_size * n, no_heads, c});
+      qkv_transpose_out->Resize({3, 
+                                 q_batch_size * rest_dim,
+                                 config->num_heads,
+                                 config->head_dim});
+      DBG_WAIT;
       // q_size == k_size
       int64_t q_size = config->GetQuerySize();
       q_ptr = qkv_transpose_out->data<T>();
@@ -394,80 +459,106 @@ class FMHAGateRef {
 
       // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
       phi::DenseTensor cu_seq_q, cu_seq_k;
-      int64_t temp_size = (q_batch_size + 1) * n;
+      int64_t end_size = (q_batch_size + 1);
       int64_t seq_size = 0;
-      int64_t start = 0, end = temp_size, step = n;
+      int64_t start = 0, end = end_size, step = rest_dim;
       phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
-      cu_seq_q.Resize({seq_size});
-      cu_seq_k.Resize({seq_size});
+      cu_seq_q.Resize({end_size});
+      cu_seq_k.Resize({end_size});
       AllocWithDebugInfo<int32_t>(dev_ctx_, "flash_attn: cu_seq_q", &cu_seq_q);
       AllocWithDebugInfo<int32_t>(dev_ctx_, "flash_attn: cu_seq_k", &cu_seq_k);
-
       int64_t block = std::min(seq_size, static_cast<int64_t>(256));
-      int64_t grid = (range_size + block - 1) / block;
-      FlashAttRange<T><<<grid, block, 0, dev_ctx_.stream()>>>(
-          start, step, end, ` cu_seq_q.data<T>(), cu_seq_k.data<T>());
-
-      // 1. Dealing with mask and bias for flash_attn.
-      phi::DenseTensor tmp_mask, tmp_bias;
+      int64_t grid = (seq_size + block - 1) / block;
+      FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
+          start, step, end, cu_seq_q.data<int32_t>(), cu_seq_k.data<int32_t>());
+      VLOG(4) << "[Flash_attn] cu_seq_len : start = " << start
+              << ", step = " << step << ", end = " << end;
+      DBG_WAIT;
+
+      // 3. Dealing with mask and bias for flash_attn.
+      phi::DenseTensor temp_mask, temp_bias;
       if (src_mask) {
-        int64_t mask_first_dim = 0;
-        auto& mask_dim = src_mask->dims();
+        int64_t mask_first_dim = 1;
+        temp_mask.ShareDataWith(*src_mask);
+
+        auto mask_dim = temp_mask.dims();
         for (int i = 0; i < mask_dim.size() - 3; ++i) {
           mask_first_dim *= mask_dim[i];
         }
-        tmp_mask.ShareDataWith(*src_mask);
-        tmp_mask.Resize(
-            {mask_first_dim, mask_dim[-3], mask_dim[-2], mask_dim[-1]});
+        auto mask_dim_rank = mask_dim.size();
+        temp_mask.Resize({mask_first_dim, 
+                         mask_dim[mask_dim_rank - 3],
+                         mask_dim[mask_dim_rank - 2],
+                         mask_dim[mask_dim_rank - 1]});
+        GetFlashAttnDimsString("mask_dim", temp_mask.dims());
       }
       if (nonbatched_bias) {
-        int64_t bias_first_dim = 0;
-        auto& bias_dim = nonbatched_bias->dims();
+        int64_t bias_first_dim = 1;
+        temp_bias.ShareDataWith(*nonbatched_bias);
+
+        auto bias_dim = nonbatched_bias->dims();
         for (int i = 0; i < bias_dim.size() - 3; ++i) {
           bias_first_dim *= bias_dim[i];
         }
-        tmp_bias.ShareDataWith(*nonbatched_bias);
-        tmp_bias.Resize(
-            {bias_first_dim, bias_dim[-3], bias_dim[-2], bias_dim[-1]});
+        auto bias_dim_rank = temp_bias.dims().size();
+        temp_bias.Resize({bias_first_dim,
+                          bias_dim[bias_dim_rank - 3],
+                          bias_dim[bias_dim_rank - 2],
+                          bias_dim[bias_dim_rank - 1]});
+        GetFlashAttnDimsString("bias_dim", temp_bias.dims());
       }
-
-      // For flash_attn
-      bool is_bf16_ =
-          qkv_transpose_out.dtype() == DataType::BFLOAT16 ? true : false;
-      int64_t batch_size_ = cu_seq_q.numel() - 1;
-      int64_t total_q_ = qkv_dims[1];    // q.dims()[0]
-      int64_t total_k_ = qkv_dims[1];    // q.dims()[0]
-      int64_t num_heads_ = qkv_dims[2];  // q.dims()[1]
-      int64_t head_size_ = qkv_dims[3];  // q.dims()[2]
-      int max_seqlen_q_;
-      int max_seqlen_k_;
+      DBG_WAIT;
+      GetFlashAttnDimsString("qkv_transpose_out", qkv_transpose_out->dims());
+
+      // 4. flash_attn parameter setting.
+      int batch_size_ = q_batch_size;
+      int total_q_ = qkv_dims[1];    // q.dims()[0]
+      int total_k_ = qkv_dims[1];    // q.dims()[0]
+      int num_heads_ = qkv_dims[2];  // q.dims()[1]
+      int head_size_ = qkv_dims[3];  // q.dims()[2]
+      int max_seqlen_q_ = batch_size_;
+      int max_seqlen_k_ = batch_size_;
       int num_splits = 0;  // 0 for an internal heuristic, which is optimal
-
+      DBG_WAIT;
+      
+      VLOG(6) << "[Flash_attn] batch_size : " << batch_size_;
+      VLOG(6) << "[Flash_attn] total_q   : " << total_q_;
+      VLOG(6) << "[Flash_attn] total_k   : " << total_k_;
+      VLOG(6) << "[Flash_attn] num_heads : " << num_heads_;
+      VLOG(6) << "[Flash_attn] head_size : " << head_size_;
+      VLOG(6) << "[Flash_attn] max_seqlen_q : " << max_seqlen_q_;
+      VLOG(6) << "[Flash_attn] max_seqlen_k : " << max_seqlen_k_;
+
+      // 5. construct softmax_lse
       phi::DenseTensor softmax_lse;
-      softmax_lse.Resize({batch_size_, num_heads_, max_seqlen_q_});
-      AllocWithDebugInfo<float>(
-          dev_ctx_, "flash_attn: softmax_lse", &softmax_lse);
-
-      auto gen = ctx.GetGenerator();
-      uint64_t inc = batch_size * num_heads * 32;
+      int softmax_lse_last_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
+      softmax_lse.Resize({batch_size_, num_heads_, softmax_lse_last_dim});
+      AllocWithDebugInfo<float>(dev_ctx_, "flash_attn: softmax_lse", &softmax_lse);
+
+      DBG_WAIT;
+      // 6. construct random seed
+      auto gen = dev_ctx_.GetGenerator();
+      uint64_t inc = batch_size_ * num_heads_ * 32;
       auto seed_offset_pair = gen->IncrementOffset(inc);
-
       uint64_t seed = seed_offset_pair.first;
       uint64_t offset = seed_offset_pair.second;
 
-      seed_offset->Resize({2});
-      auto* seed_offset_data = ctx.template HostAlloc<int64_t>(seed_offset);
-      seed_offset_data[0] = static_cast<int64_t>(seed);
-      seed_offset_data[1] = static_cast<int64_t>(offset);
+      GetFlashAttnDimsString("softmax_out", softmax_out->dims());
+      GetFlashAttnDimsString("softmax_lse", softmax_lse.dims());
+      DBG_WAIT;
 
+      // 7. flas_attn part one, get temp worksapce size.
+      float p_dropout = 0.f;
+      float softmax_scale = static_cast<float>(1);
+      cudaStream_t stream = dev_ctx_.stream();
       uint64_t workspace_size;
-      bool succ = phi::dynload::flash_attn_fwd(
+      bool succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
           static_cast<const void*>(q_ptr),
           static_cast<const void*>(k_ptr),
           static_cast<const void*>(v_ptr),
           nullptr,  // for calculation workspace size
-          static_cast<const void*>(cu_seq_q.data<T>()),
-          static_cast<const void*>(cu_seq_k.data<T>()),
+          static_cast<const void*>(cu_seq_q.data<int32_t>()),
+          static_cast<const void*>(cu_seq_k.data<int32_t>()),
           total_q_,
           total_k_,
           batch_size_,
@@ -475,38 +566,41 @@ class FMHAGateRef {
           head_size_,
           max_seqlen_q_,
           max_seqlen_k_,
-          /*p_dropout=*/0.f,
-          /*softmax_scale=*/static_cast<float>(1.0 / sqrt(config->head_size_));
+          p_dropout,
+          softmax_scale,
           /*zero_tensors=*/false,
           /*is_causal=*/false,
-          is_bf16_,
+          is_bf16,
           num_splits,
-          softmax_lse.data();
-          softmax_out->data();
+          softmax_lse.data(),
+          softmax_out->data(),
           nullptr,
           &workspace_size,
-          dev_ctx_.stream(),
+          stream,
           seed,
           offset,
-          src_mask ? src_mask->data() : nullptr,
-          nonbatched_bias ? nonbatched_bias->data() : nullptr,
-          src_mask->dims().Get(),
-          nonbatched_bias->dims().Get());
+          src_mask ? temp_mask.data() : nullptr,
+          nonbatched_bias ? temp_bias.data() : nullptr,
+          temp_mask.dims().Get(),
+          temp_bias.dims().Get()
+      );
       if (!succ) {
         PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
       }
-      DenseTensor workspace;
+      DBG_WAIT;
+      phi::DenseTensor workspace;
       if (workspace_size > 0) {
-        workspace =
-            Empty<float>(ctx, {int64_t(workspace_size / sizeof(float))});
+        workspace = phi::Empty<float>(dev_ctx_, {int64_t(workspace_size / sizeof(float))});
       }
-      bool succ = phi::dynload::flash_attn_fwd(
+      DBG_WAIT;
+      // 8. flas_attn part two, run impl.
+      succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
           static_cast<const void*>(q_ptr),
           static_cast<const void*>(k_ptr),
           static_cast<const void*>(v_ptr),
           static_cast<void*>(fmha_out),  // for calculation workspace size
-          static_cast<const void*>(cu_seq_q.data<T>()),
-          static_cast<const void*>(cu_seq_k.data<T>()),
+          static_cast<const void*>(cu_seq_q.data<int32_t>()),
+          static_cast<const void*>(cu_seq_k.data<int32_t>()),
           total_q_,
           total_k_,
           batch_size_,
@@ -514,23 +608,29 @@ class FMHAGateRef {
           head_size_,
           max_seqlen_q_,
           max_seqlen_k_,
-          /*p_dropout=*/0.f,
-          /*softmax_scale=*/static_cast<float>(1.0 / sqrt(config->head_size_));
+          p_dropout,
+          softmax_scale,
           /*zero_tensors=*/false,
           /*is_causal=*/false,
-          is_bf16_,
+          is_bf16,
           num_splits,
-          softmax_lse.data();
-          softmax_out->data();
-          workspace_size > 0 ? workspace.data() : nullptr,
+          softmax_lse.data(),
+          softmax_out->data(),
+          workspace_size > 0 ? static_cast<void*>(workspace.data()) : nullptr,
           &workspace_size,
-          dev_ctx_.stream(),
+          stream,
           seed,
           offset,
-          src_mask ? src_mask->data() : nullptr,
-          nonbatched_bias ? nonbatched_bias->data() : nullptr,
-          src_mask->dims().Get(),
-          nonbatched_bias->dims().Get());
+          src_mask ? temp_mask.data() : nullptr,
+          nonbatched_bias ? temp_bias.data() : nullptr,
+          temp_mask.dims().Get(),
+          temp_bias.dims().Get()
+      );
+      DBG_WAIT;
+      if (!succ) {
+        PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
+      }
+      DBG_WAIT;
     } else {
       if (merge_qkv_) {
         // qkv_transpose_out = transpose(qkv_out)
@@ -609,7 +709,7 @@ class FMHAGateRef {
 
       // qktv_out = BatchedGEMM(softmax_out, V)
       // [batch_size, seq_len_m, num_heads, seq_len_r, m_size] *
-      //               [batch_size, seq_len_m, num_heads, m_size, head_dim]
+      // [batch_size, seq_len_m, num_heads, m_size,    head_dim]
       // -> [batch_size, seq_len_m, num_heads, seq_len_r, head_dim]
       phi::DenseTensor* qktv_out = config->GetQKTVOut(gate_out);
       T* qktv_out_ptr = qktv_out->data<T>();
@@ -854,6 +954,15 @@ class FMHAGateRef {
         dev_ctx_, qkv_out, perm, qkv_transpose_out);
   }
 
+  // [batch_size, seq_len_m, seq_len_r, 3, num_heads, head_dim] ->
+  //         [3, batch_size, seq_len_m, seq_len_r, num_heads, head_dim]
+  void ComputeQKVTransposeForwardForFlashAttn(const phi::DenseTensor& qkv_out,
+                                              phi::DenseTensor* qkv_transpose_out) {
+    std::vector<int> perm = {3, 0, 1, 2, 4, 5};
+    phi::funcs::TransposeGPUKernelDriver<T>(
+        dev_ctx_, qkv_out, perm, qkv_transpose_out);
+  }
+
   void ComputeQKVTransposeBackward(
       const phi::DenseTensor& qkv_transpose_out_grad,
       phi::DenseTensor* qkv_out_grad) {
diff --git a/paddle/phi/backends/dynload/flashattn.h b/paddle/phi/backends/dynload/flashattn.h
index 87836a49a700d..1d235fcdecfc5 100644
--- a/paddle/phi/backends/dynload/flashattn.h
+++ b/paddle/phi/backends/dynload/flashattn.h
@@ -46,8 +46,8 @@ extern void* flashattn_dso_handle;
 #define FLASHATTN_ROUTINE_EACH(__macro) \
   __macro(flash_attn_fwd);              \
   __macro(flash_attn_bwd);              \
-  __macro(flash_attn_error);            \
-  __macro(flash_attn_fwd_with_bias_and_mask);
+  __macro(flash_attn_error);
+  __macro(flash_attn_fwd_with_bias_and_mask); \
 
 FLASHATTN_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_FLASHATTN_WRAP);
 

From a26df023f28d39d0288736b17dc2861988da9e44 Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Tue, 18 Apr 2023 18:44:00 +0800
Subject: [PATCH 003/405] fix some bugs

---
 .../operators/fused/fused_gate_attention.h    | 118 ++++++++++--------
 1 file changed, 63 insertions(+), 55 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index fcbebdda20a52..b58e9dbc59215 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -15,18 +15,18 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/phi/kernels/arange_kernel.h"
+#include "paddle/phi/kernels/empty_kernel.h"
 #include "paddle/phi/kernels/funcs/broadcast_function.h"
 #include "paddle/phi/kernels/funcs/elementwise_base.h"
 #include "paddle/phi/kernels/funcs/elementwise_functor.h"
+#include "paddle/phi/kernels/funcs/range_function.h"
 #include "paddle/phi/kernels/funcs/reduce_function.h"
 #include "paddle/phi/kernels/funcs/transpose_function.cu.h"
 #include "paddle/phi/kernels/gpudnn/softmax_gpudnn.h"
-#include "paddle/phi/kernels/funcs/range_function.h"
-#include "paddle/phi/kernels/empty_kernel.h"
 
 #ifdef PADDLE_WITH_FLASHATTN
-#include "paddle/phi/kernels/flash_attn_kernel.h"
 #include "paddle/phi/backends/dynload/flashattn.h"
+#include "paddle/phi/kernels/flash_attn_kernel.h"
 #endif
 
 namespace paddle {
@@ -155,7 +155,7 @@ struct GateAttentionConfig {
   }
 
   bool UseFlashAttn(const bool merge_qkv, const bool is_amp) {
-    if (!is_amp) { 
+    if (!is_amp) {
       return false;
     }
 
@@ -345,28 +345,36 @@ struct GateAttentionGradConfig : public GateAttentionConfig<T> {
   phi::DenseTensor qk_out_grad;
 };
 
-
 #define DEBUG_HERE printf("[%s, %d]: Run here!\n", __func__, __LINE__);
-#define DEBUG_DATA_INT(name, x)  do { \
-  printf("[%s, %d]: %s = %d\n", __func__, __LINE__, name, static_cast<int>(x)); \
-} whilie(0);
-
-#define DEBUG_DATA_FlOAT(name, x) do {\
-  printf("[%s, %d]: %s = %f\n", __func__, __LINE__, std::string(name), static_cast<float>(x)); \
-} whilie(0);
-
-#define DEBUG_DIMS(x) do { \
-  printf("[%s, %d]: dims is : [", __func__, __LINE__); \
-  for (int i = 0; i < x.size(); ++i) { \
-    printf("%d, ", x[i]);              \
-  }                                    \
-  printf(" ]\n");                      \
-} whilie(0);
-
+#define DEBUG_DATA_INT(name, x)                                                \
+  do {                                                                         \
+    printf(                                                                    \
+        "[%s, %d]: %s = %d\n", __func__, __LINE__, name, static_cast<int>(x)); \
+  }                                                                            \
+  whilie(0);
+
+#define DEBUG_DATA_FlOAT(name, x)  \
+  do {                             \
+    printf("[%s, %d]: %s = %f\n",  \
+           __func__,               \
+           __LINE__,               \
+           std::string(name),      \
+           static_cast<float>(x)); \
+  }                                \
+  whilie(0);
+
+#define DEBUG_DIMS(x)                                    \
+  do {                                                   \
+    printf("[%s, %d]: dims is : [", __func__, __LINE__); \
+    for (int i = 0; i < x.size(); ++i) {                 \
+      printf("%d, ", x[i]);                              \
+    }                                                    \
+    printf(" ]\n");                                      \
+  }                                                      \
+  whilie(0);
 
 template <typename T>
-__global__ void FlashAttRange(
-    int start, int step, int size, T* out1, T* out2) {
+__global__ void FlashAttRange(int start, int step, int size, T* out1, T* out2) {
   CUDA_KERNEL_LOOP(index, size) {
     out1[index] = static_cast<T>(start + step * index);
     out2[index] = static_cast<T>(start + step * index);
@@ -374,23 +382,24 @@ __global__ void FlashAttRange(
 }
 
 static void GetFlashAttnDimsString(const std::string& prefix,
-                                   const phi::DDim dim_val) {   
-  // if (VLOG_IS_ON(4)) {
+                                   const phi::DDim dim_val) {
+  if (VLOG_IS_ON(4)) {
     std::ostringstream out_string;
     out_string << "FlashAttn - " << prefix << ".dims() is [ ";
     for (int i = 0; i < dim_val.size(); ++i) {
-        out_string << dim_val[i] << ", ";
+      out_string << dim_val[i] << ", ";
     }
     out_string << "]\n";
     VLOG(4) << out_string.str();
     std::cout << out_string.str();
-  // }
+  }
 }
 
-#define DBG_WAIT  do {\
-  printf("[%s, %d] Run here.\n", __func__, __LINE__); \
-  dev_ctx_.Wait(); \
-} while(0);
+#define DBG_WAIT                                        \
+  do {                                                  \
+    printf("[%s, %d] Run here.\n", __func__, __LINE__); \
+    dev_ctx_.Wait();                                    \
+  } while (0);
 
 template <typename T>
 class FMHAGateRef {
@@ -411,7 +420,8 @@ class FMHAGateRef {
     T* q_ptr = nullptr;
     T* k_ptr = nullptr;
     T* v_ptr = nullptr;
-    bool is_bf16 = qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
+    bool is_bf16 =
+        qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
 
     if (std::is_same<T, phi::dtype::float16>::value) {
       std::cout << "T is phi::dtype::float16. \n";
@@ -429,14 +439,14 @@ class FMHAGateRef {
       phi::DenseTensor* qkv_out = config->GetQKVOut();
       ComputeQKVTransposeForwardForFlashAttn(*qkv_out, qkv_transpose_out);
       config->ClearQKVOut();
-      
+
       dev_ctx_.Wait();
       qkv_transpose_out->Resize({3,
-          static_cast<int>(config->batch_size),
-          static_cast<int>(config->seq_len_m), 
-          static_cast<int>(config->seq_len_r),
-          static_cast<int>(config->num_heads),
-          static_cast<int>(config->head_dim)});
+                                 static_cast<int>(config->batch_size),
+                                 static_cast<int>(config->seq_len_m),
+                                 static_cast<int>(config->seq_len_r),
+                                 static_cast<int>(config->num_heads),
+                                 static_cast<int>(config->head_dim)});
 
       // 1. Dealing with qkv_out for flash_attn.
       auto& qkv_dims = qkv_transpose_out->dims();
@@ -446,10 +456,8 @@ class FMHAGateRef {
       for (int i = 1; i < (rank - 3); ++i) {
         q_batch_size *= qkv_dims[i];
       }
-      qkv_transpose_out->Resize({3, 
-                                 q_batch_size * rest_dim,
-                                 config->num_heads,
-                                 config->head_dim});
+      qkv_transpose_out->Resize(
+          {3, q_batch_size * rest_dim, config->num_heads, config->head_dim});
       DBG_WAIT;
       // q_size == k_size
       int64_t q_size = config->GetQuerySize();
@@ -486,10 +494,10 @@ class FMHAGateRef {
           mask_first_dim *= mask_dim[i];
         }
         auto mask_dim_rank = mask_dim.size();
-        temp_mask.Resize({mask_first_dim, 
-                         mask_dim[mask_dim_rank - 3],
-                         mask_dim[mask_dim_rank - 2],
-                         mask_dim[mask_dim_rank - 1]});
+        temp_mask.Resize({mask_first_dim,
+                          mask_dim[mask_dim_rank - 3],
+                          mask_dim[mask_dim_rank - 2],
+                          mask_dim[mask_dim_rank - 1]});
         GetFlashAttnDimsString("mask_dim", temp_mask.dims());
       }
       if (nonbatched_bias) {
@@ -520,7 +528,7 @@ class FMHAGateRef {
       int max_seqlen_k_ = batch_size_;
       int num_splits = 0;  // 0 for an internal heuristic, which is optimal
       DBG_WAIT;
-      
+
       VLOG(6) << "[Flash_attn] batch_size : " << batch_size_;
       VLOG(6) << "[Flash_attn] total_q   : " << total_q_;
       VLOG(6) << "[Flash_attn] total_k   : " << total_k_;
@@ -533,7 +541,8 @@ class FMHAGateRef {
       phi::DenseTensor softmax_lse;
       int softmax_lse_last_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
       softmax_lse.Resize({batch_size_, num_heads_, softmax_lse_last_dim});
-      AllocWithDebugInfo<float>(dev_ctx_, "flash_attn: softmax_lse", &softmax_lse);
+      AllocWithDebugInfo<float>(
+          dev_ctx_, "flash_attn: softmax_lse", &softmax_lse);
 
       DBG_WAIT;
       // 6. construct random seed
@@ -582,15 +591,15 @@ class FMHAGateRef {
           src_mask ? temp_mask.data() : nullptr,
           nonbatched_bias ? temp_bias.data() : nullptr,
           temp_mask.dims().Get(),
-          temp_bias.dims().Get()
-      );
+          temp_bias.dims().Get());
       if (!succ) {
         PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
       }
       DBG_WAIT;
       phi::DenseTensor workspace;
       if (workspace_size > 0) {
-        workspace = phi::Empty<float>(dev_ctx_, {int64_t(workspace_size / sizeof(float))});
+        workspace = phi::Empty<float>(
+            dev_ctx_, {int64_t(workspace_size / sizeof(float))});
       }
       DBG_WAIT;
       // 8. flas_attn part two, run impl.
@@ -624,8 +633,7 @@ class FMHAGateRef {
           src_mask ? temp_mask.data() : nullptr,
           nonbatched_bias ? temp_bias.data() : nullptr,
           temp_mask.dims().Get(),
-          temp_bias.dims().Get()
-      );
+          temp_bias.dims().Get());
       DBG_WAIT;
       if (!succ) {
         PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
@@ -956,8 +964,8 @@ class FMHAGateRef {
 
   // [batch_size, seq_len_m, seq_len_r, 3, num_heads, head_dim] ->
   //         [3, batch_size, seq_len_m, seq_len_r, num_heads, head_dim]
-  void ComputeQKVTransposeForwardForFlashAttn(const phi::DenseTensor& qkv_out,
-                                              phi::DenseTensor* qkv_transpose_out) {
+  void ComputeQKVTransposeForwardForFlashAttn(
+      const phi::DenseTensor& qkv_out, phi::DenseTensor* qkv_transpose_out) {
     std::vector<int> perm = {3, 0, 1, 2, 4, 5};
     phi::funcs::TransposeGPUKernelDriver<T>(
         dev_ctx_, qkv_out, perm, qkv_transpose_out);

From cc0889933c0f34c5e6c0c75321cec7bb11043857 Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Wed, 19 Apr 2023 19:15:50 +0800
Subject: [PATCH 004/405] fix bugs in flashattn.h

---
 .../operators/fused/fused_gate_attention.h    | 207 ++++++++++++------
 .../fused/fused_gate_attention_op.cc          |   9 +
 .../fused/fused_gate_attention_op.cu          |  11 +-
 paddle/phi/backends/dynload/flashattn.h       |   8 +-
 .../phi/kernels/funcs/blas/blaslt_impl.cu.h   |  46 +++-
 5 files changed, 196 insertions(+), 85 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index b58e9dbc59215..58ab2d9e378a2 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -16,6 +16,7 @@ limitations under the License. */
 
 #include "paddle/phi/kernels/arange_kernel.h"
 #include "paddle/phi/kernels/empty_kernel.h"
+#include "paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h"
 #include "paddle/phi/kernels/funcs/broadcast_function.h"
 #include "paddle/phi/kernels/funcs/elementwise_base.h"
 #include "paddle/phi/kernels/funcs/elementwise_functor.h"
@@ -69,6 +70,7 @@ struct GateAttentionConfig {
 
   bool merge_qkv;
   bool has_gating;
+  bool use_flash_attn;
 
   int64_t batch_size;
   int64_t seq_len_m;
@@ -98,8 +100,12 @@ struct GateAttentionConfig {
                       const phi::DenseTensor* query_weight,
                       const phi::DenseTensor* qkv_weight,
                       bool merge_qkv,
-                      bool has_gating)
-      : dev_ctx(dev_ctx), merge_qkv(merge_qkv), has_gating(has_gating) {
+                      bool has_gating,
+                      bool use_flash_attn = false)
+      : dev_ctx(dev_ctx),
+        merge_qkv(merge_qkv),
+        has_gating(has_gating),
+        use_flash_attn(use_flash_attn) {
     // query: shape=[batch_size, seq_len_m, seq_len_r, q_dim]
     batch_size = query->dims()[0];
     seq_len_m = query->dims()[1];
@@ -383,16 +389,16 @@ __global__ void FlashAttRange(int start, int step, int size, T* out1, T* out2) {
 
 static void GetFlashAttnDimsString(const std::string& prefix,
                                    const phi::DDim dim_val) {
-  if (VLOG_IS_ON(4)) {
-    std::ostringstream out_string;
-    out_string << "FlashAttn - " << prefix << ".dims() is [ ";
-    for (int i = 0; i < dim_val.size(); ++i) {
-      out_string << dim_val[i] << ", ";
-    }
-    out_string << "]\n";
-    VLOG(4) << out_string.str();
-    std::cout << out_string.str();
+  // if (VLOG_IS_ON(4)) {
+  std::ostringstream out_string;
+  out_string << "FlashAttn - " << prefix << ".dims() is [ ";
+  for (int i = 0; i < dim_val.size(); ++i) {
+    out_string << dim_val[i] << ", ";
   }
+  out_string << "]\n";
+  VLOG(4) << out_string.str();
+  std::cout << out_string.str();
+  // }
 }
 
 #define DBG_WAIT                                        \
@@ -431,34 +437,26 @@ class FMHAGateRef {
       std::cout << "T is float. \n";
     }
 
-    if (config->UseFlashAttn(merge_qkv_, is_bf16)) {
+    if (config->UseFlashAttn(merge_qkv_, config->use_flash_attn && is_bf16)) {
       PADDLE_ENFORCE_NOT_NULL(
           qkv_transpose_out,
           platform::errors::NotFound("The input qkv_transpose_out can not be "
                                      "nullptr when merge_qkv is true."));
+
+      // 1. Dealing with qkv_out for flash_attn.
       phi::DenseTensor* qkv_out = config->GetQKVOut();
       ComputeQKVTransposeForwardForFlashAttn(*qkv_out, qkv_transpose_out);
       config->ClearQKVOut();
 
-      dev_ctx_.Wait();
-      qkv_transpose_out->Resize({3,
-                                 static_cast<int>(config->batch_size),
-                                 static_cast<int>(config->seq_len_m),
-                                 static_cast<int>(config->seq_len_r),
-                                 static_cast<int>(config->num_heads),
-                                 static_cast<int>(config->head_dim)});
-
-      // 1. Dealing with qkv_out for flash_attn.
-      auto& qkv_dims = qkv_transpose_out->dims();
-      auto rank = qkv_dims.size();
-      int64_t q_batch_size = 1;
-      int64_t rest_dim = qkv_dims[rank - 3];
-      for (int i = 1; i < (rank - 3); ++i) {
-        q_batch_size *= qkv_dims[i];
-      }
+      int seq_batch_size = static_cast<int>(config->batch_size) *
+                           static_cast<int>(config->seq_len_m);
       qkv_transpose_out->Resize(
-          {3, q_batch_size * rest_dim, config->num_heads, config->head_dim});
+          {3,
+           seq_batch_size * static_cast<int>(config->seq_len_r),
+           static_cast<int>(config->num_heads),
+           static_cast<int>(config->head_dim)});
       DBG_WAIT;
+
       // q_size == k_size
       int64_t q_size = config->GetQuerySize();
       q_ptr = qkv_transpose_out->data<T>();
@@ -467,9 +465,10 @@ class FMHAGateRef {
 
       // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
       phi::DenseTensor cu_seq_q, cu_seq_k;
-      int64_t end_size = (q_batch_size + 1);
+      int64_t end_size = (seq_batch_size + 1);
       int64_t seq_size = 0;
-      int64_t start = 0, end = end_size, step = rest_dim;
+      int64_t start = 0, end = end_size,
+              step = static_cast<int>(config->seq_len_r);
       phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
       cu_seq_q.Resize({end_size});
       cu_seq_k.Resize({end_size});
@@ -485,41 +484,32 @@ class FMHAGateRef {
 
       // 3. Dealing with mask and bias for flash_attn.
       phi::DenseTensor temp_mask, temp_bias;
-      if (src_mask) {
-        int64_t mask_first_dim = 1;
-        temp_mask.ShareDataWith(*src_mask);
-
-        auto mask_dim = temp_mask.dims();
-        for (int i = 0; i < mask_dim.size() - 3; ++i) {
-          mask_first_dim *= mask_dim[i];
-        }
-        auto mask_dim_rank = mask_dim.size();
-        temp_mask.Resize({mask_first_dim,
-                          mask_dim[mask_dim_rank - 3],
-                          mask_dim[mask_dim_rank - 2],
-                          mask_dim[mask_dim_rank - 1]});
-        GetFlashAttnDimsString("mask_dim", temp_mask.dims());
-      }
-      if (nonbatched_bias) {
-        int64_t bias_first_dim = 1;
-        temp_bias.ShareDataWith(*nonbatched_bias);
-
-        auto bias_dim = nonbatched_bias->dims();
-        for (int i = 0; i < bias_dim.size() - 3; ++i) {
-          bias_first_dim *= bias_dim[i];
+      auto dims_merge_func = [&](const phi::DenseTensor* src_tensor,
+                                 phi::DenseTensor* dst_tensor,
+                                 const std::string& prefix) {
+        if (src_tensor) {
+          int64_t first_dim = 1;
+          dst_tensor->ShareDataWith(*src_tensor);
+          auto dims_ = src_tensor->dims();
+          for (int i = 0; i < dims_.size() - 3; ++i) {
+            first_dim *= dims_[i];
+          }
+          auto dims_rank = dims_.size();
+          dst_tensor->Resize({first_dim,
+                              dims_[dims_rank - 3],
+                              dims_[dims_rank - 2],
+                              dims_[dims_rank - 1]});
+          GetFlashAttnDimsString(prefix, temp_mask.dims());
         }
-        auto bias_dim_rank = temp_bias.dims().size();
-        temp_bias.Resize({bias_first_dim,
-                          bias_dim[bias_dim_rank - 3],
-                          bias_dim[bias_dim_rank - 2],
-                          bias_dim[bias_dim_rank - 1]});
-        GetFlashAttnDimsString("bias_dim", temp_bias.dims());
-      }
+      };
+      auto& qkv_dims = qkv_transpose_out->dims();
+      dims_merge_func(src_mask, &temp_mask, "mask_dim");
+      dims_merge_func(nonbatched_bias, &temp_bias, "bias_dim");
+      GetFlashAttnDimsString("qkv_transpose_out", qkv_dims);
       DBG_WAIT;
-      GetFlashAttnDimsString("qkv_transpose_out", qkv_transpose_out->dims());
-
       // 4. flash_attn parameter setting.
-      int batch_size_ = q_batch_size;
+
+      int batch_size_ = seq_batch_size;
       int total_q_ = qkv_dims[1];    // q.dims()[0]
       int total_k_ = qkv_dims[1];    // q.dims()[0]
       int num_heads_ = qkv_dims[2];  // q.dims()[1]
@@ -527,8 +517,6 @@ class FMHAGateRef {
       int max_seqlen_q_ = batch_size_;
       int max_seqlen_k_ = batch_size_;
       int num_splits = 0;  // 0 for an internal heuristic, which is optimal
-      DBG_WAIT;
-
       VLOG(6) << "[Flash_attn] batch_size : " << batch_size_;
       VLOG(6) << "[Flash_attn] total_q   : " << total_q_;
       VLOG(6) << "[Flash_attn] total_k   : " << total_k_;
@@ -554,6 +542,9 @@ class FMHAGateRef {
 
       GetFlashAttnDimsString("softmax_out", softmax_out->dims());
       GetFlashAttnDimsString("softmax_lse", softmax_lse.dims());
+
+      GetFlashAttnDimsString("cu_seq_q", cu_seq_q.dims());
+      GetFlashAttnDimsString("cu_seq_k", cu_seq_k.dims());
       DBG_WAIT;
 
       // 7. flas_attn part one, get temp worksapce size.
@@ -699,6 +690,27 @@ class FMHAGateRef {
       int64_t gemm_k = config->head_dim;
       // attn = torch.matmul(q, k.transpose(-1, -2))
       T alpha = static_cast<T>(1.0 / sqrt(config->head_dim));
+      // ComputeBatchedGEMM(merge_qkv_ ?
+      //                         phi::slice_ddim(qkv_transpose_out->dims(),
+      //                                         1,
+      //                                         qkv_transpose_out->dims().size()
+      //                                         - 1) : q_transpose_out->dims(),
+      //                    merge_qkv_ ?
+      //                         phi::slice_ddim(qkv_transpose_out->dims(),
+      //                                         1,
+      //                                         qkv_transpose_out->dims().size()
+      //                                         - 1) : k_transpose_out->dims(),
+      //                    q_ptr,
+      //                    k_ptr,
+      //                    qk_out_ptr,
+      //                    false,
+      //                    true,
+      //                    gemm_m,
+      //                    gemm_n,
+      //                    gemm_k,
+      //                    gemm_batch_size,
+      //                    alpha);
+
       ComputeBatchedGEMM(q_ptr,
                          k_ptr,
                          qk_out_ptr,
@@ -728,6 +740,22 @@ class FMHAGateRef {
 
       // o = torch.matmul(attn, v)
       T* softmax_out_ptr = softmax_out->data<T>();
+      // ComputeBatchedGEMM(softmax_out->dims(),
+      //                    merge_qkv_ ?
+      //                         phi::slice_ddim(qkv_transpose_out->dims(),
+      //                                         1,
+      //                                         qkv_transpose_out->dims().size()
+      //                                         - 1) : v_transpose_out->dims(),
+      //                    softmax_out_ptr,
+      //                    v_ptr,
+      //                    qktv_out_ptr,
+      //                    false,
+      //                    false,
+      //                    gemm_m,
+      //                    gemm_n,
+      //                    gemm_k,
+      //                    gemm_batch_size);
+
       ComputeBatchedGEMM(softmax_out_ptr,
                          v_ptr,
                          qktv_out_ptr,
@@ -737,6 +765,7 @@ class FMHAGateRef {
                          gemm_n,
                          gemm_k,
                          gemm_batch_size);
+
       // fmha_out = transpose(qktv_out)
       // o = o.transpose(-2, -3).contiguous()
       ComputeQKTVTransposeForward(*qktv_out, fmha_out);
@@ -1056,6 +1085,50 @@ class FMHAGateRef {
   }
 
  private:
+  void ComputeBatchedGEMM(const phi::DDim& a_dims,
+                          const phi::DDim& b_dims,
+                          const T* a_ptr,
+                          const T* b_ptr,
+                          T* c_ptr,
+                          bool trans_a,
+                          bool trans_b,
+                          int64_t m,
+                          int64_t n,
+                          int64_t k,
+                          int64_t batch_size,
+                          T alpha = static_cast<T>(1.0),
+                          T beta = static_cast<T>(0.0)) {
+    int64_t stride_a = m * k;
+    int64_t stride_b = k * n;
+    int64_t stride_out = m * n;
+
+    phi::funcs::MatmulPlanner matmul_planner(
+        vectorize(a_dims),
+        vectorize(b_dims),
+        trans_a,
+        trans_b,
+        phi::CppTypeToDataType<T>::Type(),
+        phi::funcs::MatmulFusedType::kMatmul);
+
+    using blaslt = phi::funcs::MatmulWithCublasLt<T>;
+    blaslt::RunWithBatch(dev_ctx_,
+                         a_ptr,
+                         b_ptr,
+                         c_ptr,
+                         m,
+                         n,
+                         k,
+                         trans_a,
+                         trans_b,
+                         batch_size,
+                         stride_a,
+                         stride_b,
+                         stride_out,
+                         &matmul_planner,
+                         alpha,
+                         beta);
+  }
+
   void ComputeBatchedGEMM(const T* a_ptr,
                           const T* b_ptr,
                           T* c_ptr,
@@ -1067,11 +1140,11 @@ class FMHAGateRef {
                           int64_t batch_size,
                           T alpha = static_cast<T>(1.0),
                           T beta = static_cast<T>(0.0)) {
-    CBLAS_TRANSPOSE cblas_trans_a = trans_a ? CblasTrans : CblasNoTrans;
-    CBLAS_TRANSPOSE cblas_trans_b = trans_b ? CblasTrans : CblasNoTrans;
     int64_t stride_a = m * k;
     int64_t stride_b = k * n;
 
+    CBLAS_TRANSPOSE cblas_trans_a = trans_a ? CblasTrans : CblasNoTrans;
+    CBLAS_TRANSPOSE cblas_trans_b = trans_b ? CblasTrans : CblasNoTrans;
     auto blas = phi::funcs::GetBlas<phi::GPUContext, T>(dev_ctx_);
     blas.BatchedGEMM(cblas_trans_a,
                      cblas_trans_b,
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cc b/paddle/fluid/operators/fused/fused_gate_attention_op.cc
index c91bca47cf42f..e2d8926f53516 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cc
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cc
@@ -162,6 +162,9 @@ class FusedGateAttentionOpMaker : public framework::OpProtoAndCheckerMaker {
         .AsIntermediate()
         .AsDispensable();
     AddOutput("Out", "Result after attention.");
+    AddOutput("SoftmaxLse", "Result of the gating module.")
+        .AsIntermediate()
+        .AsDispensable();
     AddAttr<bool>("has_gating",
                   "if true, the attention op uses gate architecure, "
                   "[default true].")
@@ -170,6 +173,11 @@ class FusedGateAttentionOpMaker : public framework::OpProtoAndCheckerMaker {
                   "if true, calculation with merged qkv, "
                   "[default true].")
         .SetDefault(true);
+    AddAttr<bool>(
+        "use_flash_attn",
+        "if true, the attention op will be computed in flash_attn branch, "
+        "[default false].")
+        .SetDefault(false);
     AddComment(R"DOC(
   Add fused attention op whose logic is as follows:
   {
@@ -276,6 +284,7 @@ class FusedGateAttentionGradOpMaker : public framework::SingleGradOpMaker<T> {
 
     op->SetAttrMap(this->Attrs());
     bool merge_qkv = PADDLE_GET_CONST(bool, op->GetAttr("merge_qkv"));
+
     if (merge_qkv) {
       op->SetInput("QKVWeight", this->Input("QKVWeight"));
       op->SetOutput(framework::GradVarName("QKVWeight"),
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cu b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
index ca7b70b220f38..5927f03eca09d 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cu
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
@@ -378,6 +378,7 @@ class FusedGateAttentionOpKernel : public framework::OpKernel<T> {
 
     const bool merge_qkv = ctx.Attr<bool>("merge_qkv");
     const bool has_gating = ctx.Attr<bool>("has_gating");
+    const bool use_flash_attn = ctx.Attr<bool>("use_flash_attn");
 
     bool use_fused_matmul_bias = true;
     auto &dev_ctx = ctx.template device_context<phi::GPUContext>();
@@ -389,8 +390,14 @@ class FusedGateAttentionOpKernel : public framework::OpKernel<T> {
     AllocWithDebugInfo<T>(dev_ctx, "out", out);
 
     // When seq_len_r = m_size, q_dim = kv_dim, QKV matmul can be merged.
-    GateAttentionConfig<T> config(
-        dev_ctx, query, key, query_weight, qkv_weight, merge_qkv, has_gating);
+    GateAttentionConfig<T> config(dev_ctx,
+                                  query,
+                                  key,
+                                  query_weight,
+                                  qkv_weight,
+                                  merge_qkv,
+                                  has_gating,
+                                  use_flash_attn);
 
     if (merge_qkv) {
       PADDLE_ENFORCE_EQ(
diff --git a/paddle/phi/backends/dynload/flashattn.h b/paddle/phi/backends/dynload/flashattn.h
index 1d235fcdecfc5..36ec693218f3c 100644
--- a/paddle/phi/backends/dynload/flashattn.h
+++ b/paddle/phi/backends/dynload/flashattn.h
@@ -43,11 +43,11 @@ extern void* flashattn_dso_handle;
 #define DECLARE_DYNAMIC_LOAD_FLASHATTN_WRAP(__name) \
   DYNAMIC_LOAD_FLASHATTN_WRAP(__name)
 
-#define FLASHATTN_ROUTINE_EACH(__macro) \
-  __macro(flash_attn_fwd);              \
-  __macro(flash_attn_bwd);              \
-  __macro(flash_attn_error);
+#define FLASHATTN_ROUTINE_EACH(__macro)       \
+  __macro(flash_attn_fwd);                    \
+  __macro(flash_attn_bwd);                    \
   __macro(flash_attn_fwd_with_bias_and_mask); \
+  __macro(flash_attn_error);
 
 FLASHATTN_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_FLASHATTN_WRAP);
 
diff --git a/paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h b/paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h
index ab3b5af1d54ac..79a653fb48772 100644
--- a/paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h
+++ b/paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h
@@ -328,10 +328,20 @@ struct MatmulWithCublasLt {
                   const int K,
                   const bool trans_x,
                   const bool trans_y,
-                  phi::funcs::MatmulPlanner* planner = nullptr) {
+                  phi::funcs::MatmulPlanner* planner = nullptr,
+                  T alpha = static_cast<T>(1),
+                  T beta = static_cast<T>(0)) {
     auto setter = DescriptorSetter<T>(planner, M, N, K, trans_x, trans_y);
-    RunImpl(
-        ctx, &setter.desc, setter.sub_key, x_data, y_data, out_data, planner);
+
+    RunImpl(ctx,
+            &setter.desc,
+            setter.sub_key,
+            x_data,
+            y_data,
+            out_data,
+            planner,
+            static_cast<MT>(alpha),
+            static_cast<MT>(beta));
   }
 
   static void RunWithBatch(const phi::GPUContext& ctx,
@@ -347,7 +357,9 @@ struct MatmulWithCublasLt {
                            int64_t stride_x,
                            int64_t stride_y,
                            int64_t stride_out,
-                           phi::funcs::MatmulPlanner* planner = nullptr) {
+                           phi::funcs::MatmulPlanner* planner = nullptr,
+                           T alpha = static_cast<T>(1),
+                           T beta = static_cast<T>(0)) {
     auto setter = DescriptorSetter<T>(planner,
                                       M,
                                       N,
@@ -358,8 +370,15 @@ struct MatmulWithCublasLt {
                                       stride_x,
                                       stride_y,
                                       stride_out);
-    RunImpl(
-        ctx, &setter.desc, setter.sub_key, x_data, y_data, out_data, planner);
+    RunImpl(ctx,
+            &setter.desc,
+            setter.sub_key,
+            x_data,
+            y_data,
+            out_data,
+            planner,
+            static_cast<MT>(alpha),
+            static_cast<MT>(beta));
   }
 
   static void RunWithBatch(const phi::GPUContext& ctx,
@@ -372,7 +391,9 @@ struct MatmulWithCublasLt {
                            bool trans_x,
                            bool trans_y,
                            int batch_size,
-                           phi::funcs::MatmulPlanner* planner = nullptr) {
+                           phi::funcs::MatmulPlanner* planner = nullptr,
+                           T alpha = static_cast<T>(1),
+                           T beta = static_cast<T>(0)) {
     for (int i = 0; i < batch_size; ++i) {
       Run(ctx,
           x_data[i],
@@ -383,7 +404,9 @@ struct MatmulWithCublasLt {
           K,
           trans_x,
           trans_y,
-          planner);
+          planner,
+          alpha,
+          beta);
     }
   }
 
@@ -402,10 +425,9 @@ struct MatmulWithCublasLt {
                       const T* x_ptr,
                       const T* y_ptr,
                       T* out_ptr,
-                      phi::funcs::MatmulPlanner* planner) {
-    MT alpha = static_cast<MT>(1);
-    MT beta = static_cast<MT>(0);
-
+                      phi::funcs::MatmulPlanner* planner,
+                      MT alpha,
+                      MT beta) {
     cublasLtHandle_t cublaslt_handle = ctx.cublaslt_handle();
     size_t workspace_size = static_cast<size_t>(4) * 1024 * 1024;
     phi::Allocator::AllocationPtr workspace = GetWorkspace(ctx, workspace_size);

From 5bee3a69932444ea079e5ca57c2a5564156e1bd3 Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Wed, 19 Apr 2023 20:23:06 +0800
Subject: [PATCH 005/405] :qix pointer bugs for my yesterday errors

---
 cmake/external/flashattn.cmake                |  6 +-
 .../operators/fused/fused_gate_attention.h    | 59 +++++++++++++++----
 .../fused/fused_gate_attention_op.cu          | 11 +++-
 3 files changed, 60 insertions(+), 16 deletions(-)

diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index a37559c483421..8846fab1d9c76 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -20,10 +20,10 @@ set(FLASHATTN_PREFIX_DIR ${THIRD_PARTY_PATH}/flashattn)
 set(FLASHATTN_SOURCE_SUBDIR csrc/flash_attn)
 set(FLASHATTN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flashattn)
 set(FLASHATTN_REPOSITORY ${GIT_URL}/JamesLim-sy/flash-attention.git)
-set(FLASHATTN_TAG c7442255b553e8f8ddee21b48a4bca992678cb89)
+set(FLASHATTN_TAG f009e2b67a98e279cf9d5b446cd21f3260d9cb0f)
 
-# set(FLASHATTN_REPOSITORY ${GIT_URL}/PaddlePaddle/flash-attention.git)
-# set(FLASHATTN_TAG f0edf243a813a65d05c75fcb331b2a95faf96bbc)
+#set(FLASHATTN_REPOSITORY ${GIT_URL}/PaddlePaddle/flash-attention.git)
+#set(FLASHATTN_TAG f0edf243a813a65d05c75fcb331b2a95faf96bbc)
 
 set(FLASHATTN_INCLUDE_DIR
     "${FLASHATTN_INSTALL_DIR}/include"
diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 58ab2d9e378a2..d4db86ec67196 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -15,10 +15,8 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/phi/kernels/arange_kernel.h"
-#include "paddle/phi/kernels/empty_kernel.h"
 #include "paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h"
 #include "paddle/phi/kernels/funcs/broadcast_function.h"
-#include "paddle/phi/kernels/funcs/elementwise_base.h"
 #include "paddle/phi/kernels/funcs/elementwise_functor.h"
 #include "paddle/phi/kernels/funcs/range_function.h"
 #include "paddle/phi/kernels/funcs/reduce_function.h"
@@ -287,14 +285,16 @@ struct GateAttentionGradConfig : public GateAttentionConfig<T> {
                           const phi::DenseTensor* query_weight,
                           const phi::DenseTensor* qkv_weight,
                           bool merge_qkv,
-                          bool has_gating)
+                          bool has_gating,
+                          bool use_flash_attn)
       : GateAttentionConfig<T>(dev_ctx,
                                query,
                                key,
                                query_weight,
                                qkv_weight,
                                merge_qkv,
-                               has_gating) {}
+                               has_gating,
+                               use_flash_attn) {}
 
   phi::DenseTensor* GetQKVOutGrad() {
     if (!qkv_out_grad.IsInitialized()) {
@@ -401,6 +401,16 @@ static void GetFlashAttnDimsString(const std::string& prefix,
   // }
 }
 
+#define DBGPTR(ptr, prefix)                                              \
+  do {                                                                   \
+    std::ostringstream out_string;                                       \
+    void* data = static_cast<void*>(ptr);                                \
+    out_string << "[" << __func__ << ", " << __LINE__ << "]: " << prefix \
+               << "`s addr is ";                                         \
+    out_string << ptr << std::endl;                                      \
+    std::cout << out_string.str();                                       \
+  } while (0);
+
 #define DBG_WAIT                                        \
   do {                                                  \
     printf("[%s, %d] Run here.\n", __func__, __LINE__); \
@@ -472,8 +482,8 @@ class FMHAGateRef {
       phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
       cu_seq_q.Resize({end_size});
       cu_seq_k.Resize({end_size});
-      AllocWithDebugInfo<int32_t>(dev_ctx_, "flash_attn: cu_seq_q", &cu_seq_q);
-      AllocWithDebugInfo<int32_t>(dev_ctx_, "flash_attn: cu_seq_k", &cu_seq_k);
+      AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_q", &cu_seq_q);
+      AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_k", &cu_seq_k);
       int64_t block = std::min(seq_size, static_cast<int64_t>(256));
       int64_t grid = (seq_size + block - 1) / block;
       FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
@@ -538,11 +548,10 @@ class FMHAGateRef {
       uint64_t inc = batch_size_ * num_heads_ * 32;
       auto seed_offset_pair = gen->IncrementOffset(inc);
       uint64_t seed = seed_offset_pair.first;
-      uint64_t offset = seed_offset_pair.second;
+      uint64_t offset = 1908576;  // seed_offset_pair.second;
 
       GetFlashAttnDimsString("softmax_out", softmax_out->dims());
       GetFlashAttnDimsString("softmax_lse", softmax_lse.dims());
-
       GetFlashAttnDimsString("cu_seq_q", cu_seq_q.dims());
       GetFlashAttnDimsString("cu_seq_k", cu_seq_k.dims());
       DBG_WAIT;
@@ -587,18 +596,23 @@ class FMHAGateRef {
         PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
       }
       DBG_WAIT;
+
       phi::DenseTensor workspace;
+      printf("workspace_size = %d\n", workspace_size);
       if (workspace_size > 0) {
-        workspace = phi::Empty<float>(
+        workspace = phi::Empty<float, phi::GPUContext>(
             dev_ctx_, {int64_t(workspace_size / sizeof(float))});
+        DBGPTR(workspace.data(), "workspace");
       }
       DBG_WAIT;
+
       // 8. flas_attn part two, run impl.
       succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
           static_cast<const void*>(q_ptr),
           static_cast<const void*>(k_ptr),
           static_cast<const void*>(v_ptr),
-          static_cast<void*>(fmha_out),  // for calculation workspace size
+          static_cast<void*>(
+              fmha_out->data()),  // for calculation workspace size
           static_cast<const void*>(cu_seq_q.data<int32_t>()),
           static_cast<const void*>(cu_seq_k.data<int32_t>()),
           total_q_,
@@ -616,7 +630,7 @@ class FMHAGateRef {
           num_splits,
           softmax_lse.data(),
           softmax_out->data(),
-          workspace_size > 0 ? static_cast<void*>(workspace.data()) : nullptr,
+          (workspace_size > 0) ? static_cast<void*>(workspace.data()) : nullptr,
           &workspace_size,
           stream,
           seed,
@@ -798,6 +812,29 @@ class FMHAGateRef {
     phi::DenseTensor k_transpose_out_grad;
     phi::DenseTensor v_transpose_out_grad;
     phi::DenseTensor qkv_transpose_out_grad;
+
+    bool is_bf16 =
+        qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
+
+    // if (config->UseFlashAttn(merge_qkv_, config->use_flash_attn && is_bf16))
+    // {
+    //   PADDLE_ENFORCE_NOT_NULL(
+    //       qkv_transpose_out,
+    //       platform::errors::NotFound("The input qkv_transpose_out can not be
+    //       "
+    //                                  "nullptr when merge_qkv is true."));
+    //   int64_t q_size = config->GetQuerySize();
+    //   q_ptr = qkv_transpose_out->data<T>();
+    //   k_ptr = q_ptr + q_size;
+    //   v_ptr = k_ptr + q_size;
+
+    //   qkv_transpose_out_grad.Resize(config->qkv_transpose_out_dims);
+    //   AllocWithDebugInfo<T>(
+    //       dev_ctx_, "qkv_transpose_out_grad", &qkv_transpose_out_grad);
+
+    // } else {
+
+    // }
     if (merge_qkv_) {
       PADDLE_ENFORCE_NOT_NULL(
           qkv_transpose_out,
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cu b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
index 5927f03eca09d..584cb48c1ebe4 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cu
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
@@ -484,13 +484,20 @@ class FusedGateAttentionGradKernel : public framework::OpKernel<T> {
 
     bool has_gating = ctx.Attr<bool>("has_gating");
     bool merge_qkv = ctx.Attr<bool>("merge_qkv");
+    const bool use_flash_attn = ctx.Attr<bool>("use_flash_attn");
 
     bool use_fused_matmul_bias = true;
     auto &dev_ctx = ctx.template device_context<phi::GPUContext>();
     AllocWithDebugInfo<T>(dev_ctx, "query_grad", query_grad);
 
-    GateAttentionGradConfig<T> config(
-        dev_ctx, query, key, query_weight, qkv_weight, merge_qkv, has_gating);
+    GateAttentionGradConfig<T> config(dev_ctx,
+                                      query,
+                                      key,
+                                      query_weight,
+                                      qkv_weight,
+                                      merge_qkv,
+                                      has_gating,
+                                      use_flash_attn);
 
     phi::DenseTensor fmha_out_grad;
     fmha_out_grad.Resize(config.gate_out_dims);

From afce9d6bf6a6b202845af3a63a181a3716e7f9ff Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Sun, 23 Apr 2023 17:20:25 +0800
Subject: [PATCH 006/405] add for backward

---
 cmake/external/flashattn.cmake                |   2 +-
 .../operators/fused/fused_gate_attention.h    | 474 +++++++++++++-----
 .../fused/fused_gate_attention_op.cc          |   7 +
 .../fused/fused_gate_attention_op.cu          |  53 +-
 paddle/phi/backends/dynload/flashattn.h       |   1 +
 5 files changed, 400 insertions(+), 137 deletions(-)

diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index 8846fab1d9c76..2cf1ddafaafdc 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -20,7 +20,7 @@ set(FLASHATTN_PREFIX_DIR ${THIRD_PARTY_PATH}/flashattn)
 set(FLASHATTN_SOURCE_SUBDIR csrc/flash_attn)
 set(FLASHATTN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flashattn)
 set(FLASHATTN_REPOSITORY ${GIT_URL}/JamesLim-sy/flash-attention.git)
-set(FLASHATTN_TAG f009e2b67a98e279cf9d5b446cd21f3260d9cb0f)
+set(FLASHATTN_TAG edeaea701e0c0e712f1a43a8970b5d59f5256e3b)
 
 #set(FLASHATTN_REPOSITORY ${GIT_URL}/PaddlePaddle/flash-attention.git)
 #set(FLASHATTN_TAG f0edf243a813a65d05c75fcb331b2a95faf96bbc)
diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index d4db86ec67196..e43a9b77c848c 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -15,7 +15,8 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/phi/kernels/arange_kernel.h"
-#include "paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h"
+// #include "paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h"
+#include "paddle/phi/backends/dynload/flashattn.h"
 #include "paddle/phi/kernels/funcs/broadcast_function.h"
 #include "paddle/phi/kernels/funcs/elementwise_functor.h"
 #include "paddle/phi/kernels/funcs/range_function.h"
@@ -24,7 +25,6 @@ limitations under the License. */
 #include "paddle/phi/kernels/gpudnn/softmax_gpudnn.h"
 
 #ifdef PADDLE_WITH_FLASHATTN
-#include "paddle/phi/backends/dynload/flashattn.h"
 #include "paddle/phi/kernels/flash_attn_kernel.h"
 #endif
 
@@ -158,8 +158,8 @@ struct GateAttentionConfig {
     gate_out_dims = {batch_size, seq_len_m, seq_len_r, num_heads, head_dim};
   }
 
-  bool UseFlashAttn(const bool merge_qkv, const bool is_amp) {
-    if (!is_amp) {
+  bool UseFlashAttn(const bool merge_qkv, const bool run_flash_attn) {
+    if (!run_flash_attn) {
       return false;
     }
 
@@ -527,13 +527,13 @@ class FMHAGateRef {
       int max_seqlen_q_ = batch_size_;
       int max_seqlen_k_ = batch_size_;
       int num_splits = 0;  // 0 for an internal heuristic, which is optimal
-      VLOG(6) << "[Flash_attn] batch_size : " << batch_size_;
-      VLOG(6) << "[Flash_attn] total_q   : " << total_q_;
-      VLOG(6) << "[Flash_attn] total_k   : " << total_k_;
-      VLOG(6) << "[Flash_attn] num_heads : " << num_heads_;
-      VLOG(6) << "[Flash_attn] head_size : " << head_size_;
-      VLOG(6) << "[Flash_attn] max_seqlen_q : " << max_seqlen_q_;
-      VLOG(6) << "[Flash_attn] max_seqlen_k : " << max_seqlen_k_;
+      VLOG(6) << "[Flash_attn Fwd] batch_size : " << batch_size_;
+      VLOG(6) << "[Flash_attn Fwd] total_q   : " << total_q_;
+      VLOG(6) << "[Flash_attn Fwd] total_k   : " << total_k_;
+      VLOG(6) << "[Flash_attn Fwd] num_heads : " << num_heads_;
+      VLOG(6) << "[Flash_attn Fwd] head_size : " << head_size_;
+      VLOG(6) << "[Flash_attn Fwd] max_seqlen_q : " << max_seqlen_q_;
+      VLOG(6) << "[Flash_attn Fwd] max_seqlen_k : " << max_seqlen_k_;
 
       // 5. construct softmax_lse
       phi::DenseTensor softmax_lse;
@@ -606,6 +606,21 @@ class FMHAGateRef {
       }
       DBG_WAIT;
 
+#define DBG_INIT(prefix, x)                                       \
+  do {                                                            \
+    printf("[%s, %d] ", __func__, __LINE__);                      \
+    if (x->initialized()) {                                       \
+      std::cout << prefix << " is initialized." << std::endl;     \
+    } else {                                                      \
+      std::cout << prefix << " is not initialized." << std::endl; \
+    }                                                             \
+  } while (0);
+      DBG_INIT("qkv_transpose_out", qkv_transpose_out);
+      DBG_INIT("softmax_out", softmax_out);
+      DBG_INIT("src_mask", src_mask);
+      DBG_INIT("fmha_out", fmha_out);
+      DBG_INIT("gate_out", gate_out);
+
       // 8. flas_attn part two, run impl.
       succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
           static_cast<const void*>(q_ptr),
@@ -799,7 +814,11 @@ class FMHAGateRef {
                        const phi::DenseTensor* fmha_out_grad,
                        phi::DenseTensor* src_mask_grad,
                        phi::DenseTensor* nonbatched_bias_grad,
-                       GateAttentionGradConfig<T>* config) {
+                       GateAttentionGradConfig<T>* config,
+                       const phi::DenseTensor* fmha_out = nullptr,
+                       const phi::DenseTensor* softmax_lse = nullptr,
+                       const phi::DenseTensor* nonbatched_bias = nullptr,
+                       const phi::DenseTensor* src_mask = = nullptr) {
     const T* q_ptr = nullptr;
     const T* k_ptr = nullptr;
     const T* v_ptr = nullptr;
@@ -816,31 +835,19 @@ class FMHAGateRef {
     bool is_bf16 =
         qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
 
-    // if (config->UseFlashAttn(merge_qkv_, config->use_flash_attn && is_bf16))
-    // {
-    //   PADDLE_ENFORCE_NOT_NULL(
-    //       qkv_transpose_out,
-    //       platform::errors::NotFound("The input qkv_transpose_out can not be
-    //       "
-    //                                  "nullptr when merge_qkv is true."));
-    //   int64_t q_size = config->GetQuerySize();
-    //   q_ptr = qkv_transpose_out->data<T>();
-    //   k_ptr = q_ptr + q_size;
-    //   v_ptr = k_ptr + q_size;
-
-    //   qkv_transpose_out_grad.Resize(config->qkv_transpose_out_dims);
-    //   AllocWithDebugInfo<T>(
-    //       dev_ctx_, "qkv_transpose_out_grad", &qkv_transpose_out_grad);
-
-    // } else {
-
-    // }
-    if (merge_qkv_) {
+    if (std::is_same<T, phi::dtype::float16>::value) {
+      std::cout << "[Grad]: T is phi::dtype::float16. \n";
+    } else if (std::is_same<T, phi::dtype::bfloat16>::value) {
+      std::cout << "[Grad]: T is phi::dtype::bfloat16. \n";
+    } else if (std::is_same<T, float>::value) {
+      std::cout << "[Grad]: T is float. \n";
+    }
+
+    if (config->UseFlashAttn(merge_qkv_, config->use_flash_attn && is_bf16)) {
       PADDLE_ENFORCE_NOT_NULL(
           qkv_transpose_out,
-          platform::errors::NotFound("The input qkv_transpose_out can not be "
+          platform::errors::NotFound("The input qkv_transpose_out can not be"
                                      "nullptr when merge_qkv is true."));
-
       int64_t q_size = config->GetQuerySize();
       q_ptr = qkv_transpose_out->data<T>();
       k_ptr = q_ptr + q_size;
@@ -850,27 +857,95 @@ class FMHAGateRef {
       AllocWithDebugInfo<T>(
           dev_ctx_, "qkv_transpose_out_grad", &qkv_transpose_out_grad);
 
-      q_grad_ptr = qkv_transpose_out_grad.data<T>();
-      k_grad_ptr = q_grad_ptr + q_size;
-      v_grad_ptr = k_grad_ptr + q_size;
-    } else {
-      PADDLE_ENFORCE_NOT_NULL(
-          q_transpose_out,
-          platform::errors::NotFound("The input q_transpose_out can not be "
-                                     "nullptr when merge_qkv is false."));
-      PADDLE_ENFORCE_NOT_NULL(
-          k_transpose_out,
-          platform::errors::NotFound("The input k_transpose_out can not be "
-                                     "nullptr when merge_qkv is false."));
-      PADDLE_ENFORCE_NOT_NULL(
-          v_transpose_out,
-          platform::errors::NotFound("The input v_transpose_out can not be "
-                                     "nullptr when merge_qkv is false."));
+      int seq_batch_size = static_cast<int>(config->batch_size) *
+                           static_cast<int>(config->seq_len_m);
+      qkv_transpose_out->Resize(
+          {3,
+           seq_batch_size * static_cast<int>(config->seq_len_r),
+           static_cast<int>(config->num_heads),
+           static_cast<int>(config->head_dim)});
+      DBG_WAIT;
+
+      // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
+      phi::DenseTensor cu_seq_q, cu_seq_k;
+      int64_t start = 0;
+      int64_t step = static_cast<int>(config->seq_len_r);
+      int64_t end_size = (seq_batch_size + 1);
+      int64_t end = end_size, int64_t seq_size = 0;
+      phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
+      cu_seq_q.Resize({end_size});
+      cu_seq_k.Resize({end_size});
+      AllocWithDebugInfo<int32_t>(dev_ctx_, "Grad: cu_seq_q", &cu_seq_q);
+      AllocWithDebugInfo<int32_t>(dev_ctx_, "Grad: cu_seq_k", &cu_seq_k);
+      int64_t block = std::min(seq_size, static_cast<int64_t>(256));
+      int64_t grid = (seq_size + block - 1) / block;
+      FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
+          start, step, end, cu_seq_q.data<int32_t>(), cu_seq_k.data<int32_t>());
+      VLOG(4) << "[Flash_attn] cu_seq_len : start = " << start
+              << ", step = " << step << ", end = " << end;
+      DBG_WAIT;
+
+      // 3. Dealing with mask and bias for flash_attn.
+      phi::DenseTensor temp_mask, temp_bias;
+      auto dims_merge_func = [&](const phi::DenseTensor* src_tensor,
+                                 phi::DenseTensor* dst_tensor,
+                                 const std::string& prefix) {
+        if (src_tensor) {
+          int64_t first_dim = 1;
+          dst_tensor->ShareDataWith(*src_tensor);
+          auto dims_ = src_tensor->dims();
+          for (int i = 0; i < dims_.size() - 3; ++i) {
+            first_dim *= dims_[i];
+          }
+          auto dims_rank = dims_.size();
+          dst_tensor->Resize({first_dim,
+                              dims_[dims_rank - 3],
+                              dims_[dims_rank - 2],
+                              dims_[dims_rank - 1]});
+          GetFlashAttnDimsString(prefix, temp_mask.dims());
+        }
+      };
+      dims_merge_func(src_mask, &temp_mask, "[Grad] mask_dim");
+      dims_merge_func(nonbatched_bias, &temp_bias, "[Grad] bias_dim");
+
+      auto& qkv_dims = qkv_transpose_out->dims();
+      int batch_size_ = seq_batch_size;
+      int total_q_ = qkv_dims[1];    // q.dims()[0]
+      int total_k_ = qkv_dims[1];    // q.dims()[0]
+      int num_heads_ = qkv_dims[2];  // q.dims()[1]
+      int head_size_ = qkv_dims[3];  // q.dims()[2]
+      int max_seqlen_q_ = batch_size_;
+      int max_seqlen_k_ = batch_size_;
+      int num_splits = 0;
+      VLOG(6) << "[Flash_attn Grad] batch_size : " << batch_size_;
+      VLOG(6) << "[Flash_attn Grad] total_q   : " << total_q_;
+      VLOG(6) << "[Flash_attn Grad] total_k   : " << total_k_;
+      VLOG(6) << "[Flash_attn Grad] num_heads : " << num_heads_;
+      VLOG(6) << "[Flash_attn Grad] head_size : " << head_size_;
+      VLOG(6) << "[Flash_attn Grad] max_seqlen_q : " << max_seqlen_q_;
+      VLOG(6) << "[Flash_attn Grad] max_seqlen_k : " << max_seqlen_k_;
+
+      // 5. construct softmax_lse
+      int last_q_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
+      softmax_lse->Resize({batch_size_, num_heads_, last_q_dim});
+      AllocWithDebugInfo<float>(
+          dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
+      DBG_WAIT;
+
+      phi::DenseTensor softmax_d = phi::Empty<float, phi::GPUContext>(
+          dev_ctx_, {batch_size_, num_heads_, last_q_dim});
+      DBG_WAIT;
+
+      phi::DenseTensor bias_d;
+      if (nonbatched_bias) {
+        bias_d = phi::Empty<T, phi::GPUContext>(
+            dev_ctx_, {batch_size_, num_heads_, max_seqlen_q_, max_seqlen_k_});
+      }
+      DBG_WAIT;
 
       q_ptr = q_transpose_out->data<T>();
       k_ptr = k_transpose_out->data<T>();
       v_ptr = v_transpose_out->data<T>();
-
       q_transpose_out_grad.Resize(config->q_transpose_out_dims);
       k_transpose_out_grad.Resize(config->kv_transpose_out_dims);
       v_transpose_out_grad.Resize(config->kv_transpose_out_dims);
@@ -881,99 +956,246 @@ class FMHAGateRef {
                                      k_transpose_out_grad.numel() * sizeof(T));
       v_grad_ptr = dev_ctx_.Alloc<T>(&v_transpose_out_grad,
                                      v_transpose_out_grad.numel() * sizeof(T));
-    }
 
-    phi::DenseTensor softmax_out_grad;
-    softmax_out_grad.Resize(config->softmax_out_dims);
-    AllocWithDebugInfo<T>(dev_ctx_, "softmax_out_grad", &softmax_out_grad);
-
-    int64_t gemm_batch_size =
-        config->batch_size * config->seq_len_m * config->num_heads;
-    {
-      // Forward: fmha_out = transpose(qktv_out)
-      phi::DenseTensor qktv_out_grad;
-      qktv_out_grad.Resize(config->qktv_out_dims);
-      AllocWithDebugInfo<T>(dev_ctx_, "qktv_out_grad", &qktv_out_grad);
-      ComputeQKTVTransposeBackward(*fmha_out_grad, &qktv_out_grad);
-
-      // Forward: qktv_out = BatchedGEMM(softmax_out, V)
-      // Backward:
-      //  V_grad = BatchedGEMM(softmax_out^T, qktv_out_grad) (dy = x^T * dout)
+      // 7. flas_attn part one, get temp worksapce size.
+      uint64_t workspace_size;
+      float p_dropout = 0.f;
+      float softmax_scale = static_cast<float>(1);
+      cudaStream_t stream = dev_ctx_.stream();
+      int num_splits = 0;  // 0 for an internal heuristic, which is optimal
+      succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
+          static_cast<const void*>(q_ptr),
+          static_cast<const void*>(k_ptr),
+          static_cast<const void*>(v_ptr),
+          q_grad_ptr,
+          k_grad_ptr,
+          v_grad_ptr,
+          static_cast<void*>(
+              fmha_out->data()),  // total_q x num_heads x head_size, total_k :=
+                                  // \sum_{i=0}^{b} s_i
+          static_cast<void*>(
+              fmha_out_grad->data()),  // total_q x num_heads, x head_size
+          static_cast<const void*>(cu_seq_q.data<int32_t>()),
+          static_cast<const void*>(cu_seq_k.data<int32_t>()),
+          total_q_,
+          total_k_,
+          batch_size_,
+          num_heads_,
+          head_size_,
+          max_seqlen_q_,
+          max_seqlen_k_,
+          p_dropout,
+          softmax_scale,
+          /*zero_tensors=*/false,
+          /*is_causal=*/false,
+          is_bf16,
+          num_splits,
+          softmax_lse.data(),
+          softmax_d.data(),
+          nullptr,
+          bias_d.data(),
+          &workspace_size,
+          stream,
+          seed,
+          offset,
+          src_mask ? temp_mask.data() : nullptr,
+          nonbatched_bias ? temp_bias.data() : nullptr,
+          temp_mask.dims().Get(),
+          temp_bias.dims().Get());
+      if (!succ) {
+        PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
+      }
+      DBG_WAIT;
+
+      phi::DenseTensor workspace;
+      printf("workspace_size = %d\n", workspace_size);
+      if (workspace_size > 0) {
+        workspace = phi::Empty<float, phi::GPUContext>(
+            dev_ctx_, {int64_t(workspace_size / sizeof(float))});
+        DBGPTR(workspace.data(), "workspace");
+      }
+      DBG_WAIT;
+
+      succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
+          static_cast<const void*>(q_ptr),
+          static_cast<const void*>(k_ptr),
+          static_cast<const void*>(v_ptr),
+          q_grad_ptr,
+          k_grad_ptr,
+          v_grad_ptr,
+          static_cast<void*>(
+              fmha_out->data()),  // total_q x num_heads x head_size, total_k :=
+                                  // \sum_{i=0}^{b} s_i
+          static_cast<void*>(
+              fmha_out_grad->data()),  // total_q x num_heads, x head_size
+          static_cast<const void*>(cu_seq_q.data<int32_t>()),
+          static_cast<const void*>(cu_seq_k.data<int32_t>()),
+          total_q_,
+          total_k_,
+          batch_size_,
+          num_heads_,
+          head_size_,
+          max_seqlen_q_,
+          max_seqlen_k_,
+          p_dropout,
+          softmax_scale,
+          /*zero_tensors=*/false,
+          /*is_causal=*/false,
+          is_bf16,
+          num_splits,
+          softmax_lse.data(),
+          softmax_d.data(),
+          workspace.data(),
+          bias_d.data(),
+          &workspace_size,
+          stream,
+          seed,
+          offset,
+          src_mask ? temp_mask.data() : nullptr,
+          nonbatched_bias ? temp_bias.data() : nullptr,
+          temp_mask.dims().Get(),
+          temp_bias.dims().Get());
+    } else {
+      if (merge_qkv_) {
+        PADDLE_ENFORCE_NOT_NULL(
+            qkv_transpose_out,
+            platform::errors::NotFound("The input qkv_transpose_out can not be "
+                                       "nullptr when merge_qkv is true."));
+
+        int64_t q_size = config->GetQuerySize();
+        q_ptr = qkv_transpose_out->data<T>();
+        k_ptr = q_ptr + q_size;
+        v_ptr = k_ptr + q_size;
+
+        qkv_transpose_out_grad.Resize(config->qkv_transpose_out_dims);
+        AllocWithDebugInfo<T>(
+            dev_ctx_, "qkv_transpose_out_grad", &qkv_transpose_out_grad);
+
+        q_grad_ptr = qkv_transpose_out_grad.data<T>();
+        k_grad_ptr = q_grad_ptr + q_size;
+        v_grad_ptr = k_grad_ptr + q_size;
+      } else {
+        PADDLE_ENFORCE_NOT_NULL(
+            q_transpose_out,
+            platform::errors::NotFound("The input q_transpose_out can not be "
+                                       "nullptr when merge_qkv is false."));
+        PADDLE_ENFORCE_NOT_NULL(
+            k_transpose_out,
+            platform::errors::NotFound("The input k_transpose_out can not be "
+                                       "nullptr when merge_qkv is false."));
+        PADDLE_ENFORCE_NOT_NULL(
+            v_transpose_out,
+            platform::errors::NotFound("The input v_transpose_out can not be "
+                                       "nullptr when merge_qkv is false."));
+
+        q_ptr = q_transpose_out->data<T>();
+        k_ptr = k_transpose_out->data<T>();
+        v_ptr = v_transpose_out->data<T>();
+
+        q_transpose_out_grad.Resize(config->q_transpose_out_dims);
+        k_transpose_out_grad.Resize(config->kv_transpose_out_dims);
+        v_transpose_out_grad.Resize(config->kv_transpose_out_dims);
+
+        q_grad_ptr = dev_ctx_.Alloc<T>(
+            &q_transpose_out_grad, q_transpose_out_grad.numel() * sizeof(T));
+        k_grad_ptr = dev_ctx_.Alloc<T>(
+            &k_transpose_out_grad, k_transpose_out_grad.numel() * sizeof(T));
+        v_grad_ptr = dev_ctx_.Alloc<T>(
+            &v_transpose_out_grad, v_transpose_out_grad.numel() * sizeof(T));
+      }
+
+      phi::DenseTensor softmax_out_grad;
+      softmax_out_grad.Resize(config->softmax_out_dims);
+      AllocWithDebugInfo<T>(dev_ctx_, "softmax_out_grad", &softmax_out_grad);
+
+      int64_t gemm_batch_size =
+          config->batch_size * config->seq_len_m * config->num_heads;
+      {
+        // Forward: fmha_out = transpose(qktv_out)
+        phi::DenseTensor qktv_out_grad;
+        qktv_out_grad.Resize(config->qktv_out_dims);
+        AllocWithDebugInfo<T>(dev_ctx_, "qktv_out_grad", &qktv_out_grad);
+        ComputeQKTVTransposeBackward(*fmha_out_grad, &qktv_out_grad);
+
+        // Forward: qktv_out = BatchedGEMM(softmax_out, V)
+        // Backward:
+        //  V_grad = BatchedGEMM(softmax_out^T, qktv_out_grad) (dy = x^T * dout)
+        int64_t gemm_m = config->m_size;
+        int64_t gemm_n = config->head_dim;
+        int64_t gemm_k = config->seq_len_r;
+
+        const T* softmax_out_ptr = softmax_out->data<T>();
+        const T* qktv_out_grad_ptr = qktv_out_grad.data<T>();
+        ComputeBatchedGEMM(softmax_out_ptr,
+                           qktv_out_grad_ptr,
+                           v_grad_ptr,
+                           true,
+                           false,
+                           gemm_m,
+                           gemm_n,
+                           gemm_k,
+                           gemm_batch_size);
+
+        // Backward: softmax_out_grad = qktv_out_grad * V^T (dx = dout * y^T)
+        gemm_m = config->seq_len_r;
+        gemm_n = config->m_size;
+        gemm_k = config->head_dim;
+
+        T* softmax_out_grad_ptr = softmax_out_grad.data<T>();
+        ComputeBatchedGEMM(qktv_out_grad_ptr,
+                           v_ptr,
+                           softmax_out_grad_ptr,
+                           false,
+                           true,
+                           gemm_m,
+                           gemm_n,
+                           gemm_k,
+                           gemm_batch_size);
+      }
+
+      phi::DenseTensor* qk_out_grad = config->GetQKOutGrad(&softmax_out_grad);
+      ComputeBiasMaskSoftmaxBackward(&softmax_out_grad,
+                                     softmax_out,
+                                     src_mask_grad,
+                                     qk_out_grad,
+                                     nonbatched_bias_grad);
+
+      // Forward: qk_out = BatchedGEMM(Q, K^T)
+      // Backward: k_grad = BatchedGEMM(qk_out_grad^T, Q) (dy = dout^t * x)
       int64_t gemm_m = config->m_size;
       int64_t gemm_n = config->head_dim;
       int64_t gemm_k = config->seq_len_r;
+      T alpha = static_cast<T>(1.0 / sqrt(config->head_dim));
 
-      const T* softmax_out_ptr = softmax_out->data<T>();
-      const T* qktv_out_grad_ptr = qktv_out_grad.data<T>();
-      ComputeBatchedGEMM(softmax_out_ptr,
-                         qktv_out_grad_ptr,
-                         v_grad_ptr,
+      T* qk_out_grad_ptr = qk_out_grad->data<T>();
+      ComputeBatchedGEMM(qk_out_grad_ptr,
+                         q_ptr,
+                         k_grad_ptr,
                          true,
                          false,
                          gemm_m,
                          gemm_n,
                          gemm_k,
-                         gemm_batch_size);
+                         gemm_batch_size,
+                         alpha);
 
-      // Backward: softmax_out_grad = qktv_out_grad * V^T (dx = dout * y^T)
+      // Backward: q_grad = BatchedGEMM(qk_out_grad, K) (dx = dout * y)
       gemm_m = config->seq_len_r;
-      gemm_n = config->m_size;
-      gemm_k = config->head_dim;
-
-      T* softmax_out_grad_ptr = softmax_out_grad.data<T>();
-      ComputeBatchedGEMM(qktv_out_grad_ptr,
-                         v_ptr,
-                         softmax_out_grad_ptr,
+      gemm_n = config->head_dim;
+      gemm_k = config->m_size;
+      ComputeBatchedGEMM(qk_out_grad_ptr,
+                         k_ptr,
+                         q_grad_ptr,
+                         false,
                          false,
-                         true,
                          gemm_m,
                          gemm_n,
                          gemm_k,
-                         gemm_batch_size);
+                         gemm_batch_size,
+                         alpha);
     }
 
-    phi::DenseTensor* qk_out_grad = config->GetQKOutGrad(&softmax_out_grad);
-    ComputeBiasMaskSoftmaxBackward(&softmax_out_grad,
-                                   softmax_out,
-                                   src_mask_grad,
-                                   qk_out_grad,
-                                   nonbatched_bias_grad);
-
-    // Forward: qk_out = BatchedGEMM(Q, K^T)
-    // Backward: k_grad = BatchedGEMM(qk_out_grad^T, Q) (dy = dout^t * x)
-    int64_t gemm_m = config->m_size;
-    int64_t gemm_n = config->head_dim;
-    int64_t gemm_k = config->seq_len_r;
-    T alpha = static_cast<T>(1.0 / sqrt(config->head_dim));
-
-    T* qk_out_grad_ptr = qk_out_grad->data<T>();
-    ComputeBatchedGEMM(qk_out_grad_ptr,
-                       q_ptr,
-                       k_grad_ptr,
-                       true,
-                       false,
-                       gemm_m,
-                       gemm_n,
-                       gemm_k,
-                       gemm_batch_size,
-                       alpha);
-
-    // Backward: q_grad = BatchedGEMM(qk_out_grad, K) (dx = dout * y)
-    gemm_m = config->seq_len_r;
-    gemm_n = config->head_dim;
-    gemm_k = config->m_size;
-    ComputeBatchedGEMM(qk_out_grad_ptr,
-                       k_ptr,
-                       q_grad_ptr,
-                       false,
-                       false,
-                       gemm_m,
-                       gemm_n,
-                       gemm_k,
-                       gemm_batch_size,
-                       alpha);
-
-    if (merge_qkv_) {
+    if (merge_qkv_ || config->use_flash_attn) {
       phi::DenseTensor* qkv_out_grad = config->GetQKVOutGrad();
       ComputeQKVTransposeBackward(qkv_transpose_out_grad, qkv_out_grad);
     } else {
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cc b/paddle/fluid/operators/fused/fused_gate_attention_op.cc
index e2d8926f53516..0c965ac08745a 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cc
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cc
@@ -284,12 +284,19 @@ class FusedGateAttentionGradOpMaker : public framework::SingleGradOpMaker<T> {
 
     op->SetAttrMap(this->Attrs());
     bool merge_qkv = PADDLE_GET_CONST(bool, op->GetAttr("merge_qkv"));
+    bool use_flash_attn = PADDLE_GET_CONST(bool, op->GetAttr("use_flash_attn"));
 
     if (merge_qkv) {
       op->SetInput("QKVWeight", this->Input("QKVWeight"));
       op->SetOutput(framework::GradVarName("QKVWeight"),
                     this->InputGrad("QKVWeight"));
       op->SetInput("QKVTransposeOut", this->Output("QKVTransposeOut"));
+
+      if (use_flash_attn) {
+        op->SetInput("NonbatchedBias", this->Input("NonbatchedBias"));
+        op->SetInput("SrcMask", this->Input("SrcMask"));
+        op->SetInput("SoftmaxLse", this->Output("SoftmaxLse"));
+      }
     } else {
       op->SetInput("Key", this->Input("Key"));
       op->SetOutput(framework::GradVarName("Key"), this->InputGrad("Key"));
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cu b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
index 584cb48c1ebe4..407ffe0a0820d 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cu
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
@@ -484,7 +484,7 @@ class FusedGateAttentionGradKernel : public framework::OpKernel<T> {
 
     bool has_gating = ctx.Attr<bool>("has_gating");
     bool merge_qkv = ctx.Attr<bool>("merge_qkv");
-    const bool use_flash_attn = ctx.Attr<bool>("use_flash_attn");
+    bool use_flash_attn = ctx.Attr<bool>("use_flash_attn");
 
     bool use_fused_matmul_bias = true;
     auto &dev_ctx = ctx.template device_context<phi::GPUContext>();
@@ -499,6 +499,17 @@ class FusedGateAttentionGradKernel : public framework::OpKernel<T> {
                                       has_gating,
                                       use_flash_attn);
 
+    const phi::DenseTensor *fwd_out = nullptr;
+    const phi::DenseTensor *fwd_bias = nullptr;
+    const phi::DenseTensor *fwd_mask = nullptr;
+    const phi::DenseTensor *fwd_softmax_lse = nullptr;
+    if (merge_qkv && use_flash_attn) {
+      fwd_bias = ctx.Input<phi::DenseTensor>("NonbatchedBias");
+      fwd_mask = ctx.Input<phi::DenseTensor>("SrcMask");
+      fwd_softmax_lse = ctx.Input<phi::DenseTensor>("SoftmaxLse");
+      fwd_out = fmha_out;
+    }
+
     phi::DenseTensor fmha_out_grad;
     fmha_out_grad.Resize(config.gate_out_dims);
     AllocWithDebugInfo<T>(dev_ctx, "fmha_out_grad", &fmha_out_grad);
@@ -533,15 +544,37 @@ class FusedGateAttentionGradKernel : public framework::OpKernel<T> {
     }
 
     auto fmha_compute = FMHAGateRef<T>(dev_ctx, merge_qkv);
-    fmha_compute.ComputeBackward(q_transpose_out,
-                                 k_transpose_out,
-                                 v_transpose_out,
-                                 qkv_transpose_out,
-                                 softmax_out,
-                                 &fmha_out_grad,
-                                 nullptr,
-                                 nonbatched_bias_grad,
-                                 &config);
+
+    if (use_flash_attn) {
+      const phi::DenseTensor *fwd_bias =
+          ctx.Input<phi::DenseTensor>("NonbatchedBias");
+      const phi::DenseTensor *fwd_mask = ctx.Input<phi::DenseTensor>("SrcMask");
+      const phi::DenseTensor *fwd_softmax_lse =
+          ctx.Input<phi::DenseTensor>("SoftmaxLse");
+      fmha_compute.ComputeBackward(q_transpose_out,
+                                   k_transpose_out,
+                                   v_transpose_out,
+                                   qkv_transpose_out,
+                                   softmax_out,
+                                   &fmha_out_grad,
+                                   nullptr,
+                                   nonbatched_bias_grad,
+                                   &config,
+                                   fmha_out,  // fwd_out
+                                   fwd_softmax_lse,
+                                   fwd_bias,
+                                   fwd_mask);
+    } else {
+      fmha_compute.ComputeBackward(q_transpose_out,
+                                   k_transpose_out,
+                                   v_transpose_out,
+                                   qkv_transpose_out,
+                                   softmax_out,
+                                   &fmha_out_grad,
+                                   nullptr,
+                                   nonbatched_bias_grad,
+                                   &config);
+    }
 
     bool use_addto = has_gating ? true : false;
     if (merge_qkv) {
diff --git a/paddle/phi/backends/dynload/flashattn.h b/paddle/phi/backends/dynload/flashattn.h
index 36ec693218f3c..8948ec6a46988 100644
--- a/paddle/phi/backends/dynload/flashattn.h
+++ b/paddle/phi/backends/dynload/flashattn.h
@@ -47,6 +47,7 @@ extern void* flashattn_dso_handle;
   __macro(flash_attn_fwd);                    \
   __macro(flash_attn_bwd);                    \
   __macro(flash_attn_fwd_with_bias_and_mask); \
+  __macro(flash_attn_bwd_with_bias_and_mask); \
   __macro(flash_attn_error);
 
 FLASHATTN_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_FLASHATTN_WRAP);

From 9f76b5f0650dfb4dbec6254f5501d1cffece6d24 Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Sun, 23 Apr 2023 17:27:35 +0800
Subject: [PATCH 007/405] fix bugs for backward

---
 cmake/external/flashattn.cmake                |   2 +-
 .../operators/fused/fused_gate_attention.h    | 108 +++++++++++-------
 2 files changed, 69 insertions(+), 41 deletions(-)

diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index 2cf1ddafaafdc..fd9459aad7072 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -20,7 +20,7 @@ set(FLASHATTN_PREFIX_DIR ${THIRD_PARTY_PATH}/flashattn)
 set(FLASHATTN_SOURCE_SUBDIR csrc/flash_attn)
 set(FLASHATTN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flashattn)
 set(FLASHATTN_REPOSITORY ${GIT_URL}/JamesLim-sy/flash-attention.git)
-set(FLASHATTN_TAG edeaea701e0c0e712f1a43a8970b5d59f5256e3b)
+set(FLASHATTN_TAG opitmization_for_alphafold2)
 
 #set(FLASHATTN_REPOSITORY ${GIT_URL}/PaddlePaddle/flash-attention.git)
 #set(FLASHATTN_TAG f0edf243a813a65d05c75fcb331b2a95faf96bbc)
diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index e43a9b77c848c..282b1940aa325 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -548,7 +548,7 @@ class FMHAGateRef {
       uint64_t inc = batch_size_ * num_heads_ * 32;
       auto seed_offset_pair = gen->IncrementOffset(inc);
       uint64_t seed = seed_offset_pair.first;
-      uint64_t offset = 1908576;  // seed_offset_pair.second;
+      uint64_t offset = seed_offset_pair.second;
 
       GetFlashAttnDimsString("softmax_out", softmax_out->dims());
       GetFlashAttnDimsString("softmax_lse", softmax_lse.dims());
@@ -566,8 +566,8 @@ class FMHAGateRef {
           static_cast<const void*>(k_ptr),
           static_cast<const void*>(v_ptr),
           nullptr,  // for calculation workspace size
-          static_cast<const void*>(cu_seq_q.data<int32_t>()),
-          static_cast<const void*>(cu_seq_k.data<int32_t>()),
+          cu_seq_q.data<int32_t>(),
+          cu_seq_k.data<int32_t>(),
           total_q_,
           total_k_,
           batch_size_,
@@ -628,8 +628,8 @@ class FMHAGateRef {
           static_cast<const void*>(v_ptr),
           static_cast<void*>(
               fmha_out->data()),  // for calculation workspace size
-          static_cast<const void*>(cu_seq_q.data<int32_t>()),
-          static_cast<const void*>(cu_seq_k.data<int32_t>()),
+          cu_seq_q.data<int32_t>(),
+          cu_seq_k.data<int32_t>(),
           total_q_,
           total_k_,
           batch_size_,
@@ -859,11 +859,6 @@ class FMHAGateRef {
 
       int seq_batch_size = static_cast<int>(config->batch_size) *
                            static_cast<int>(config->seq_len_m);
-      qkv_transpose_out->Resize(
-          {3,
-           seq_batch_size * static_cast<int>(config->seq_len_r),
-           static_cast<int>(config->num_heads),
-           static_cast<int>(config->head_dim)});
       DBG_WAIT;
 
       // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
@@ -871,7 +866,8 @@ class FMHAGateRef {
       int64_t start = 0;
       int64_t step = static_cast<int>(config->seq_len_r);
       int64_t end_size = (seq_batch_size + 1);
-      int64_t end = end_size, int64_t seq_size = 0;
+      int64_t end = end_size;
+      int64_t seq_size = 0;
       phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
       cu_seq_q.Resize({end_size});
       cu_seq_k.Resize({end_size});
@@ -908,7 +904,10 @@ class FMHAGateRef {
       dims_merge_func(src_mask, &temp_mask, "[Grad] mask_dim");
       dims_merge_func(nonbatched_bias, &temp_bias, "[Grad] bias_dim");
 
-      auto& qkv_dims = qkv_transpose_out->dims();
+      phi::DDim qkv_dims({3,
+                          seq_batch_size * static_cast<int>(config->seq_len_r),
+                          static_cast<int>(config->num_heads),
+                          static_cast<int>(config->head_dim)});
       int batch_size_ = seq_batch_size;
       int total_q_ = qkv_dims[1];    // q.dims()[0]
       int total_k_ = qkv_dims[1];    // q.dims()[0]
@@ -916,7 +915,6 @@ class FMHAGateRef {
       int head_size_ = qkv_dims[3];  // q.dims()[2]
       int max_seqlen_q_ = batch_size_;
       int max_seqlen_k_ = batch_size_;
-      int num_splits = 0;
       VLOG(6) << "[Flash_attn Grad] batch_size : " << batch_size_;
       VLOG(6) << "[Flash_attn Grad] total_q   : " << total_q_;
       VLOG(6) << "[Flash_attn Grad] total_k   : " << total_k_;
@@ -927,9 +925,9 @@ class FMHAGateRef {
 
       // 5. construct softmax_lse
       int last_q_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
-      softmax_lse->Resize({batch_size_, num_heads_, last_q_dim});
-      AllocWithDebugInfo<float>(
-          dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
+      // softmax_lse->Resize({batch_size_, num_heads_, last_q_dim});
+      // AllocWithDebugInfo<float>(
+      //     dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
       DBG_WAIT;
 
       phi::DenseTensor softmax_d = phi::Empty<float, phi::GPUContext>(
@@ -957,26 +955,33 @@ class FMHAGateRef {
       v_grad_ptr = dev_ctx_.Alloc<T>(&v_transpose_out_grad,
                                      v_transpose_out_grad.numel() * sizeof(T));
 
+      // 6. construct random seed
+      auto gen = dev_ctx_.GetGenerator();
+      uint64_t inc = batch_size_ * num_heads_ * 32;
+      auto seed_offset_pair = gen->IncrementOffset(inc);
+      uint64_t seed = seed_offset_pair.first;
+      uint64_t offset = seed_offset_pair.second;
+
       // 7. flas_attn part one, get temp worksapce size.
       uint64_t workspace_size;
       float p_dropout = 0.f;
       float softmax_scale = static_cast<float>(1);
       cudaStream_t stream = dev_ctx_.stream();
       int num_splits = 0;  // 0 for an internal heuristic, which is optimal
-      succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
+      bool succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
           static_cast<const void*>(q_ptr),
           static_cast<const void*>(k_ptr),
           static_cast<const void*>(v_ptr),
-          q_grad_ptr,
-          k_grad_ptr,
-          v_grad_ptr,
-          static_cast<void*>(
-              fmha_out->data()),  // total_q x num_heads x head_size, total_k :=
+          static_cast<void*>(q_grad_ptr),
+          static_cast<void*>(k_grad_ptr),
+          static_cast<void*>(v_grad_ptr),
+          static_cast<const void*>(
+              fmha_out->data()),  // total_q x num_heads x head_size, total_k :
                                   // \sum_{i=0}^{b} s_i
-          static_cast<void*>(
+          static_cast<const void*>(
               fmha_out_grad->data()),  // total_q x num_heads, x head_size
-          static_cast<const void*>(cu_seq_q.data<int32_t>()),
-          static_cast<const void*>(cu_seq_k.data<int32_t>()),
+          cu_seq_q.data<int32_t>(),
+          cu_seq_k.data<int32_t>(),
           total_q_,
           total_k_,
           batch_size_,
@@ -990,16 +995,16 @@ class FMHAGateRef {
           /*is_causal=*/false,
           is_bf16,
           num_splits,
-          softmax_lse.data(),
+          softmax_lse->data(),
           softmax_d.data(),
-          nullptr,
           bias_d.data(),
+          nullptr,
           &workspace_size,
           stream,
           seed,
           offset,
-          src_mask ? temp_mask.data() : nullptr,
           nonbatched_bias ? temp_bias.data() : nullptr,
+          src_mask ? temp_mask.data() : nullptr,
           temp_mask.dims().Get(),
           temp_bias.dims().Get());
       if (!succ) {
@@ -1020,16 +1025,13 @@ class FMHAGateRef {
           static_cast<const void*>(q_ptr),
           static_cast<const void*>(k_ptr),
           static_cast<const void*>(v_ptr),
-          q_grad_ptr,
-          k_grad_ptr,
-          v_grad_ptr,
-          static_cast<void*>(
-              fmha_out->data()),  // total_q x num_heads x head_size, total_k :=
-                                  // \sum_{i=0}^{b} s_i
-          static_cast<void*>(
-              fmha_out_grad->data()),  // total_q x num_heads, x head_size
-          static_cast<const void*>(cu_seq_q.data<int32_t>()),
-          static_cast<const void*>(cu_seq_k.data<int32_t>()),
+          static_cast<void*>(q_grad_ptr),
+          static_cast<void*>(k_grad_ptr),
+          static_cast<void*>(v_grad_ptr),
+          static_cast<const void*>(fmha_out->data()),
+          static_cast<const void*>(fmha_out_grad->data()),
+          cu_seq_q.data<int32_t>(),
+          cu_seq_k.data<int32_t>(),
           total_q_,
           total_k_,
           batch_size_,
@@ -1043,10 +1045,10 @@ class FMHAGateRef {
           /*is_causal=*/false,
           is_bf16,
           num_splits,
-          softmax_lse.data(),
+          softmax_lse->data(),
           softmax_d.data(),
-          workspace.data(),
           bias_d.data(),
+          workspace.data(),
           &workspace_size,
           stream,
           seed,
@@ -1055,6 +1057,32 @@ class FMHAGateRef {
           nonbatched_bias ? temp_bias.data() : nullptr,
           temp_mask.dims().Get(),
           temp_bias.dims().Get());
+      if (!succ) {
+        PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
+      }
+      DBG_WAIT;
+
+      if (nonbatched_bias) {
+        // compare block reduce
+        // auto size = attn_bias->sizes();
+        // dbias = ds.reshape({ -1, size[0], size[1], size[2], size[3] }).sum({
+        // 0 }); result.push_back( dbias );
+        const auto temp_bias_num = temp_bias.numel();
+        const auto bias_d_num = bias_d.numel();
+        auto dbias_first_dim = bias_d_num / temp_bias_num;
+        bias_d.Resize({dbias_first_dim,
+                       temp_bias.dims()[0],
+                       temp_bias.dims()[1],
+                       temp_bias.dims()[2],
+                       temp_bias.dims()[3]});
+        phi::funcs::
+            ReduceKernel<T, T, kps::AddFunctor, kps::IdentityFunctor<T>>(
+                dev_ctx_,
+                bias_d,
+                nonbatched_bias_grad,
+                kps::IdentityFunctor<T>(),
+                {0});
+      }
     } else {
       if (merge_qkv_) {
         PADDLE_ENFORCE_NOT_NULL(

From 05b3444faab07c59e37ac52d408252a730e31f46 Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Mon, 24 Apr 2023 10:57:37 +0800
Subject: [PATCH 008/405] 04-24 first commit

---
 paddle/fluid/operators/fused/fused_gate_attention.h   | 11 ++++-------
 .../fluid/operators/fused/fused_gate_attention_op.cu  |  2 +-
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 282b1940aa325..7cacf8f1b1f31 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -818,7 +818,7 @@ class FMHAGateRef {
                        const phi::DenseTensor* fmha_out = nullptr,
                        const phi::DenseTensor* softmax_lse = nullptr,
                        const phi::DenseTensor* nonbatched_bias = nullptr,
-                       const phi::DenseTensor* src_mask = = nullptr) {
+                       const phi::DenseTensor* src_mask = nullptr) {
     const T* q_ptr = nullptr;
     const T* k_ptr = nullptr;
     const T* v_ptr = nullptr;
@@ -975,11 +975,8 @@ class FMHAGateRef {
           static_cast<void*>(q_grad_ptr),
           static_cast<void*>(k_grad_ptr),
           static_cast<void*>(v_grad_ptr),
-          static_cast<const void*>(
-              fmha_out->data()),  // total_q x num_heads x head_size, total_k :
-                                  // \sum_{i=0}^{b} s_i
-          static_cast<const void*>(
-              fmha_out_grad->data()),  // total_q x num_heads, x head_size
+          static_cast<const void*>(fmha_out->data()),
+          static_cast<const void*>(fmha_out_grad->data()),
           cu_seq_q.data<int32_t>(),
           cu_seq_k.data<int32_t>(),
           total_q_,
@@ -1003,8 +1000,8 @@ class FMHAGateRef {
           stream,
           seed,
           offset,
-          nonbatched_bias ? temp_bias.data() : nullptr,
           src_mask ? temp_mask.data() : nullptr,
+          nonbatched_bias ? temp_bias.data() : nullptr,
           temp_mask.dims().Get(),
           temp_bias.dims().Get());
       if (!succ) {
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cu b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
index 407ffe0a0820d..446d89accfcae 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cu
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
@@ -560,7 +560,7 @@ class FusedGateAttentionGradKernel : public framework::OpKernel<T> {
                                    nullptr,
                                    nonbatched_bias_grad,
                                    &config,
-                                   fmha_out,  // fwd_out
+                                   fmha_out,
                                    fwd_softmax_lse,
                                    fwd_bias,
                                    fwd_mask);

From 41e90283ac58a1f46216cf7b6f1f4426bac688ad Mon Sep 17 00:00:00 2001
From: Zhang Ting <zhangting_2017@163.com>
Date: Mon, 24 Apr 2023 14:45:44 +0800
Subject: [PATCH 009/405] [AMP]expand blacklists for amp training (#50940)

---
 python/paddle/amp/__init__.py                 |   6 +-
 python/paddle/amp/amp_lists.py                | 110 ++++++++++++++++++
 python/paddle/amp/auto_cast.py                | 110 ++----------------
 ...perative_auto_mixed_precision_for_eager.py |   8 +-
 test/amp/test_amp_list.py                     |  63 +++++++---
 5 files changed, 175 insertions(+), 122 deletions(-)
 create mode 100644 python/paddle/amp/amp_lists.py

diff --git a/python/paddle/amp/__init__.py b/python/paddle/amp/__init__.py
index 60df9de03ad11..5fa8055ba233b 100644
--- a/python/paddle/amp/__init__.py
+++ b/python/paddle/amp/__init__.py
@@ -16,10 +16,8 @@
 from .auto_cast import decorate  # noqa: F401
 from .auto_cast import amp_guard  # noqa: F401
 from .auto_cast import amp_decorate  # noqa: F401
-from .auto_cast import FP16_WHITE_LIST  # noqa: F401
-from .auto_cast import FP16_BLACK_LIST  # noqa: F401
-from .auto_cast import PURE_FP16_WHITE_LIST  # noqa: F401
-from .auto_cast import PURE_FP16_BLACK_LIST  # noqa: F401
+from .amp_lists import white_list  # noqa: F401
+from .amp_lists import black_list  # noqa: F401
 
 from . import grad_scaler  # noqa: F401
 from .grad_scaler import GradScaler  # noqa: F401
diff --git a/python/paddle/amp/amp_lists.py b/python/paddle/amp/amp_lists.py
new file mode 100644
index 0000000000000..f70c8f5ed7f91
--- /dev/null
+++ b/python/paddle/amp/amp_lists.py
@@ -0,0 +1,110 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The set of ops that support fp16 calculation and are considered numerically-
+# safe and performance-critical. These ops are always converted to fp16.
+FP16_WHITE_LIST = {
+    'conv2d',
+    'matmul',
+    'matmul_v2',
+    'max_pool2d_with_index',
+    'mul',
+    'fake_quantize_dequantize_abs_max',
+    'fake_quantize_dequantize_moving_average_abs_max',
+}
+
+# The set of ops that support fp16 calculation and are considered numerically-
+# dangerous and whose effects may also be observed in downstream ops.
+FP16_BLACK_LIST = {
+    'tan',
+    'acos',
+    'asin',
+    'sinh',
+    'cosh',
+    'atanh',
+    'tanh_shrink',
+    'cos_sim',
+    'erfinv',
+    'exp',
+    'expm1',
+    'log',
+    'log10',
+    'log2',
+    'reciprocal',
+    'rsqrt',
+    'pow',
+    'square',
+    'reduce_sum',
+    'mean',
+    'reduce_mean',
+    'reduce_prod',
+    'cumprod',
+    'cumsum',
+    'dist',
+    'pnorm',
+    'frobenius_norm',
+    'renorm',
+    'group_norm',
+    'layer_norm',
+    'softmax',
+    'softmin',
+    'softplus',
+    'log_softmax',
+    'softmax_with_cross_entropy',
+    'sigmoid_cross_entropy_with_logits',
+    'c_softmax_with_cross_entropy',
+    'cross_entropy',
+    'cross_entropy2',
+    'nll_loss',
+    'huber_loss',
+    'triplet_margin_loss',
+    'log_loss',
+    'hsigmoid_loss',
+    'margin_cross_entropy',
+}
+
+# FP16 performance of grad op is worse than that of FP32. Use FP32 by default.
+FP16_EXTRA_BLACK_LIST = {
+    'linear_interp_v2',
+    'nearest_interp_v2',
+    'bilinear_interp_v2',
+    'bicubic_interp_v2',
+    'trilinear_interp_v2',
+    'lookup_table',
+    'lookup_table_v2',
+    'scatter',
+    'depthwise_conv2d',
+}
+
+BF16_WHITE_LIST = {'conv2d', 'matmul_v2'}
+BF16_BLACK_LIST = set()
+
+
+def white_list():
+    white_list = {
+        "float16": {"O1": FP16_WHITE_LIST, "O2": FP16_WHITE_LIST},
+        "bfloat16": {"O1": BF16_WHITE_LIST, "O2": BF16_WHITE_LIST},
+    }
+    return white_list
+
+
+def black_list():
+    black_list = {
+        "float16": {
+            "O1": FP16_BLACK_LIST | FP16_EXTRA_BLACK_LIST,
+            "O2": FP16_EXTRA_BLACK_LIST,
+        },
+        "bfloat16": {"O1": BF16_BLACK_LIST, "O2": set()},
+    }
+    return black_list
diff --git a/python/paddle/amp/auto_cast.py b/python/paddle/amp/auto_cast.py
index e8f552607affc..1f82533edbfb3 100644
--- a/python/paddle/amp/auto_cast.py
+++ b/python/paddle/amp/auto_cast.py
@@ -20,45 +20,7 @@
 from paddle.fluid.framework import _dygraph_tracer, dygraph_only
 from paddle.fluid.wrapped_decorator import signature_safe_contextmanager
 
-AMP_LEVEL = core.AmpLevel
-
-# The set of ops that support fp16 calculation and are considered numerically-
-# safe and performance-critical. These ops are always converted to fp16.
-FP16_WHITE_LIST = {
-    'conv2d',
-    'matmul',
-    'matmul_v2',
-    'max_pool2d_with_index',
-    'mul',
-    'fake_quantize_dequantize_abs_max',
-    'fake_quantize_dequantize_moving_average_abs_max',
-}
-
-# The set of ops that support fp16 calculation and are considered numerically-
-# dangerous and whose effects may also be observed in downstream ops.
-FP16_BLACK_LIST = {
-    'exp',
-    'square',
-    'log',
-    'mean',
-    'sum',
-    'cos_sim',
-    'softmax',
-    'softmax_with_cross_entropy',
-    'sigmoid_cross_entropy_with_logits',
-    'c_softmax_with_cross_entropy',
-    'cross_entropy',
-    'cross_entropy2',
-    # default fp32 can avoid return inf when the sum value large than 65504
-    'reduce_sum',
-    # FP16 performance of grad op is worse than that of FP32. Use FP32 by default.
-    'linear_interp_v2',
-    'nearest_interp_v2',
-    'bilinear_interp_v2',
-    'bicubic_interp_v2',
-    'trilinear_interp_v2',
-}
-
+from .amp_lists import black_list, white_list
 
 AMP_RELATED_FLAGS = [
     'FLAGS_cudnn_exhaustive_search',
@@ -72,27 +34,7 @@
     'FLAGS_cudnn_batchnorm_spatial_persistent': 1,
 }
 
-PURE_FP16_WHITE_LIST = copy.copy(FP16_WHITE_LIST)
-
-PURE_FP16_BLACK_LIST = {
-    'lookup_table',
-    'lookup_table_v2',
-    'scatter',
-    'scatter_grad',
-    # FP16 performance of grad op is worse than that of FP32. Use FP32 by default.
-    'linear_interp_v2',
-    'nearest_interp_v2',
-    'bilinear_interp_v2',
-    'bicubic_interp_v2',
-    'trilinear_interp_v2',
-}
-
-BF16_WHITE_LIST = {'conv2d', 'matmul_v2'}
-BF16_BLACK_LIST = set()
-
-PURE_BF16_WHITE_LIST = copy.copy(BF16_WHITE_LIST)
-PURE_BF16_BLACK_LIST = set()
-
+AMP_LEVEL = core.AmpLevel
 _g_amp_state_ = None
 
 
@@ -126,20 +68,12 @@ def _update_list(
     """
     Update black and white list according to users' custom list.
     """
-    if dtype == 'float16':
-        if level == 'O1':
-            _white_list = copy.copy(FP16_WHITE_LIST)
-            _black_list = copy.copy(FP16_BLACK_LIST)
-        else:
-            _white_list = copy.copy(PURE_FP16_WHITE_LIST)
-            _black_list = copy.copy(PURE_FP16_BLACK_LIST)
-    else:
-        if level == 'O1':
-            _white_list = copy.copy(BF16_WHITE_LIST)
-            _black_list = copy.copy(BF16_BLACK_LIST)
-        else:
-            _white_list = copy.copy(PURE_BF16_WHITE_LIST)
-            _black_list = copy.copy(PURE_BF16_BLACK_LIST)
+    if level == 'O0':
+        _white_list = set()
+        _black_list = set()
+        return _white_list, _black_list
+    _white_list = copy.copy(white_list()[dtype][level])
+    _black_list = copy.copy(black_list()[dtype][level])
     if custom_white_list and custom_black_list:
         for op_name in custom_white_list:
             if op_name in custom_black_list:
@@ -453,34 +387,14 @@ def amp_guard(
 
     if level == 'O1':
         amp_level = AMP_LEVEL.O1
-        if dtype == 'float16':
-            _white_list = FP16_WHITE_LIST
-            _black_list = FP16_BLACK_LIST
-        elif dtype == 'bfloat16':
-            _white_list = BF16_WHITE_LIST
-            _black_list = BF16_BLACK_LIST
-
     elif level == 'O2':
         amp_level = AMP_LEVEL.O2
-        if dtype == 'float16':
-            _white_list = PURE_FP16_WHITE_LIST
-            _black_list = PURE_FP16_BLACK_LIST
-        elif dtype == 'bfloat16':
-            _white_list = BF16_WHITE_LIST
-            _black_list = BF16_BLACK_LIST
     elif level == 'O0':
         amp_level = AMP_LEVEL.O0
-        if dtype == 'float16':
-            _white_list = FP16_WHITE_LIST
-            _black_list = FP16_BLACK_LIST
-        elif dtype == 'bfloat16':
-            _white_list = BF16_WHITE_LIST
-            _black_list = BF16_BLACK_LIST
-
-    if custom_white_list or custom_black_list:
-        _white_list, _black_list = _update_list(
-            custom_white_list, custom_black_list, level, dtype
-        )
+
+    _white_list, _black_list = _update_list(
+        custom_white_list, custom_black_list, level, dtype
+    )
 
     if not enable:
         amp_level = AMP_LEVEL.O0
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py
index 7a7d65d27d556..8d24febaff213 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py
@@ -88,8 +88,8 @@ def test_amp_guard_black_op(self):
     def custom_op_list(self):
         with fluid.dygraph.guard():
             tracer = fluid.framework._dygraph_tracer()
-            base_white_list = paddle.amp.FP16_WHITE_LIST
-            base_black_list = paddle.amp.FP16_BLACK_LIST
+            base_white_list = paddle.amp.white_list()["float16"]["O1"]
+            base_black_list = paddle.amp.black_list()["float16"]["O1"]
             with paddle.amp.amp_guard(
                 custom_white_list=["log"], custom_black_list=["conv2d"]
             ):
@@ -104,8 +104,8 @@ def custom_op_list(self):
                     == (set(base_black_list) - {"log"}) | {"conv2d"}
                 )
 
-            base_white_list = paddle.amp.PURE_FP16_WHITE_LIST
-            base_black_list = paddle.amp.PURE_FP16_BLACK_LIST
+            base_white_list = paddle.amp.white_list()["float16"]["O2"]
+            base_black_list = paddle.amp.black_list()["float16"]["O2"]
             with paddle.amp.amp_guard(
                 custom_white_list=["log"],
                 custom_black_list=["conv2d"],
diff --git a/test/amp/test_amp_list.py b/test/amp/test_amp_list.py
index 11bcdbfd3ba6a..9b0bf5129c36d 100644
--- a/test/amp/test_amp_list.py
+++ b/test/amp/test_amp_list.py
@@ -14,32 +14,63 @@
 
 import unittest
 
+import paddle
 from paddle.fluid import core
-from paddle.static.amp import fp16_lists
-from paddle.static.amp.fp16_lists import AutoMixedPrecisionLists
+from paddle.static.amp import AutoMixedPrecisionLists, fp16_lists
 
 
 class TestAMPList(unittest.TestCase):
-    def test_main(self):
-        custom_white_list = [
-            'lookup_table',
-            'lookup_table_v2',
-        ]
-        amp_list = AutoMixedPrecisionLists(custom_white_list=custom_white_list)
-        for op in custom_white_list:
-            self.assertTrue(op in amp_list.white_list)
-            self.assertTrue(op not in amp_list.black_list)
-            self.assertTrue(op not in amp_list.unsupported_list)
-
-        default_black_list = [
+    def setUp(self):
+        self.default_black_list = [
             'linear_interp_v2',
             'nearest_interp_v2',
             'bilinear_interp_v2',
             'bicubic_interp_v2',
             'trilinear_interp_v2',
         ]
-        for op in default_black_list:
-            self.assertTrue(op in amp_list.black_list)
+        self.custom_white_list = [
+            'lookup_table',
+            'lookup_table_v2',
+        ]
+
+    def check_if_op_in_list(self, op_list, amp_list):
+        for op in op_list:
+            self.assertTrue(op in amp_list)
+
+    def check_if_op_not_in_list(self, op_list, amp_list):
+        for op in op_list:
+            self.assertTrue(op not in amp_list)
+
+    def test_static(self):
+        amp_list = AutoMixedPrecisionLists(
+            custom_white_list=self.custom_white_list
+        )
+        self.check_if_op_in_list(self.default_black_list, amp_list.black_list)
+        self.check_if_op_in_list(self.custom_white_list, amp_list.white_list)
+        self.check_if_op_not_in_list(
+            self.custom_white_list, amp_list.black_list
+        )
+        self.check_if_op_not_in_list(
+            self.custom_white_list, amp_list.unsupported_list
+        )
+
+    def test_eager(self):
+        if not paddle.amp.is_float16_supported():
+            return
+        white_list = paddle.amp.white_list()
+        black_list = paddle.amp.black_list()
+        self.check_if_op_in_list(
+            self.default_black_list, black_list["float16"]["O2"]
+        )
+        self.check_if_op_not_in_list(['log', 'elementwise_add'], white_list)
+        with paddle.amp.auto_cast(custom_white_list={'elementwise_add'}):
+            out1 = paddle.rand([2, 3]) + paddle.rand([2, 3])
+            out2 = out1.mean()
+            out3 = paddle.log(out2)
+        self.check_if_op_not_in_list(['log', 'elementwise_add'], white_list)
+        self.assertEqual(out1.dtype, paddle.float16)
+        self.assertEqual(out2.dtype, paddle.float32)
+        self.assertEqual(out3.dtype, paddle.float32)
 
     def test_apis(self):
         def _run_check_dtype():

From d71615dc2b07dc92034256fd705187d08d9436e1 Mon Sep 17 00:00:00 2001
From: csy0225 <78470701+csy0225@users.noreply.github.com>
Date: Mon, 24 Apr 2023 14:54:49 +0800
Subject: [PATCH 010/405] shared_external mermory add xpu (#53240)

---
 paddle/fluid/inference/api/details/zero_copy_tensor.cc | 9 ++++++++-
 paddle/phi/kernels/xpu/linspace_kernel.cc              | 6 +++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/paddle/fluid/inference/api/details/zero_copy_tensor.cc b/paddle/fluid/inference/api/details/zero_copy_tensor.cc
index 147db87a101d4..c10e6b4a43fdd 100644
--- a/paddle/fluid/inference/api/details/zero_copy_tensor.cc
+++ b/paddle/fluid/inference/api/details/zero_copy_tensor.cc
@@ -345,9 +345,16 @@ void Tensor::ShareExternalData(const T *data,
             const_cast<T *>(data), size, paddle::platform::CUDAPlace(device_)),
         meta);
     *tensor = std::move(dtensor);
+  } else if (place == PlaceType::kXPU) {
+    phi::DenseTensor dtensor(
+        std::make_shared<phi::Allocation>(
+            const_cast<T *>(data), size, paddle::platform::XPUPlace(device_)),
+        meta);
+    *tensor = std::move(dtensor);
   } else {
     PADDLE_THROW(paddle::platform::errors::InvalidArgument(
-        "PlaceType must be PlaceType::kCPU or PlaceType::kGPU."));
+        "PlaceType must be one of [PlaceType::kCPU, PlaceType::kGPU, "
+        "PlaceType::kXPU]."));
   }
 }
 
diff --git a/paddle/phi/kernels/xpu/linspace_kernel.cc b/paddle/phi/kernels/xpu/linspace_kernel.cc
index e33a6d73f1c5d..e61676bae49c7 100644
--- a/paddle/phi/kernels/xpu/linspace_kernel.cc
+++ b/paddle/phi/kernels/xpu/linspace_kernel.cc
@@ -81,4 +81,8 @@ void LinspaceKernel(const Context& ctx,
 }  // namespace phi
 
 PD_REGISTER_KERNEL(
-    linspace, XPU, ALL_LAYOUT, phi::LinspaceKernel, float, int32_t) {}
+    linspace, XPU, ALL_LAYOUT, phi::LinspaceKernel, float, int32_t) {
+  kernel->InputAt(0).SetBackend(phi::Backend::ALL_BACKEND);
+  kernel->InputAt(1).SetBackend(phi::Backend::ALL_BACKEND);
+  kernel->InputAt(2).SetBackend(phi::Backend::ALL_BACKEND);
+}

From 7a9754a7e72d218fdf839762be820c70177ca61c Mon Sep 17 00:00:00 2001
From: HongyuJia <jiahongyu@baidu.com>
Date: Mon, 24 Apr 2023 14:59:57 +0800
Subject: [PATCH 011/405] [CppExtension Cuda] Add cuda unit test for
 CppExtension (#52900)

* [CppExtension Cuda] Add cuda unit test for CppExtension

* update extra_compile_args for CUDAExtension

* add debug info

* Add patch to fix CUDA12 compile error

* patch for all env

* add windows judgement

* Try to fix setup function not found error

* fix mix_relu_and_extension include file

* fix setup compile error

* remove useless debug comments

* add sleep, debug CI-build

* add space to disable cmake cache

* remove debug info

* add space to pass CI-build
---
 cmake/external/pybind11.cmake                 |  9 ++++
 cmake/external/warpctc.cmake                  |  1 +
 patches/pybind/cast.h.patch                   | 15 +++++++
 .../cpp_extension/cpp_extension_setup.py      | 14 ++++--
 .../tests/cpp_extension/custom_extension.cc   |  3 ++
 .../cpp_extension/custom_relu_forward.cu      | 45 +++++++++++++++++++
 .../mix_relu_and_extension_setup.py           |  2 +-
 .../cpp_extension/test_cpp_extension_jit.py   | 13 +++++-
 .../cpp_extension/test_cpp_extension_setup.py | 13 ++++++
 .../paddle/fluid/tests/cpp_extension/utils.py | 42 +++++++++++++++++
 .../utils/cpp_extension/extension_utils.py    |  9 ++--
 11 files changed, 156 insertions(+), 10 deletions(-)
 create mode 100644 patches/pybind/cast.h.patch
 create mode 100644 python/paddle/fluid/tests/cpp_extension/custom_relu_forward.cu

diff --git a/cmake/external/pybind11.cmake b/cmake/external/pybind11.cmake
index 6abd24e87305b..db53e3511be44 100644
--- a/cmake/external/pybind11.cmake
+++ b/cmake/external/pybind11.cmake
@@ -21,6 +21,14 @@ set(PYBIND_TAG v2.10.3)
 set(PYBIND_INCLUDE_DIR ${THIRD_PARTY_PATH}/pybind/src/extern_pybind/include)
 include_directories(${PYBIND_INCLUDE_DIR})
 
+set(PYBIND_PATCH_COMMAND "")
+if(NOT WIN32)
+  file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/pybind/cast.h.patch
+       native_dst)
+  set(PYBIND_PATCH_COMMAND patch -d ${PYBIND_INCLUDE_DIR}/pybind11 <
+                           ${native_dst})
+endif()
+
 ExternalProject_Add(
   extern_pybind
   ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
@@ -33,6 +41,7 @@ ExternalProject_Add(
   # third-party library version changes cannot be incorporated.
   # reference: https://cmake.org/cmake/help/latest/module/ExternalProject.html
   UPDATE_COMMAND ""
+  PATCH_COMMAND ${PYBIND_PATCH_COMMAND}
   CONFIGURE_COMMAND ""
   BUILD_COMMAND ""
   INSTALL_COMMAND ""
diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake
index e1e7234da0e25..46befee8bd255 100644
--- a/cmake/external/warpctc.cmake
+++ b/cmake/external/warpctc.cmake
@@ -82,6 +82,7 @@ else()
   set(WARPCTC_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE})
   set(WARPCTC_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
 endif()
+
 ExternalProject_Add(
   extern_warpctc
   ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
diff --git a/patches/pybind/cast.h.patch b/patches/pybind/cast.h.patch
new file mode 100644
index 0000000000000..ebd65571ebf82
--- /dev/null
+++ b/patches/pybind/cast.h.patch
@@ -0,0 +1,15 @@
+diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h
+index 3a404602..9054478c 100644
+--- a/include/pybind11/cast.h
++++ b/include/pybind11/cast.h
+@@ -42,7 +42,9 @@ using make_caster = type_caster<intrinsic_t<type>>;
+ // Shortcut for calling a caster's `cast_op_type` cast operator for casting a type_caster to a T
+ template <typename T>
+ typename make_caster<T>::template cast_op_type<T> cast_op(make_caster<T> &caster) {
+-    return caster.operator typename make_caster<T>::template cast_op_type<T>();
++    // https://github.com/pybind/pybind11/issues/4606 with CUDA 12
++    //return caster.operator typename make_caster<T>::template cast_op_type<T>();
++    return caster;
+ }
+ template <typename T>
+ typename make_caster<T>::template cast_op_type<typename std::add_rvalue_reference<T>::type>
diff --git a/python/paddle/fluid/tests/cpp_extension/cpp_extension_setup.py b/python/paddle/fluid/tests/cpp_extension/cpp_extension_setup.py
index b5c12284c117e..5a4ff2afd6c63 100644
--- a/python/paddle/fluid/tests/cpp_extension/cpp_extension_setup.py
+++ b/python/paddle/fluid/tests/cpp_extension/cpp_extension_setup.py
@@ -15,7 +15,9 @@
 import os
 from site import getsitepackages
 
-from paddle.utils.cpp_extension import CppExtension, setup
+from utils import extra_compile_args
+
+from paddle.utils.cpp_extension import CUDAExtension, setup
 
 paddle_includes = []
 for site_packages_path in getsitepackages():
@@ -30,10 +32,14 @@
 
 setup(
     name='custom_cpp_extension',
-    ext_modules=CppExtension(
-        sources=["custom_extension.cc", "custom_sub.cc"],
+    ext_modules=CUDAExtension(
+        sources=[
+            "custom_extension.cc",
+            "custom_sub.cc",
+            "custom_relu_forward.cu",
+        ],
         include_dirs=paddle_includes,
-        extra_compile_args={'cc': ['-w', '-g']},
+        extra_compile_args=extra_compile_args,
         verbose=True,
     ),
 )
diff --git a/python/paddle/fluid/tests/cpp_extension/custom_extension.cc b/python/paddle/fluid/tests/cpp_extension/custom_extension.cc
index 2334e23af536c..2fc5c42a80d75 100644
--- a/python/paddle/fluid/tests/cpp_extension/custom_extension.cc
+++ b/python/paddle/fluid/tests/cpp_extension/custom_extension.cc
@@ -20,6 +20,8 @@
 
 paddle::Tensor custom_sub(paddle::Tensor x, paddle::Tensor y);
 
+paddle::Tensor relu_cuda_forward(const paddle::Tensor& x);
+
 paddle::Tensor custom_add(const paddle::Tensor& x, const paddle::Tensor& y) {
   return x.exp() + y.exp();
 }
@@ -46,6 +48,7 @@ PYBIND11_MODULE(custom_cpp_extension, m) {
   m.def("nullable_tensor", &nullable_tensor, "returned Tensor might be None");
   m.def(
       "optional_tensor", &optional_tensor, "returned Tensor might be optional");
+  m.def("relu_cuda_forward", &relu_cuda_forward, "relu(x)");
 
   py::class_<Power>(m, "Power")
       .def(py::init<int, int>())
diff --git a/python/paddle/fluid/tests/cpp_extension/custom_relu_forward.cu b/python/paddle/fluid/tests/cpp_extension/custom_relu_forward.cu
new file mode 100644
index 0000000000000..e0405309f7add
--- /dev/null
+++ b/python/paddle/fluid/tests/cpp_extension/custom_relu_forward.cu
@@ -0,0 +1,45 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/extension.h"
+
+#define CHECK_GPU_INPUT(x) PD_CHECK(x.is_gpu(), #x " must be a GPU Tensor.")
+
+template <typename data_t>
+__global__ void relu_cuda_forward_kernel(const data_t* x,
+                                         data_t* y,
+                                         int64_t num) {
+  int64_t gid = blockIdx.x * blockDim.x + threadIdx.x;
+  for (int64_t i = gid; i < num; i += blockDim.x * gridDim.x) {
+    y[i] = x[i] > static_cast<data_t>(0.) ? x[i] : static_cast<data_t>(0.);
+  }
+}
+
+paddle::Tensor relu_cuda_forward(const paddle::Tensor& x) {
+  CHECK_GPU_INPUT(x);
+  auto out = paddle::empty_like(x);
+
+  PD_CHECK(x.place() == paddle::DefaultGPUPlace());
+
+  int64_t numel = x.numel();
+  int64_t block = 512;
+  int64_t grid = (numel + block - 1) / block;
+  PD_DISPATCH_FLOATING_AND_HALF_TYPES(
+      x.type(), "relu_cuda_forward_kernel", ([&] {
+        relu_cuda_forward_kernel<data_t><<<grid, block, 0, x.stream()>>>(
+            x.data<data_t>(), out.data<data_t>(), numel);
+      }));
+
+  return out;
+}
diff --git a/python/paddle/fluid/tests/cpp_extension/mix_relu_and_extension_setup.py b/python/paddle/fluid/tests/cpp_extension/mix_relu_and_extension_setup.py
index 3766d33f03443..823d0183cfda8 100644
--- a/python/paddle/fluid/tests/cpp_extension/mix_relu_and_extension_setup.py
+++ b/python/paddle/fluid/tests/cpp_extension/mix_relu_and_extension_setup.py
@@ -21,7 +21,7 @@
 setup(
     name='mix_relu_extension',
     ext_modules=CppExtension(
-        sources=["mix_relu_and_extension.cc", "custom_sub.cc"],
+        sources=["mix_relu_and_extension.cc"],
         include_dirs=paddle_includes
         + [os.path.dirname(os.path.abspath(__file__))],
         extra_compile_args={'cc': ['-w', '-g']},
diff --git a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py b/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py
index 9ed330a2b4ac7..bc6f8113afd91 100644
--- a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py
+++ b/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py
@@ -18,6 +18,7 @@
 from site import getsitepackages
 
 import numpy as np
+from utils import check_output
 
 import paddle
 from paddle.utils.cpp_extension import load
@@ -27,7 +28,7 @@
     sys.exit()
 
 # Compile and load cpp extension Just-In-Time.
-sources = ["custom_extension.cc", "custom_sub.cc"]
+sources = ["custom_extension.cc", "custom_sub.cc", "custom_relu_forward.cu"]
 paddle_includes = []
 for site_packages_path in getsitepackages():
     paddle_includes.append(
@@ -69,6 +70,8 @@ def test_cpp_extension(self):
         self._test_extension_class()
         self._test_nullable_tensor()
         self._test_optional_tensor()
+        if paddle.is_compiled_with_cuda():
+            self._test_cuda_relu()
 
     def _test_extension_function(self):
         for dtype in self.dtypes:
@@ -130,6 +133,14 @@ def _test_optional_tensor(self):
             err_msg=f'extension out: {x},\n numpy out: {x_np}',
         )
 
+    def _test_cuda_relu(self):
+        paddle.set_device('gpu')
+        x = np.random.uniform(-1, 1, [4, 8]).astype('float32')
+        x = paddle.to_tensor(x, dtype='float32')
+        out = custom_cpp_extension.relu_cuda_forward(x)
+        pd_out = paddle.nn.functional.relu(x)
+        check_output(out, pd_out, "out")
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py b/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py
index 5c8c91ed30356..53dffde432095 100644
--- a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py
+++ b/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py
@@ -18,6 +18,7 @@
 import unittest
 
 import numpy as np
+from utils import check_output
 
 import paddle
 from paddle import static
@@ -154,6 +155,8 @@ def test_cpp_extension(self):
         self._test_static()
         self._test_dynamic()
         self._test_double_grad_dynamic()
+        if paddle.is_compiled_with_cuda():
+            self._test_cuda_relu()
 
     def _test_extension_function_plain(self):
         import custom_cpp_extension
@@ -314,6 +317,16 @@ def _test_double_grad_dynamic(self):
                 ),
             )
 
+    def _test_cuda_relu(self):
+        import custom_cpp_extension
+
+        paddle.set_device('gpu')
+        x = np.random.uniform(-1, 1, [4, 8]).astype('float32')
+        x = paddle.to_tensor(x, dtype='float32')
+        out = custom_cpp_extension.relu_cuda_forward(x)
+        pd_out = paddle.nn.functional.relu(x)
+        check_output(out, pd_out, "out")
+
 
 if __name__ == '__main__':
     if os.name == 'nt' or sys.platform.startswith('darwin'):
diff --git a/python/paddle/fluid/tests/cpp_extension/utils.py b/python/paddle/fluid/tests/cpp_extension/utils.py
index 5c5a458a5c7a6..19659c6d5d716 100644
--- a/python/paddle/fluid/tests/cpp_extension/utils.py
+++ b/python/paddle/fluid/tests/cpp_extension/utils.py
@@ -16,6 +16,8 @@
 import sys
 from site import getsitepackages
 
+import numpy as np
+
 from paddle.utils.cpp_extension.extension_utils import IS_WINDOWS
 
 IS_MAC = sys.platform.startswith('darwin')
@@ -37,3 +39,43 @@
 extra_cc_args = ['-w', '-g'] if not IS_WINDOWS else ['/w']
 extra_nvcc_args = ['-O3']
 extra_compile_args = {'cc': extra_cc_args, 'nvcc': extra_nvcc_args}
+
+
+def check_output(out, pd_out, name):
+    if out is None and pd_out is None:
+        return
+    assert out is not None, "out value of " + name + " is None"
+    assert pd_out is not None, "pd_out value of " + name + " is None"
+    if isinstance(out, list) and isinstance(pd_out, list):
+        for idx in range(len(out)):
+            np.testing.assert_array_equal(
+                out[idx],
+                pd_out[idx],
+                err_msg='custom op {}: {},\n paddle api {}: {}'.format(
+                    name, out[idx], name, pd_out[idx]
+                ),
+            )
+    else:
+        np.testing.assert_array_equal(
+            out,
+            pd_out,
+            err_msg='custom op {}: {},\n paddle api {}: {}'.format(
+                name, out, name, pd_out
+            ),
+        )
+
+
+def check_output_allclose(out, pd_out, name, rtol=5e-5, atol=1e-2):
+    if out is None and pd_out is None:
+        return
+    assert out is not None, "out value of " + name + " is None"
+    assert pd_out is not None, "pd_out value of " + name + " is None"
+    np.testing.assert_allclose(
+        out,
+        pd_out,
+        rtol,
+        atol,
+        err_msg='custom op {}: {},\n paddle api {}: {}'.format(
+            name, out, name, pd_out
+        ),
+    )
diff --git a/python/paddle/utils/cpp_extension/extension_utils.py b/python/paddle/utils/cpp_extension/extension_utils.py
index 511b72378c78f..7da4d1f1ba122 100644
--- a/python/paddle/utils/cpp_extension/extension_utils.py
+++ b/python/paddle/utils/cpp_extension/extension_utils.py
@@ -180,6 +180,9 @@ def custom_write_stub(resource, pyfile):
 
         def __bootstrap__():
             assert os.path.exists(so_path)
+            # load custom op shared library with abs path
+            custom_ops = paddle.utils.cpp_extension.load_op_meta_info_and_register_op(so_path)
+
             if os.name == 'nt' or sys.platform.startswith('darwin'):
                 # Cpp Extension only support Linux now
                 mod = types.ModuleType(__name__)
@@ -193,10 +196,8 @@ def __bootstrap__():
                 except ImportError:
                     mod = types.ModuleType(__name__)
 
-            # load custom op shared library with abs path
-            custom_ops = paddle.utils.cpp_extension.load_op_meta_info_and_register_op(so_path)
-            for custom_ops in custom_ops:
-                setattr(mod, custom_ops, eval(custom_ops))
+            for custom_op in custom_ops:
+                setattr(mod, custom_op, eval(custom_op))
 
         __bootstrap__()
 

From 680460fd37778c5f78925a2471609b4addf7f393 Mon Sep 17 00:00:00 2001
From: Yiqun Liu <Xreki@users.noreply.github.com>
Date: Mon, 24 Apr 2023 15:12:20 +0800
Subject: [PATCH 012/405] [AMP] Allow to enable multi_precision through
 paddle.static.amp.decorate and add documents for some apis. (#53012)

* Add document for some apis. test=docs_preview

* Allow to set master_weight in paddle.static.amp.decorate.

* Polish codes and add unittest.

* Refine docs.

* Remove the repetitive function.
---
 python/paddle/static/amp/debugging.py | 101 ++++++++++++++++++++---
 python/paddle/static/amp/decorator.py | 111 ++++++++++++++++++++++++--
 test/amp/amp_base_models.py           |   1 -
 test/amp/test_model_cast_to_bf16.py   |  23 ++++++
 4 files changed, 220 insertions(+), 16 deletions(-)

diff --git a/python/paddle/static/amp/debugging.py b/python/paddle/static/amp/debugging.py
index 28abe84c39b2e..5a894495d98f5 100644
--- a/python/paddle/static/amp/debugging.py
+++ b/python/paddle/static/amp/debugging.py
@@ -13,8 +13,14 @@
 # limitations under the License.
 
 import copy
+import logging
 
 import paddle
+from paddle.fluid.log_helper import get_logger
+
+_logger = get_logger(
+    __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
+)
 
 
 class OperatorStatsUnit:
@@ -76,7 +82,7 @@ def _get_var_dtype_from_block(block, op, arg_name, is_input):
         var = block._var_recursive(var_name)
         return var.dtype
     except:
-        print(
+        _logger.warning(
             "Operator < {} > gets {} < {} : {} > error!".format(
                 op.type, "input" if is_input else "output", arg_name, var_name
             )
@@ -99,7 +105,7 @@ def _extract_compute_dtype(op, block):
                 if _is_floating_point(compute_dtype) and _is_floating_point(
                     var_dtype
                 ):
-                    print(
+                    _logger.warning(
                         "Operator < {} > has different input data types, input_names = {}, output_names = {}.".format(
                             op.type, op.input_names, op.output_names
                         )
@@ -125,7 +131,7 @@ def _extract_compute_dtype(op, block):
                 if _is_floating_point(compute_dtype) and _is_floating_point(
                     var_dtype
                 ):
-                    print(
+                    _logger.warning(
                         "Operator < {} > has different input / output data types, input_names = {}, output_names = {}.".format(
                             op.type, op.input_names, op.output_names
                         )
@@ -145,6 +151,15 @@ def _merge_op_stats(op_stats_list):
 
 
 def _get_op_stats_list(program):
+    def _is_special_ops_with_input_x(op_type):
+        # operators have input X and have inputs different dtypes.
+        special_op_list = ['cast', 'batch_norm', 'instance_norm', 'layer_norm']
+        if op_type in special_op_list:
+            return True
+        if op_type.replace("_grad", "") in special_op_list:
+            return True
+        return False
+
     op_stats_list = []
     for block in program.blocks:
         block_op_stats_dict = {}
@@ -161,13 +176,7 @@ def _get_op_stats_list(program):
                 'create_double_buffer_reader',
             ]:
                 compute_dtype = None
-            elif op.type in [
-                'cast',
-                'layer_norm',
-                'layer_norm_grad',
-                'batch_norm',
-                'batch_norm_grad',
-            ]:
+            elif _is_special_ops_with_input_x(op.type):
                 # Not check the input and output dtype difference for this operators.
                 compute_dtype = _get_var_dtype_from_block(block, op, 'X', True)
             elif "Param" in op.input_names:
@@ -183,6 +192,78 @@ def _get_op_stats_list(program):
 
 
 def collect_operator_stats(program=None, print_subblocks=False):
+    """
+    Collect the number of operators for different data types through parsing
+    the program. The statistical data are categorized according to four data
+    types, namely float32, float16, bfloat16 and others.
+
+    Args:
+        program(Program, optional): The program to parse. Default None, and the default main_program will be parsed.
+        print_subblocks(bool, optional): Whether to print the operator stats for each subblock. Default False.
+
+    Examples:
+
+     .. code-block:: python
+
+        import paddle
+
+        paddle.enable_static()
+
+        class SimpleConvNet(paddle.nn.Layer):
+            def __init__(self):
+                super().__init__()
+                self.conv = paddle.nn.Conv2D(in_channels=1, out_channels=6, kernel_size=3)
+                self.linear = paddle.nn.Linear(in_features=26, out_features=10)
+
+            def forward(self, x):
+                out = self.conv(x)
+                out = paddle.nn.functional.relu(out)
+                out = self.linear(out)
+                out = paddle.nn.functional.softmax(out)
+                return out
+
+        main_program = paddle.static.Program()
+        startup_program = paddle.static.Program()
+        with paddle.utils.unique_name.guard():
+            with paddle.static.program_guard(main_program, startup_program):
+                model = SimpleConvNet()
+                x = paddle.static.data(
+                    name='input', shape=[None, 1, 28, 28], dtype='float32'
+                )
+                out = model(x)
+                loss = paddle.mean(out)
+                optimizer = paddle.optimizer.AdamW()
+                optimizer = paddle.static.amp.decorate(optimizer)
+                optimizer.minimize(loss)
+        paddle.static.amp.debugging.collect_operator_stats(main_program)
+        # <------------------------------------------------ op list of all blocks ------------------------------------------------->
+        # <------------------------------------------------------- op list -------------------------------------------------------->
+        # <--------------- Op Name ---------------- | -- FP16 Calls --- | -- BF16 Calls --- | --- FP32 Calls--- | -- Other Calls -->
+        #   adamw                                   |  0                |  0                |  4                |  0
+        #   cast                                    |  5                |  0                |  6                |  0
+        #   check_finite_and_unscale                |  0                |  0                |  1                |  0
+        #   conv2d                                  |  1                |  0                |  0                |  0
+        #   conv2d_grad                             |  1                |  0                |  0                |  0
+        #   elementwise_add                         |  2                |  0                |  0                |  0
+        #   elementwise_add_grad                    |  2                |  0                |  0                |  0
+        #   elementwise_mul                         |  0                |  0                |  1                |  0
+        #   elementwise_mul_grad                    |  0                |  0                |  1                |  0
+        #   fill_constant                           |  0                |  0                |  1                |  0
+        #   matmul_v2                               |  1                |  0                |  0                |  0
+        #   matmul_v2_grad                          |  1                |  0                |  0                |  0
+        #   memcpy                                  |  0                |  0                |  0                |  1
+        #   reduce_mean                             |  0                |  0                |  1                |  0
+        #   reduce_mean_grad                        |  0                |  0                |  1                |  0
+        #   relu                                    |  1                |  0                |  0                |  0
+        #   relu_grad                               |  1                |  0                |  0                |  0
+        #   reshape2                                |  0                |  0                |  1                |  0
+        #   reshape2_grad                           |  0                |  0                |  1                |  0
+        #   softmax                                 |  0                |  0                |  1                |  0
+        #   softmax_grad                            |  0                |  0                |  1                |  0
+        #   update_loss_scaling                     |  0                |  0                |  1                |  0
+        # <----------------------------------------------------- op count: 22 ----------------------------------------------------->
+    """
+
     def _convert_to_list(op_stats_unit_dict):
         for key, value in op_stats_unit_dict.items():
             op_stats_unit_dict[key] = value.convert_to_list()
diff --git a/python/paddle/static/amp/decorator.py b/python/paddle/static/amp/decorator.py
index 1ef13e5bcd0c5..fc0aaac92bf46 100644
--- a/python/paddle/static/amp/decorator.py
+++ b/python/paddle/static/amp/decorator.py
@@ -34,6 +34,21 @@
 from .function_overload import FunctionType, overload
 
 
+def _set_multi_precision(optimizer, multi_precision):
+    if not isinstance(
+        optimizer,
+        (paddle.optimizer.Optimizer, paddle.fluid.optimizer.Optimizer),
+    ):
+        raise RuntimeError(
+            "Current AMP training level is O2, optimizer is expected to be paddle.optimizer.Optimizer or paddle.fluid.optimizer.Optimizer, but receive {}.".format(
+                type(optimizer)
+            )
+        )
+
+    if multi_precision and hasattr(optimizer, "_multi_precision"):
+        optimizer._multi_precision = multi_precision
+
+
 class OptimizerWithMixedPrecision:
     """
     Optimizer with mixed-precision (MP) training. This is a wrapper of a common
@@ -767,22 +782,96 @@ def decorate(
     amp_lists=None,
     level='O1',
     dtype='float16',
+    master_weight=None,
     init_loss_scaling=2**15,
     incr_every_n_steps=1000,
     decr_every_n_nan_or_inf=2,
     incr_ratio=2.0,
     decr_ratio=0.8,
-    use_dynamic_loss_scaling=True,
+    use_dynamic_loss_scaling=None,
     use_amp_guard=False,
     use_promote=False,
 ):
     """
     Decorate the given optimizer to adapt to the mixed-precision training.
-    """
-    amp_dtype = check_amp_dtype(dtype)
-    if amp_lists is None:
-        amp_lists = AutoMixedPrecisionLists(dtype=amp_dtype)
 
+    Args:
+        optimizer(Optimizer): A common Optimizer.
+        amp_lists(CustomOpLists, optional): An CustomOpLists object. The default
+            white_list and black_list will be used for AMP training when it is
+            not set. Default is None.
+        level(str, optional): Auto mixed precision level. Accepted values are
+            "O1" and "O2": O1 represent mixed precision, the input data type of
+            each operator will be casted by white_list and black_list;
+            O2 represent pure FP16 / BF16 training, all operators parameters
+            and input data will be casted to FP16 / BF16, except operators in
+            black_list, don't support FP16 / BF16 kernel and batch_norm. Default is O1.
+        dtype(str, optional): Whether to use 'float16' or 'bfloat16'. Default is 'float16'.
+        master_weight(bool, optinal): For level='O2', whether to use multi-precision
+            during weight updating. If master_weight is None, in O2 level optimizer
+            will use multi-precision. Default is None.
+        init_loss_scaling(float, optional): The initial loss scaling factor.
+            Default is 32768.
+        incr_every_n_steps(int, optional): Increases loss scaling every n
+            consecutive steps with finite gradients. Default is 1000.
+        decr_every_n_nan_or_inf(int, optional): Decreases loss scaling every n
+            accumulated steps with nan or inf gradients. Default is 2.
+        incr_ratio(float, optional): The multiplier to use when increasing the
+            loss scaling. Default is 2.
+        decr_ratio(float, optional): The less-than-one-multiplier to use when
+            decreasing the loss scaling. Default is 0.8.
+        use_dynamic_loss_scaling(bool, None): Whether to use dynamic loss
+            scaling. Default is None, which means True for float16, and False
+            for bfloat16.
+
+    Returns:
+        An optimizer acting like a normal one but with mixed-precision training
+
+    Examples:
+
+     .. code-block:: python
+
+        import paddle
+
+        paddle.enable_static()
+
+        class SimpleConvNet(paddle.nn.Layer):
+            def __init__(self):
+                super().__init__()
+                self.conv = paddle.nn.Conv2D(in_channels=1, out_channels=6, kernel_size=3)
+                self.linear = paddle.nn.Linear(in_features=26, out_features=10)
+
+            def forward(self, x):
+                out = self.conv(x)
+                out = paddle.nn.functional.relu(out)
+                out = self.linear(out)
+                out = paddle.nn.functional.softmax(out)
+                return out
+
+        main_program = paddle.static.Program()
+        startup_program = paddle.static.Program()
+        with paddle.utils.unique_name.guard():
+            with paddle.static.program_guard(main_program, startup_program):
+                model = SimpleConvNet()
+                x = paddle.static.data(
+                    name='input', shape=[None, 1, 28, 28], dtype='float32'
+                )
+                out = model(x)
+                loss = paddle.mean(out)
+                optimizer = paddle.optimizer.AdamW()
+                optimizer = paddle.static.amp.decorate(optimizer, level="O2", dtype="float16")
+                optimizer.minimize(loss)
+
+        if paddle.is_compiled_with_cuda() and len(paddle.static.cuda_places()) > 0:
+            place = paddle.CUDAPlace(0)
+            exe = paddle.static.Executor(place)
+            exe.run(startup_program)
+
+            # Call `amp_init` after FP32 parameters initialization, such as `exe.run(startup_program)`,
+            # to convert FP32 parameters to low precision FP16 / BF16.
+            optimizer.amp_init(place, scope=paddle.static.global_scope())
+
+    """
     # check amp_level: O0-O2
     level = level.upper()
     if not (level in ['O0', 'O1', 'O2']):
@@ -790,6 +879,18 @@ def decorate(
             "level should be O0, O1 or O2. O0 represents fp32 train mode, O1 represents AMP train mode, O2 represents pure fp16/bf16 train mode."
         )
 
+    amp_dtype = check_amp_dtype(dtype)
+    if amp_lists is None:
+        amp_lists = AutoMixedPrecisionLists(dtype=amp_dtype)
+
+    if use_dynamic_loss_scaling is None:
+        use_dynamic_loss_scaling = dtype == "float16"
+
+    if optimizer is not None:
+        # support master_weight
+        multi_precision = not (master_weight is False)
+        _set_multi_precision(optimizer, multi_precision)
+
     mp_optimizer = OptimizerWithMixedPrecision(
         optimizer,
         amp_lists,
diff --git a/test/amp/amp_base_models.py b/test/amp/amp_base_models.py
index 23c4b018b67d5..8b63b2391c020 100644
--- a/test/amp/amp_base_models.py
+++ b/test/amp/amp_base_models.py
@@ -42,7 +42,6 @@ def _build_optimizer(
         beta2=0.836,
         epsilon=1e-4,
         weight_decay=0.01,
-        multi_precision=True,
     )
     if use_amp:
         optimizer = paddle.static.amp.decorate(
diff --git a/test/amp/test_model_cast_to_bf16.py b/test/amp/test_model_cast_to_bf16.py
index 1a58a2905ec66..3002b623b18af 100644
--- a/test/amp/test_model_cast_to_bf16.py
+++ b/test/amp/test_model_cast_to_bf16.py
@@ -221,11 +221,29 @@ def test_graph_cast(self):
 
 
 class TestProgramBF16(AmpTestBase):
+    def _check_optimizer(self, program, expected_num_mp):
+        optimizers = []
+        for block in program.blocks:
+            for op in block.ops:
+                if "Param" in op.input_names and "Grad" in op.input_names:
+                    optimizers.append(op)
+
+        actual_num_mp = 0
+        for op in optimizers:
+            if op.has_attr("multi_precision") and op.attr("multi_precision"):
+                actual_num_mp += 1
+        self.assertEqual(
+            actual_num_mp,
+            expected_num_mp,
+            f"The number of optimizers with multi_precison = True is expected to be {expected_num_mp}, but recieved {actual_num_mp}.",
+        )
+
     def test_amp_bf16_o1(self):
         main_program, startup_program = build_embedding_model(
             True, "bfloat16", "O1"
         )
         self.assertEqual(main_program.num_blocks, 1)
+        self._check_optimizer(main_program, 0)
 
         amp.debugging.collect_operator_stats(main_program)
         op_stats_list = amp.debugging._get_op_stats_list(main_program)
@@ -255,6 +273,11 @@ def test_amp_bf16_o2(self):
             "squared_l2_norm": 2,
             "adamw": 2,
         }
+        self._check_optimizer(
+            main_program,
+            expected_bf16_calls["matmul_v2"]
+            + expected_bf16_calls["elementwise_add"],
+        )
         self._check_op_calls(op_stats_list[0], expected_bf16_calls)
 
 

From 83c2e68207c7689684fd46453d0f3e8dacd4e7cc Mon Sep 17 00:00:00 2001
From: Sonder <55493212+AndSonder@users.noreply.github.com>
Date: Mon, 24 Apr 2023 15:49:46 +0800
Subject: [PATCH 013/405] Move fused feedforward xpu (#53196)

* add sig file

* trans fused feedforward compute function to phi

* remove fluid include

* delete old register info

* fix build error

* trans fused feedforward grad xpu to phi
---
 .../fused/fused_feedforward_op_xpu.cc         | 832 ------------------
 .../kernels/fused_feedforward_grad_kernel.h   |  69 ++
 paddle/phi/kernels/fused_feedforward_kernel.h |  61 ++
 .../xpu/fused_feedforward_grad_xpu_kernel.cc  | 542 ++++++++++++
 .../xpu/fused_feedforward_xpu_kernel.cc       | 390 ++++++++
 .../phi/ops/compat/fused_feedforward_sig.cc   | 102 +++
 6 files changed, 1164 insertions(+), 832 deletions(-)
 delete mode 100644 paddle/fluid/operators/fused/fused_feedforward_op_xpu.cc
 create mode 100644 paddle/phi/kernels/fused_feedforward_grad_kernel.h
 create mode 100644 paddle/phi/kernels/fused_feedforward_kernel.h
 create mode 100644 paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_xpu_kernel.cc
 create mode 100644 paddle/phi/kernels/fusion/xpu/fused_feedforward_xpu_kernel.cc
 create mode 100644 paddle/phi/ops/compat/fused_feedforward_sig.cc

diff --git a/paddle/fluid/operators/fused/fused_feedforward_op_xpu.cc b/paddle/fluid/operators/fused/fused_feedforward_op_xpu.cc
deleted file mode 100644
index 4b9ba95143345..0000000000000
--- a/paddle/fluid/operators/fused/fused_feedforward_op_xpu.cc
+++ /dev/null
@@ -1,832 +0,0 @@
-/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#ifdef PADDLE_WITH_XPU
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/framework/op_version_registry.h"
-#include "paddle/fluid/operators/matmul_v2_op.h"
-#include "paddle/fluid/operators/xpu_api_wrapper.h"
-#include "paddle/fluid/platform/device/device_wrapper.h"
-#include "paddle/phi/core/ddim.h"
-#include "paddle/phi/kernels/funcs/blas/blas.h"
-
-#include "paddle/fluid/operators/fused/xpu_fused_common_function.h"
-
-namespace paddle {
-namespace operators {
-
-template <typename DeviceContext, typename T>
-class FusedFeedForwardXPUKernel : public framework::OpKernel<T> {
-  using XPUTypeT = typename XPUTypeTrait<T>::Type;
-
- public:
-  void FFN(const phi::XPUContext& dev_ctx,
-           const phi::DenseTensor* x,
-           const phi::DenseTensor* linear1_weight,
-           const phi::DenseTensor* linear1_bias,
-           const phi::DenseTensor* linear2_weight,
-           const phi::DenseTensor* linear2_bias,
-           const phi::DenseTensor* ln_scale,
-           const phi::DenseTensor* ln_bias,
-           phi::DenseTensor* out,
-           phi::DenseTensor* dropout1_mask,
-           phi::DenseTensor* dropout2_mask,
-           phi::DenseTensor* ln_mean,
-           phi::DenseTensor* ln_variance,
-           phi::DenseTensor* linear1_out,
-           phi::DenseTensor* ln1_out,
-           phi::DenseTensor* dropout1_out,
-           phi::DenseTensor* dropout2_out,
-           const int bsz_seq,
-           const int d_model,
-           const int dim_feedforward,
-           const std::string& act_method,
-           const bool pre_layer_norm,
-           const float epsilon1,
-           const float epsilon2,
-           const XPUDropoutParam& dropout_param1,
-           const XPUDropoutParam& dropout_param2,
-           int ring_id) const {
-    xpu::Context* xpu_ctx = dev_ctx.x_context();
-    xpu::ctx_guard RAII_GUARD(xpu_ctx);
-
-    int r = xpu::SUCCESS;
-
-    const XPUTypeT* x_ptr = reinterpret_cast<const XPUTypeT*>(x->data<T>());
-    const XPUTypeT* residual_ptr = x_ptr;
-    const XPUTypeT* linear1_weight_ptr =
-        reinterpret_cast<const XPUTypeT*>(linear1_weight->data<T>());
-    const XPUTypeT* linear1_bias_ptr =
-        reinterpret_cast<const XPUTypeT*>(linear1_bias->data<T>());
-    const XPUTypeT* linear2_weight_ptr =
-        reinterpret_cast<const XPUTypeT*>(linear2_weight->data<T>());
-    const XPUTypeT* linear2_bias_ptr =
-        reinterpret_cast<const XPUTypeT*>(linear2_bias->data<T>());
-
-    const float* ln_scale_ptr = ln_scale->data<float>();
-
-    const float* ln_bias_ptr = ln_bias->data<float>();
-
-    // out
-    XPUTypeT* out_ptr = reinterpret_cast<XPUTypeT*>(out->data<T>());
-    XPUTypeT* linear1_out_ptr =
-        reinterpret_cast<XPUTypeT*>(linear1_out->data<T>());
-    XPUTypeT* dropout1_mask_ptr =
-        reinterpret_cast<XPUTypeT*>(dropout1_mask->data<T>());
-    XPUTypeT* dropout2_mask_ptr =
-        reinterpret_cast<XPUTypeT*>(dropout2_mask->data<T>());
-    float* ln_mean_ptr = ln_mean->data<float>();
-    float* ln_variance_ptr = ln_variance->data<float>();
-
-    XPUTypeT* dropout1_out_ptr =
-        reinterpret_cast<XPUTypeT*>(dropout1_out->data<T>());
-    XPUTypeT* dropout2_out_ptr =
-        reinterpret_cast<XPUTypeT*>(dropout2_out->data<T>());
-
-    size_t l3_total_size = xpu_ctx->_l3_mgr.get_size();
-    XPUTypeT* linear2_before_tmp_ptr = NULL;  // dim_feedforward * bsz_seq
-    XPUTypeT* linear2_after_tmp_ptr = NULL;   // d_model * bsz_seq
-    if (l3_total_size >= dim_feedforward * bsz_seq * sizeof(T)) {
-      XPUTypeT* l3_ptr =
-          RAII_GUARD.alloc_l3<XPUTypeT>(dim_feedforward * bsz_seq);
-      PADDLE_ENFORCE_XDNN_NOT_NULL(l3_ptr);
-      linear2_before_tmp_ptr = linear2_after_tmp_ptr = l3_ptr;
-    } else if ((l3_total_size < dim_feedforward * bsz_seq * sizeof(T)) &&
-               (l3_total_size >= d_model * bsz_seq * sizeof(T))) {
-      XPUTypeT* l3_ptr = RAII_GUARD.alloc_l3<XPUTypeT>(d_model * bsz_seq);
-      PADDLE_ENFORCE_XDNN_NOT_NULL(l3_ptr);
-      linear2_after_tmp_ptr = l3_ptr;
-      linear2_before_tmp_ptr =
-          RAII_GUARD.alloc<XPUTypeT>(dim_feedforward * bsz_seq);
-      PADDLE_ENFORCE_XDNN_NOT_NULL(linear2_before_tmp_ptr);
-
-    } else {
-      XPUTypeT* gm_ptr = RAII_GUARD.alloc<XPUTypeT>(dim_feedforward * bsz_seq);
-      PADDLE_ENFORCE_XDNN_NOT_NULL(gm_ptr);
-      linear2_before_tmp_ptr = linear2_after_tmp_ptr = gm_ptr;
-    }
-
-    // layernorm
-    if (pre_layer_norm) {
-      XPUTypeT* ln1_out_ptr = reinterpret_cast<XPUTypeT*>(ln1_out->data<T>());
-      r = xpu::layer_norm(xpu_ctx,
-                          x_ptr,
-                          ln1_out_ptr,
-                          bsz_seq,
-                          d_model,
-                          epsilon1,
-                          ln_scale_ptr,
-                          ln_bias_ptr,
-                          ln_mean_ptr,
-                          ln_variance_ptr);
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "layer_norm ");
-      x_ptr = ln1_out_ptr;
-    }
-
-    // fc
-    phi::XpuFcInfo linear1_fc_info;
-    linear1_fc_info.InitFcInfo(0,
-                               bsz_seq,
-                               dim_feedforward,
-                               d_model,
-                               false,
-                               false,
-                               nullptr,
-                               nullptr,
-                               nullptr);
-    phi::MatMulXPUFunction<XPUTypeT>(xpu_ctx,
-                                     x_ptr,
-                                     linear1_weight_ptr,
-                                     linear2_before_tmp_ptr,
-                                     linear1_fc_info,
-                                     1.0f);
-
-    // bias
-    r = xpu::broadcast_add(xpu_ctx,
-                           linear2_before_tmp_ptr,
-                           linear1_bias_ptr,
-                           linear1_out_ptr,
-                           {bsz_seq, dim_feedforward},
-                           {dim_feedforward});
-    PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast_add");
-
-    // act
-    if (act_method == "gelu") {
-      r = xpu::gelu(xpu_ctx,
-                    linear1_out_ptr,
-                    linear2_before_tmp_ptr,
-                    linear1_out->numel());
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "gelu");
-    } else if (act_method == "relu") {
-      r = xpu::relu(xpu_ctx,
-                    linear1_out_ptr,
-                    linear2_before_tmp_ptr,
-                    linear1_out->numel());
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "relu");
-    } else {
-      PADDLE_THROW(platform::errors::Unimplemented(
-          "Currently only supports gelu or relu activation functions!"));
-    }
-
-    // dropout1
-    Dropout<XPUTypeT>(xpu_ctx,
-                      linear2_before_tmp_ptr,
-                      dropout1_mask_ptr,
-                      dropout1_out_ptr,
-                      dropout_param1,
-                      dropout1_out->numel());
-
-    // fc
-    phi::XpuFcInfo linear2_fc_info;
-    linear2_fc_info.InitFcInfo(0,
-                               bsz_seq,
-                               d_model,
-                               dim_feedforward,
-                               false,
-                               false,
-                               nullptr,
-                               nullptr,
-                               nullptr);
-    phi::MatMulXPUFunction<XPUTypeT>(xpu_ctx,
-                                     dropout1_out_ptr,
-                                     linear2_weight_ptr,
-                                     dropout2_out_ptr,
-                                     linear2_fc_info,
-                                     1.0f);
-
-    // bias
-    r = xpu::broadcast_add(xpu_ctx,
-                           dropout2_out_ptr,
-                           linear2_bias_ptr,
-                           dropout2_out_ptr,
-                           {bsz_seq, d_model},
-                           {d_model});
-    PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast_add");
-
-    // dropout2
-    Dropout<XPUTypeT>(xpu_ctx,
-                      dropout2_out_ptr,
-                      dropout2_mask_ptr,
-                      dropout2_out_ptr,
-                      dropout_param2,
-                      dropout2_out->numel());
-
-    // residual_ptr + dropout_out
-    XPUTypeT* residual_add_out_ptr = out_ptr;
-    if (pre_layer_norm == false) {
-      residual_add_out_ptr = dropout2_out_ptr;
-    }
-    r = xpu::broadcast_add(xpu_ctx,
-                           residual_ptr,
-                           dropout2_out_ptr,
-                           residual_add_out_ptr,
-                           {bsz_seq, d_model},
-                           {bsz_seq, d_model});
-    PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast_add");
-
-    if (pre_layer_norm == false) {
-      r = xpu::layer_norm(xpu_ctx,
-                          residual_add_out_ptr,
-                          out_ptr,
-                          bsz_seq,
-                          d_model,
-                          epsilon2,
-                          ln_scale_ptr,
-                          ln_bias_ptr,
-                          ln_mean_ptr,
-                          ln_variance_ptr);
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "layer_norm");
-    }
-  }
-
-  void Compute(const framework::ExecutionContext& context) const override {
-    auto place = context.GetPlace();
-
-    auto* x = context.Input<phi::DenseTensor>("X");
-
-    auto* linear1_weight = context.Input<phi::DenseTensor>("Linear1Weight");
-    auto* linear1_bias = context.Input<phi::DenseTensor>("Linear1Bias");
-    auto* linear2_weight = context.Input<phi::DenseTensor>("Linear2Weight");
-    auto* linear2_bias = context.Input<phi::DenseTensor>("Linear2Bias");
-    const bool pre_layer_norm = context.Attr<bool>("pre_layer_norm");
-
-    const phi::DenseTensor* ln_scale = nullptr;
-    const phi::DenseTensor* ln_bias = nullptr;
-    phi::DenseTensor* ln_mean = nullptr;
-    phi::DenseTensor* ln_variance = nullptr;
-    phi::DenseTensor* ln1_out = nullptr;
-
-    if (pre_layer_norm) {
-      ln_scale = context.Input<phi::DenseTensor>("Ln1Scale");
-      ln_bias = context.Input<phi::DenseTensor>("Ln1Bias");
-      ln_mean = context.Output<phi::DenseTensor>("Ln1Mean");
-      ln_variance = context.Output<phi::DenseTensor>("Ln1Variance");
-      ln1_out = context.Output<phi::DenseTensor>("Ln1Out");
-      ln1_out->mutable_data<T>(place);
-    } else {
-      ln_scale = context.Input<phi::DenseTensor>("Ln2Scale");
-      ln_bias = context.Input<phi::DenseTensor>("Ln2Bias");
-      ln_mean = context.Output<phi::DenseTensor>("Ln2Mean");
-      ln_variance = context.Output<phi::DenseTensor>("Ln2Variance");
-    }
-
-    auto* out = context.Output<phi::DenseTensor>("Out");
-    auto* dropout1_mask = context.Output<phi::DenseTensor>("Dropout1Mask");
-    auto* dropout2_mask = context.Output<phi::DenseTensor>("Dropout2Mask");
-    auto* linear1_out = context.Output<phi::DenseTensor>("Linear1Out");
-
-    auto* dropout1_out = context.Output<phi::DenseTensor>("Dropout1Out");
-    auto* dropout2_out = context.Output<phi::DenseTensor>("Dropout2Out");
-
-    const std::string act_method = context.Attr<std::string>("act_method");
-
-    const int ring_id = context.Attr<int>("ring_id");
-    const float epsilon1 = context.Attr<float>("ln1_epsilon");
-    const float epsilon2 = context.Attr<float>("ln2_epsilon");
-    XPUDropoutParam dropout_param1;
-    dropout_param1.initXPUDropoutParam(context, 1);
-    XPUDropoutParam dropout_param2;
-    dropout_param2.initXPUDropoutParam(context, 2);
-
-    ln_mean->mutable_data<float>(place);
-    ln_variance->mutable_data<float>(place);
-    out->mutable_data<T>(place);
-    dropout1_mask->mutable_data<T>(place);
-    dropout2_mask->mutable_data<T>(place);
-    dropout1_out->mutable_data<T>(place);
-    dropout2_out->mutable_data<T>(place);
-    linear1_out->mutable_data<T>(place);
-
-    auto x_dim = x->dims();
-    auto mat_dim_x = phi::funcs::CreateMatrixDescriptor(
-        RowMatrixFromVector(x_dim), 0, false);
-
-    auto dim = linear1_weight->dims();
-    int d_model = dim[0];
-    int dim_feedforward = dim[dim.size() - 1];
-    int bsz_seq = mat_dim_x.batch_size_ * mat_dim_x.height_;
-
-    auto& dev_ctx = context.template device_context<phi::XPUContext>();
-    FFN(dev_ctx,
-        x,
-        linear1_weight,
-        linear1_bias,
-        linear2_weight,
-        linear2_bias,
-        ln_scale,
-        ln_bias,
-        out,
-        dropout1_mask,
-        dropout2_mask,
-        ln_mean,
-        ln_variance,
-        linear1_out,
-        ln1_out,
-        dropout1_out,
-        dropout2_out,
-        bsz_seq,
-        d_model,
-        dim_feedforward,
-        act_method,
-        pre_layer_norm,
-        epsilon1,
-        epsilon2,
-        dropout_param1,
-        dropout_param2,
-        ring_id);
-  }
-};
-
-template <typename DeviceContext, typename T>
-class FusedFeedForwardGradXPUKernel : public framework::OpKernel<T> {
-  using XPUTypeT = typename XPUTypeTrait<T>::Type;
-
- public:
-  void FFNGrad(const phi::XPUContext& dev_ctx,
-               const phi::DenseTensor* d_out,
-               const phi::DenseTensor* x,
-               const phi::DenseTensor* dropout1_mask,
-               const phi::DenseTensor* dropout2_mask,
-               const phi::DenseTensor* linear1_out,
-               const phi::DenseTensor* ln1_out,
-               const phi::DenseTensor* dropout1_out,
-               const phi::DenseTensor* dropout2_out,
-               const phi::DenseTensor* linear1_weight,
-               const phi::DenseTensor* linear2_weight,
-               const phi::DenseTensor* ln_scale,
-               const phi::DenseTensor* ln_mean,
-               const phi::DenseTensor* ln_variance,
-               phi::DenseTensor* d_x,
-               phi::DenseTensor* d_linear1_weight,
-               phi::DenseTensor* d_linear1_bias,
-               phi::DenseTensor* d_linear2_weight,
-               phi::DenseTensor* d_linear2_bias,
-               phi::DenseTensor* d_ln_scale,
-               phi::DenseTensor* d_ln_bias,
-               const int bsz_seq,
-               const int d_model,
-               const int dim_feedforward,
-               const XPUDropoutParam& dropout_param1,
-               const XPUDropoutParam& dropout_param2,
-               const std::string& act_method,
-               const bool pre_layer_norm,
-               const float epsilon,
-               const int ring_id) const {
-    xpu::Context* xpu_ctx = dev_ctx.x_context();
-    xpu::ctx_guard RAII_GUARD(xpu_ctx);
-    int r = xpu::SUCCESS;
-
-    // inputs ptr
-    const XPUTypeT* d_out_ptr =
-        reinterpret_cast<const XPUTypeT*>(d_out->data<T>());
-    const XPUTypeT* x_ptr = reinterpret_cast<const XPUTypeT*>(x->data<T>());
-    const XPUTypeT* dropout1_mask_ptr =
-        reinterpret_cast<const XPUTypeT*>(dropout1_mask->data<T>());
-    const XPUTypeT* dropout2_mask_ptr =
-        reinterpret_cast<const XPUTypeT*>(dropout2_mask->data<T>());
-    const XPUTypeT* linear1_out_ptr =
-        reinterpret_cast<const XPUTypeT*>(linear1_out->data<T>());
-    const XPUTypeT* dropout1_out_ptr =
-        reinterpret_cast<const XPUTypeT*>(dropout1_out->data<T>());
-    const XPUTypeT* linear1_weight_ptr =
-        reinterpret_cast<const XPUTypeT*>(linear1_weight->data<T>());
-    const XPUTypeT* linear2_weight_ptr =
-        reinterpret_cast<const XPUTypeT*>(linear2_weight->data<T>());
-    const float* ln_scale_ptr = ln_scale->data<float>();
-
-    const float* ln_mean_ptr = ln_mean->data<float>();
-    const float* ln_variance_ptr = ln_variance->data<float>();
-    // outputs ptr
-    XPUTypeT* d_x_ptr = reinterpret_cast<XPUTypeT*>(d_x->data<T>());
-    XPUTypeT* d_linear1_weight_ptr =
-        reinterpret_cast<XPUTypeT*>(d_linear1_weight->data<T>());
-    XPUTypeT* d_linear1_bias_ptr =
-        reinterpret_cast<XPUTypeT*>(d_linear1_bias->data<T>());
-    XPUTypeT* d_linear2_weight_ptr =
-        reinterpret_cast<XPUTypeT*>(d_linear2_weight->data<T>());
-    XPUTypeT* d_linear2_bias_ptr =
-        reinterpret_cast<XPUTypeT*>(d_linear2_bias->data<T>());
-    float* d_ln_scale_ptr = d_ln_scale->data<float>();
-    float* d_ln_bias_ptr = d_ln_bias->data<float>();
-
-    size_t l3_total_size = xpu_ctx->_l3_mgr.get_size();
-
-    XPUTypeT* big_tmp_l3_ptr = NULL;    // dim_feedforward * bsz_seq
-    XPUTypeT* small_tmp_l3_ptr = NULL;  // d_model * bsz_seq
-    XPUTypeT* big_tmp_gm_ptr = NULL;    // dim_feedforward * bsz_seq
-    XPUTypeT* small_tmp_gm_ptr = NULL;  // d_model * bsz_seq
-
-    XPUTypeT* d_layernorm_out_ptr = NULL;  // dx9
-    XPUTypeT* d_dropout2_out_ptr = NULL;   // dx7
-
-    XPUTypeT* d_linear2_out_ptr = NULL;   // dx5
-    XPUTypeT* d_dropout1_out_ptr = NULL;  // dx4
-    XPUTypeT* d_act_out_ptr = NULL;       // dx3
-
-    XPUTypeT* d_linear1_out_ptr = NULL;  // dx1
-
-    const XPUTypeT* d_residual_ptr = d_out_ptr;
-
-    if (l3_total_size >= (dim_feedforward * bsz_seq * sizeof(T) +
-                          d_model * bsz_seq * sizeof(T))) {
-      big_tmp_l3_ptr = RAII_GUARD.alloc_l3<XPUTypeT>(dim_feedforward * bsz_seq);
-      PADDLE_ENFORCE_XDNN_NOT_NULL(big_tmp_l3_ptr);
-      small_tmp_l3_ptr = RAII_GUARD.alloc_l3<XPUTypeT>(d_model * bsz_seq);
-      PADDLE_ENFORCE_XDNN_NOT_NULL(small_tmp_l3_ptr);
-      d_layernorm_out_ptr = small_tmp_l3_ptr;
-      d_dropout2_out_ptr = small_tmp_l3_ptr;
-      d_linear2_out_ptr = big_tmp_l3_ptr;
-      d_dropout1_out_ptr = big_tmp_l3_ptr;
-      d_act_out_ptr = big_tmp_l3_ptr;
-      d_linear1_out_ptr = small_tmp_l3_ptr;
-    } else if (l3_total_size >= dim_feedforward * bsz_seq * sizeof(T)) {
-      big_tmp_l3_ptr = RAII_GUARD.alloc_l3<XPUTypeT>(dim_feedforward * bsz_seq);
-      PADDLE_ENFORCE_XDNN_NOT_NULL(big_tmp_l3_ptr);
-      small_tmp_l3_ptr = big_tmp_l3_ptr;
-      big_tmp_gm_ptr = RAII_GUARD.alloc<XPUTypeT>(dim_feedforward * bsz_seq);
-      PADDLE_ENFORCE_XDNN_NOT_NULL(big_tmp_gm_ptr);
-      small_tmp_gm_ptr = RAII_GUARD.alloc<XPUTypeT>(d_model * bsz_seq);
-      PADDLE_ENFORCE_XDNN_NOT_NULL(small_tmp_gm_ptr);
-
-      d_layernorm_out_ptr = small_tmp_l3_ptr;
-      d_dropout2_out_ptr = small_tmp_gm_ptr;
-      d_linear2_out_ptr = big_tmp_l3_ptr;
-      d_dropout1_out_ptr = big_tmp_l3_ptr;
-      d_act_out_ptr = big_tmp_gm_ptr;
-      d_linear1_out_ptr = small_tmp_l3_ptr;
-
-    } else if (l3_total_size >= d_model * bsz_seq * sizeof(T)) {
-      big_tmp_gm_ptr = RAII_GUARD.alloc<XPUTypeT>(dim_feedforward * bsz_seq);
-      PADDLE_ENFORCE_XDNN_NOT_NULL(big_tmp_gm_ptr);
-      small_tmp_l3_ptr = RAII_GUARD.alloc_l3<XPUTypeT>(d_model * bsz_seq);
-      PADDLE_ENFORCE_XDNN_NOT_NULL(small_tmp_l3_ptr);
-
-      d_layernorm_out_ptr = small_tmp_l3_ptr;
-      d_dropout2_out_ptr = small_tmp_l3_ptr;
-      d_linear2_out_ptr = big_tmp_gm_ptr;
-      d_dropout1_out_ptr = big_tmp_gm_ptr;
-      d_act_out_ptr = big_tmp_gm_ptr;
-      d_linear1_out_ptr = small_tmp_l3_ptr;
-    } else {
-      big_tmp_gm_ptr = RAII_GUARD.alloc<XPUTypeT>(dim_feedforward * bsz_seq);
-      PADDLE_ENFORCE_XDNN_NOT_NULL(big_tmp_gm_ptr);
-      small_tmp_gm_ptr = RAII_GUARD.alloc<XPUTypeT>(d_model * bsz_seq);
-      PADDLE_ENFORCE_XDNN_NOT_NULL(small_tmp_gm_ptr);
-      d_layernorm_out_ptr = small_tmp_gm_ptr;
-      d_dropout2_out_ptr = small_tmp_gm_ptr;
-      d_linear2_out_ptr = big_tmp_gm_ptr;
-      d_dropout1_out_ptr = big_tmp_gm_ptr;
-      d_act_out_ptr = big_tmp_gm_ptr;
-      d_linear1_out_ptr = small_tmp_gm_ptr;
-    }
-
-    if (pre_layer_norm == false) {
-      const XPUTypeT* dropout2_out_ptr =
-          reinterpret_cast<const XPUTypeT*>(dropout2_out->data<T>());
-      r = xpu::layer_norm_grad(xpu_ctx,
-                               dropout2_out_ptr,
-                               d_out_ptr,
-                               d_layernorm_out_ptr,
-                               bsz_seq,
-                               d_model,
-                               epsilon,
-                               ln_scale_ptr,
-                               ln_mean_ptr,
-                               ln_variance_ptr,
-                               d_ln_scale_ptr,
-                               d_ln_bias_ptr);
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "layer_norm_grad");
-      d_residual_ptr = d_layernorm_out_ptr;
-    }
-    DropoutGrad(xpu_ctx,
-                d_residual_ptr,
-                dropout2_mask_ptr,
-                d_dropout2_out_ptr,
-                dropout_param2,
-                bsz_seq * d_model);
-    // linear_grad2
-    r = xpu::reduce_sum(xpu_ctx,
-                        d_dropout2_out_ptr,
-                        d_linear2_bias_ptr,
-                        {bsz_seq, d_model},
-                        {0});
-    PADDLE_ENFORCE_XDNN_SUCCESS(r, "reduce_sum");
-
-    phi::XpuFcInfo linear2_fc_info;
-    linear2_fc_info.InitFcInfo(0,
-                               bsz_seq,
-                               d_model,
-                               dim_feedforward,
-                               false,
-                               false,
-                               nullptr,
-                               nullptr,
-                               nullptr);
-
-    const XPUTypeT* a_1 = reinterpret_cast<const XPUTypeT*>(NULL);
-    const XPUTypeT* b_1 = reinterpret_cast<const XPUTypeT*>(NULL);
-    const XPUTypeT* a_2 = reinterpret_cast<const XPUTypeT*>(NULL);
-    const XPUTypeT* b_2 = reinterpret_cast<const XPUTypeT*>(NULL);
-    XPUTypeT* c_1 = d_linear2_out_ptr;
-    XPUTypeT* c_2 = d_linear2_weight_ptr;
-    phi::XpuFcInfo info_d_dropout1;
-    phi::XpuFcInfo info_dw2;
-
-    std::tuple<phi::XpuFcInfo,
-               phi::XpuFcInfo,
-               const XPUTypeT*,
-               const XPUTypeT*,
-               const XPUTypeT*,
-               const XPUTypeT*>
-        fc_info = phi::MatmulGradFcInfo(xpu_ctx,
-                                        &RAII_GUARD,
-                                        linear2_fc_info,
-                                        false,
-                                        false,
-                                        dropout1_out_ptr,
-                                        linear2_weight_ptr,
-                                        d_dropout2_out_ptr);
-
-    std::tie(info_d_dropout1, info_dw2, a_1, b_1, a_2, b_2) = fc_info;
-
-    // if l3_total_size >= dim_feedforward * bsz_seq * sizeof(T), first transpos
-    if (l3_total_size >= dim_feedforward * bsz_seq * sizeof(T) &&
-        info_dw2.trans_x) {
-      r = xpu::transpose<XPUTypeT>(xpu_ctx,
-                                   dropout1_out_ptr,
-                                   big_tmp_l3_ptr,
-                                   {bsz_seq, dim_feedforward},
-                                   {1, 0});
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "transpose");
-      a_2 = big_tmp_l3_ptr;
-      info_dw2.trans_x = !info_dw2.trans_x;
-      info_dw2.stride_x = info_dw2.k;
-    }
-
-    phi::MatMulXPUFunction<XPUTypeT>(
-        xpu_ctx, a_1, b_1, c_1, info_d_dropout1, 1.0f, true);
-
-    phi::MatMulXPUFunction<XPUTypeT>(
-        xpu_ctx, a_2, b_2, c_2, info_dw2, 1.0f, true);
-
-    // dropout_grad1
-    DropoutGrad(xpu_ctx,
-                d_linear2_out_ptr,
-                dropout1_mask_ptr,
-                d_dropout1_out_ptr,
-                dropout_param1,
-                bsz_seq * dim_feedforward);
-
-    // act_grad
-    if (act_method == "gelu") {
-      r = xpu::gelu_grad(xpu_ctx,
-                         linear1_out_ptr,
-                         linear1_out_ptr,
-                         d_dropout1_out_ptr,
-                         d_act_out_ptr,
-                         bsz_seq * dim_feedforward);
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "gelu_grad");
-    } else if (act_method == "relu") {
-      r = xpu::relu_grad(xpu_ctx,
-                         linear1_out_ptr,
-                         linear1_out_ptr,
-                         d_dropout1_out_ptr,
-                         d_act_out_ptr,
-                         bsz_seq * dim_feedforward);
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "relu_grad");
-    } else {
-      PADDLE_THROW(platform::errors::Unimplemented(
-          "Currently only supports gelu or relu activation functions!"));
-    }
-
-    // linear1_grad
-    r = xpu::reduce_sum(xpu_ctx,
-                        d_act_out_ptr,
-                        d_linear1_bias_ptr,
-                        {bsz_seq, dim_feedforward},
-                        {0});
-    PADDLE_ENFORCE_XDNN_SUCCESS(r, "reduce_sum");
-
-    phi::XpuFcInfo linear1_fc_info;
-    linear1_fc_info.InitFcInfo(0,
-                               bsz_seq,
-                               dim_feedforward,
-                               d_model,
-                               false,
-                               false,
-                               nullptr,
-                               nullptr,
-                               nullptr);
-
-    a_1 = reinterpret_cast<const XPUTypeT*>(NULL);
-    b_1 = reinterpret_cast<const XPUTypeT*>(NULL);
-    a_2 = reinterpret_cast<const XPUTypeT*>(NULL);
-    b_2 = reinterpret_cast<const XPUTypeT*>(NULL);
-
-    c_1 = (pre_layer_norm == true ? d_linear1_out_ptr : d_x_ptr);
-    c_2 = d_linear1_weight_ptr;
-    phi::XpuFcInfo info_dx;
-    phi::XpuFcInfo info_dw1;
-
-    const XPUTypeT* linear1_x_ptr =
-        (pre_layer_norm == true
-             ? reinterpret_cast<const XPUTypeT*>(ln1_out->data<T>())
-             : x_ptr);
-
-    if (l3_total_size >= d_model * bsz_seq * sizeof(T) && info_dw1.trans_x) {
-      r = xpu::transpose<XPUTypeT>(
-          xpu_ctx, linear1_x_ptr, small_tmp_l3_ptr, {bsz_seq, d_model}, {1, 0});
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "transpose");
-      a_2 = small_tmp_l3_ptr;
-      info_dw1.trans_x = !info_dw1.trans_x;
-      info_dw1.stride_x = info_dw1.k;
-    }
-
-    fc_info = phi::MatmulGradFcInfo(xpu_ctx,
-                                    &RAII_GUARD,
-                                    linear1_fc_info,
-                                    false,
-                                    false,
-                                    linear1_x_ptr,
-                                    linear1_weight_ptr,
-                                    d_act_out_ptr);
-
-    std::tie(info_dx, info_dw1, a_1, b_1, a_2, b_2) = fc_info;
-
-    phi::MatMulXPUFunction<XPUTypeT>(
-        xpu_ctx, a_1, b_1, c_1, info_dx, 1.0f, true);
-
-    phi::MatMulXPUFunction<XPUTypeT>(
-        xpu_ctx, a_2, b_2, c_2, info_dw1, 1.0f, true);
-
-    if (pre_layer_norm) {
-      r = xpu::layer_norm_grad(xpu_ctx,
-                               x_ptr,
-                               c_1,
-                               c_1,
-                               bsz_seq,
-                               d_model,
-                               epsilon,
-                               ln_scale_ptr,
-                               ln_mean_ptr,
-                               ln_variance_ptr,
-                               d_ln_scale_ptr,
-                               d_ln_bias_ptr);
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "layer_norm_grad");
-    }
-
-    r = xpu::add(xpu_ctx, c_1, d_residual_ptr, d_x_ptr, d_model * bsz_seq);
-    PADDLE_ENFORCE_XDNN_SUCCESS(r, "add");
-  }
-
-  void Compute(const framework::ExecutionContext& context) const override {
-    auto place = context.GetPlace();
-    const bool pre_layer_norm = context.Attr<bool>("pre_layer_norm");
-    // inputs
-    auto* d_out =
-        context.Input<phi::DenseTensor>(framework::GradVarName("Out"));
-    auto* x = context.Input<phi::DenseTensor>("X");
-
-    auto* dropout1_mask = context.Input<phi::DenseTensor>("Dropout1Mask");
-    auto* dropout2_mask = context.Input<phi::DenseTensor>("Dropout2Mask");
-    auto* linear1_out = context.Input<phi::DenseTensor>("Linear1Out");
-    auto* ln1_out =
-        pre_layer_norm ? context.Input<phi::DenseTensor>("Ln1Out") : nullptr;
-
-    auto* dropout1_out = context.Input<phi::DenseTensor>("Dropout1Out");
-    auto* dropout2_out = context.Input<phi::DenseTensor>("Dropout2Out");
-    auto* linear1_weight = context.Input<phi::DenseTensor>("Linear1Weight");
-    auto* linear2_weight = context.Input<phi::DenseTensor>("Linear2Weight");
-
-    const phi::DenseTensor* ln_mean = nullptr;
-    const phi::DenseTensor* ln_variance = nullptr;
-    const phi::DenseTensor* ln_scale = nullptr;
-
-    if (pre_layer_norm) {
-      ln_mean = context.Input<phi::DenseTensor>("Ln1Mean");
-      ln_variance = context.Input<phi::DenseTensor>("Ln1Variance");
-      ln_scale = context.Input<phi::DenseTensor>("Ln1Scale");
-    } else {
-      ln_mean = context.Input<phi::DenseTensor>("Ln2Mean");
-      ln_variance = context.Input<phi::DenseTensor>("Ln2Variance");
-      ln_scale = context.Input<phi::DenseTensor>("Ln2Scale");
-    }
-
-    // output
-    auto* d_x = context.Output<phi::DenseTensor>(framework::GradVarName("X"));
-
-    phi::DenseTensor* d_ln_scale = nullptr;
-    phi::DenseTensor* d_ln_bias = nullptr;
-
-    if (pre_layer_norm) {
-      d_ln_scale =
-          context.Output<phi::DenseTensor>(framework::GradVarName("Ln1Scale"));
-      d_ln_bias =
-          context.Output<phi::DenseTensor>(framework::GradVarName("Ln1Bias"));
-    } else {
-      d_ln_scale =
-          context.Output<phi::DenseTensor>(framework::GradVarName("Ln2Scale"));
-      d_ln_bias =
-          context.Output<phi::DenseTensor>(framework::GradVarName("Ln2Bias"));
-    }
-
-    auto* d_linear1_weight = context.Output<phi::DenseTensor>(
-        framework::GradVarName("Linear1Weight"));
-    auto* d_linear1_bias =
-        context.Output<phi::DenseTensor>(framework::GradVarName("Linear1Bias"));
-    auto* d_linear2_weight = context.Output<phi::DenseTensor>(
-        framework::GradVarName("Linear2Weight"));
-    auto* d_linear2_bias =
-        context.Output<phi::DenseTensor>(framework::GradVarName("Linear2Bias"));
-
-    float epsilon = 0.0f;
-    if (pre_layer_norm) {
-      epsilon = context.Attr<float>("ln1_epsilon");
-    } else {
-      epsilon = context.Attr<float>("ln2_epsilon");
-    }
-
-    const std::string act_method = context.Attr<std::string>("act_method");
-
-    XPUDropoutParam dropout_param1(context, 1);
-    XPUDropoutParam dropout_param2(context, 2);
-
-    const int ring_id = context.Attr<int>("ring_id");
-
-    d_x->mutable_data<T>(place);
-    d_ln_scale->mutable_data<float>(place);
-    d_ln_bias->mutable_data<float>(place);
-    d_linear1_bias->mutable_data<T>(place);
-    d_linear2_bias->mutable_data<T>(place);
-    d_linear1_weight->mutable_data<T>(place);
-    d_linear2_weight->mutable_data<T>(place);
-
-    auto x_dim = x->dims();
-    auto mat_dim_x = phi::funcs::CreateMatrixDescriptor(
-        RowMatrixFromVector(x_dim), 0, false);
-
-    auto linear1_weight_dim = linear1_weight->dims();
-    int d_model = linear1_weight_dim[0];
-    int dim_feedforward = linear1_weight_dim[linear1_weight_dim.size() - 1];
-    int bsz_seq = mat_dim_x.batch_size_ * mat_dim_x.height_;
-    auto& dev_ctx = context.template device_context<phi::XPUContext>();
-
-    FFNGrad(dev_ctx,
-            d_out,
-            x,
-            dropout1_mask,
-            dropout2_mask,
-            linear1_out,
-            ln1_out,
-            dropout1_out,
-            dropout2_out,
-            linear1_weight,
-            linear2_weight,
-            ln_scale,
-            ln_mean,
-            ln_variance,
-            d_x,
-            d_linear1_weight,
-            d_linear1_bias,
-            d_linear2_weight,
-            d_linear2_bias,
-            d_ln_scale,
-            d_ln_bias,
-            bsz_seq,
-            d_model,
-            dim_feedforward,
-            dropout_param1,
-            dropout_param2,
-            act_method,
-            pre_layer_norm,
-            epsilon,
-            ring_id);
-  }
-};
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-REGISTER_OP_XPU_KERNEL(
-    fused_feedforward,
-    ops::FusedFeedForwardXPUKernel<phi::XPUContext, float>,
-    ops::FusedFeedForwardXPUKernel<phi::XPUContext, paddle::platform::float16>);
-
-REGISTER_OP_XPU_KERNEL(
-    fused_feedforward_grad,
-    ops::FusedFeedForwardGradXPUKernel<phi::XPUContext, float>,
-    ops::FusedFeedForwardGradXPUKernel<phi::XPUContext,
-                                       paddle::platform::float16>);
-
-#endif
diff --git a/paddle/phi/kernels/fused_feedforward_grad_kernel.h b/paddle/phi/kernels/fused_feedforward_grad_kernel.h
new file mode 100644
index 0000000000000..9eee46a83987e
--- /dev/null
+++ b/paddle/phi/kernels/fused_feedforward_grad_kernel.h
@@ -0,0 +1,69 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void FusedFeedForwardGradKernel(
+    const Context& dev_ctx,
+    const DenseTensor& out_grad,
+    const DenseTensor& x,
+    const DenseTensor& linear1_weight,
+    const DenseTensor& linear1_bias,
+    const DenseTensor& linear2_weight,
+    const DenseTensor& dropout1_mask,
+    const DenseTensor& dropout2_mask,
+    const DenseTensor& linear1_out,
+    const DenseTensor& dropout1_out,
+    const DenseTensor& dropout2_out,
+    const paddle::optional<DenseTensor>& ln1_scale,
+    const paddle::optional<DenseTensor>& ln1_bias,
+    const paddle::optional<DenseTensor>& ln1_out,
+    const paddle::optional<DenseTensor>& ln1_mean,
+    const paddle::optional<DenseTensor>& ln1_variance,
+    const paddle::optional<DenseTensor>& ln2_scale,
+    const paddle::optional<DenseTensor>& ln2_bias,
+    const paddle::optional<DenseTensor>& ln2_mean,
+    const paddle::optional<DenseTensor>& ln2_variance,
+    const paddle::optional<DenseTensor>& linear2_bias,
+    bool pre_layer_norm,
+    float ln1_epsilon,
+    float ln2_epsilon,
+    const std::string& act_method,
+    float dropout1_prob,
+    float dropout2_prob,
+    const std::string& dropout1_implementation,
+    const std::string& dropout2_implementation,
+    bool is_test,
+    bool dropout1_fix_seed,
+    bool dropout2_fix_seed,
+    int dropout1_seed_val,
+    int dropout2_seed_val,
+    bool add_residual,
+    int ring_id,
+    DenseTensor* x_grad,
+    DenseTensor* ln1_scale_grad,
+    DenseTensor* ln1_bias_grad,
+    DenseTensor* ln2_scale_grad,
+    DenseTensor* ln2_bias_grad,
+    DenseTensor* linear1_weight_grad,
+    DenseTensor* linear1_bias_grad,
+    DenseTensor* linear2_weight_grad,
+    DenseTensor* linear2_bias_grad);
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/fused_feedforward_kernel.h b/paddle/phi/kernels/fused_feedforward_kernel.h
new file mode 100644
index 0000000000000..cade7adc0c7c2
--- /dev/null
+++ b/paddle/phi/kernels/fused_feedforward_kernel.h
@@ -0,0 +1,61 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void FusedFeedForwardKernel(const Context& dev_ctx,
+                            const DenseTensor& x,
+                            const paddle::optional<DenseTensor>& dropout1_seed,
+                            const paddle::optional<DenseTensor>& dropout2_seed,
+                            const DenseTensor& linear1_weight,
+                            const paddle::optional<DenseTensor>& linear1_bias,
+                            const DenseTensor& linear2_weight,
+                            const paddle::optional<DenseTensor>& linear2_bias,
+                            const paddle::optional<DenseTensor>& ln1_scale,
+                            const paddle::optional<DenseTensor>& ln1_bias,
+                            const paddle::optional<DenseTensor>& ln2_scale,
+                            const paddle::optional<DenseTensor>& ln2_bias,
+                            bool pre_layer_norm,
+                            float ln1_epsilon,
+                            float ln2_epsilon,
+                            const std::string& act_method,
+                            float dropout1_prob,
+                            float dropout2_prob,
+                            const std::string& dropout1_implementation,
+                            const std::string& dropout2_implementation,
+                            bool is_test,
+                            bool dropout1_fix_seed,
+                            bool dropout2_fix_seed,
+                            int dropout1_seed_val,
+                            int dropout2_seed_val,
+                            bool add_residual,
+                            int ring_id,
+                            DenseTensor* out,
+                            DenseTensor* dropout1_mask,
+                            DenseTensor* dropout2_mask,
+                            DenseTensor* ln1_mean,
+                            DenseTensor* ln1_variance,
+                            DenseTensor* ln2_mean,
+                            DenseTensor* ln2_variance,
+                            DenseTensor* linear1_out,
+                            DenseTensor* ln1_out,
+                            DenseTensor* dropout1_out,
+                            DenseTensor* dropout2_out);
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_xpu_kernel.cc b/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_xpu_kernel.cc
new file mode 100644
index 0000000000000..cb10930dc9b3e
--- /dev/null
+++ b/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_xpu_kernel.cc
@@ -0,0 +1,542 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/backends/xpu/enforce_xpu.h"
+#include "paddle/phi/backends/xpu/xpu_context.h"
+#include "paddle/phi/core/ddim.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/blas/blas.h"
+#include "paddle/phi/kernels/impl/matmul_grad_kernel_impl.h"
+#include "paddle/phi/kernels/xpu/xpu_api_wrapper.h"
+#include "paddle/phi/kernels/xpu/xpu_fused_common_function.h"
+
+namespace phi {
+namespace fusion {
+
+template <typename T, typename Context>
+void FFNGrad(const phi::XPUContext& dev_ctx,
+             const phi::DenseTensor* d_out,
+             const phi::DenseTensor* x,
+             const phi::DenseTensor* dropout1_mask,
+             const phi::DenseTensor* dropout2_mask,
+             const phi::DenseTensor* linear1_out,
+             const phi::DenseTensor* ln1_out,
+             const phi::DenseTensor* dropout1_out,
+             const phi::DenseTensor* dropout2_out,
+             const phi::DenseTensor* linear1_weight,
+             const phi::DenseTensor* linear2_weight,
+             const phi::DenseTensor* ln_scale,
+             const phi::DenseTensor* ln_mean,
+             const phi::DenseTensor* ln_variance,
+             phi::DenseTensor* d_x,
+             phi::DenseTensor* d_linear1_weight,
+             phi::DenseTensor* d_linear1_bias,
+             phi::DenseTensor* d_linear2_weight,
+             phi::DenseTensor* d_linear2_bias,
+             phi::DenseTensor* d_ln_scale,
+             phi::DenseTensor* d_ln_bias,
+             const int bsz_seq,
+             const int d_model,
+             const int dim_feedforward,
+             const XPUDropoutParam& dropout_param1,
+             const XPUDropoutParam& dropout_param2,
+             const std::string& act_method,
+             const bool pre_layer_norm,
+             const float epsilon,
+             const int ring_id) {
+  using XPUTypeT = typename XPUTypeTrait<T>::Type;
+  xpu::Context* xpu_ctx = dev_ctx.x_context();
+  xpu::ctx_guard RAII_GUARD(xpu_ctx);
+  int r = xpu::SUCCESS;
+
+  // inputs ptr
+  const XPUTypeT* d_out_ptr =
+      reinterpret_cast<const XPUTypeT*>(d_out->data<T>());
+  const XPUTypeT* x_ptr = reinterpret_cast<const XPUTypeT*>(x->data<T>());
+  const XPUTypeT* dropout1_mask_ptr =
+      reinterpret_cast<const XPUTypeT*>(dropout1_mask->data<T>());
+  const XPUTypeT* dropout2_mask_ptr =
+      reinterpret_cast<const XPUTypeT*>(dropout2_mask->data<T>());
+  const XPUTypeT* linear1_out_ptr =
+      reinterpret_cast<const XPUTypeT*>(linear1_out->data<T>());
+  const XPUTypeT* dropout1_out_ptr =
+      reinterpret_cast<const XPUTypeT*>(dropout1_out->data<T>());
+  const XPUTypeT* linear1_weight_ptr =
+      reinterpret_cast<const XPUTypeT*>(linear1_weight->data<T>());
+  const XPUTypeT* linear2_weight_ptr =
+      reinterpret_cast<const XPUTypeT*>(linear2_weight->data<T>());
+  const float* ln_scale_ptr = ln_scale->data<float>();
+
+  const float* ln_mean_ptr = ln_mean->data<float>();
+  const float* ln_variance_ptr = ln_variance->data<float>();
+  // outputs ptr
+  XPUTypeT* d_x_ptr = reinterpret_cast<XPUTypeT*>(d_x->data<T>());
+  XPUTypeT* d_linear1_weight_ptr =
+      reinterpret_cast<XPUTypeT*>(d_linear1_weight->data<T>());
+  XPUTypeT* d_linear1_bias_ptr =
+      reinterpret_cast<XPUTypeT*>(d_linear1_bias->data<T>());
+  XPUTypeT* d_linear2_weight_ptr =
+      reinterpret_cast<XPUTypeT*>(d_linear2_weight->data<T>());
+  XPUTypeT* d_linear2_bias_ptr =
+      reinterpret_cast<XPUTypeT*>(d_linear2_bias->data<T>());
+  float* d_ln_scale_ptr = d_ln_scale->data<float>();
+  float* d_ln_bias_ptr = d_ln_bias->data<float>();
+
+  size_t l3_total_size = xpu_ctx->_l3_mgr.get_size();
+
+  XPUTypeT* big_tmp_l3_ptr = NULL;    // dim_feedforward * bsz_seq
+  XPUTypeT* small_tmp_l3_ptr = NULL;  // d_model * bsz_seq
+  XPUTypeT* big_tmp_gm_ptr = NULL;    // dim_feedforward * bsz_seq
+  XPUTypeT* small_tmp_gm_ptr = NULL;  // d_model * bsz_seq
+
+  XPUTypeT* d_layernorm_out_ptr = NULL;  // dx9
+  XPUTypeT* d_dropout2_out_ptr = NULL;   // dx7
+
+  XPUTypeT* d_linear2_out_ptr = NULL;   // dx5
+  XPUTypeT* d_dropout1_out_ptr = NULL;  // dx4
+  XPUTypeT* d_act_out_ptr = NULL;       // dx3
+
+  XPUTypeT* d_linear1_out_ptr = NULL;  // dx1
+
+  const XPUTypeT* d_residual_ptr = d_out_ptr;
+
+  if (l3_total_size >=
+      (dim_feedforward * bsz_seq * sizeof(T) + d_model * bsz_seq * sizeof(T))) {
+    big_tmp_l3_ptr = RAII_GUARD.alloc_l3<XPUTypeT>(dim_feedforward * bsz_seq);
+    PADDLE_ENFORCE_XDNN_NOT_NULL(big_tmp_l3_ptr);
+    small_tmp_l3_ptr = RAII_GUARD.alloc_l3<XPUTypeT>(d_model * bsz_seq);
+    PADDLE_ENFORCE_XDNN_NOT_NULL(small_tmp_l3_ptr);
+    d_layernorm_out_ptr = small_tmp_l3_ptr;
+    d_dropout2_out_ptr = small_tmp_l3_ptr;
+    d_linear2_out_ptr = big_tmp_l3_ptr;
+    d_dropout1_out_ptr = big_tmp_l3_ptr;
+    d_act_out_ptr = big_tmp_l3_ptr;
+    d_linear1_out_ptr = small_tmp_l3_ptr;
+  } else if (l3_total_size >= dim_feedforward * bsz_seq * sizeof(T)) {
+    big_tmp_l3_ptr = RAII_GUARD.alloc_l3<XPUTypeT>(dim_feedforward * bsz_seq);
+    PADDLE_ENFORCE_XDNN_NOT_NULL(big_tmp_l3_ptr);
+    small_tmp_l3_ptr = big_tmp_l3_ptr;
+    big_tmp_gm_ptr = RAII_GUARD.alloc<XPUTypeT>(dim_feedforward * bsz_seq);
+    PADDLE_ENFORCE_XDNN_NOT_NULL(big_tmp_gm_ptr);
+    small_tmp_gm_ptr = RAII_GUARD.alloc<XPUTypeT>(d_model * bsz_seq);
+    PADDLE_ENFORCE_XDNN_NOT_NULL(small_tmp_gm_ptr);
+
+    d_layernorm_out_ptr = small_tmp_l3_ptr;
+    d_dropout2_out_ptr = small_tmp_gm_ptr;
+    d_linear2_out_ptr = big_tmp_l3_ptr;
+    d_dropout1_out_ptr = big_tmp_l3_ptr;
+    d_act_out_ptr = big_tmp_gm_ptr;
+    d_linear1_out_ptr = small_tmp_l3_ptr;
+
+  } else if (l3_total_size >= d_model * bsz_seq * sizeof(T)) {
+    big_tmp_gm_ptr = RAII_GUARD.alloc<XPUTypeT>(dim_feedforward * bsz_seq);
+    PADDLE_ENFORCE_XDNN_NOT_NULL(big_tmp_gm_ptr);
+    small_tmp_l3_ptr = RAII_GUARD.alloc_l3<XPUTypeT>(d_model * bsz_seq);
+    PADDLE_ENFORCE_XDNN_NOT_NULL(small_tmp_l3_ptr);
+
+    d_layernorm_out_ptr = small_tmp_l3_ptr;
+    d_dropout2_out_ptr = small_tmp_l3_ptr;
+    d_linear2_out_ptr = big_tmp_gm_ptr;
+    d_dropout1_out_ptr = big_tmp_gm_ptr;
+    d_act_out_ptr = big_tmp_gm_ptr;
+    d_linear1_out_ptr = small_tmp_l3_ptr;
+  } else {
+    big_tmp_gm_ptr = RAII_GUARD.alloc<XPUTypeT>(dim_feedforward * bsz_seq);
+    PADDLE_ENFORCE_XDNN_NOT_NULL(big_tmp_gm_ptr);
+    small_tmp_gm_ptr = RAII_GUARD.alloc<XPUTypeT>(d_model * bsz_seq);
+    PADDLE_ENFORCE_XDNN_NOT_NULL(small_tmp_gm_ptr);
+    d_layernorm_out_ptr = small_tmp_gm_ptr;
+    d_dropout2_out_ptr = small_tmp_gm_ptr;
+    d_linear2_out_ptr = big_tmp_gm_ptr;
+    d_dropout1_out_ptr = big_tmp_gm_ptr;
+    d_act_out_ptr = big_tmp_gm_ptr;
+    d_linear1_out_ptr = small_tmp_gm_ptr;
+  }
+
+  if (pre_layer_norm == false) {
+    const XPUTypeT* dropout2_out_ptr =
+        reinterpret_cast<const XPUTypeT*>(dropout2_out->data<T>());
+    r = xpu::layer_norm_grad(xpu_ctx,
+                             dropout2_out_ptr,
+                             d_out_ptr,
+                             d_layernorm_out_ptr,
+                             bsz_seq,
+                             d_model,
+                             epsilon,
+                             ln_scale_ptr,
+                             ln_mean_ptr,
+                             ln_variance_ptr,
+                             d_ln_scale_ptr,
+                             d_ln_bias_ptr);
+    PADDLE_ENFORCE_XDNN_SUCCESS(r, "layer_norm_grad");
+    d_residual_ptr = d_layernorm_out_ptr;
+  }
+  phi::DropoutGrad(xpu_ctx,
+                   d_residual_ptr,
+                   dropout2_mask_ptr,
+                   d_dropout2_out_ptr,
+                   dropout_param2,
+                   bsz_seq * d_model);
+  // linear_grad2
+  r = xpu::reduce_sum(
+      xpu_ctx, d_dropout2_out_ptr, d_linear2_bias_ptr, {bsz_seq, d_model}, {0});
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "reduce_sum");
+
+  phi::XpuFcInfo linear2_fc_info;
+  linear2_fc_info.InitFcInfo(0,
+                             bsz_seq,
+                             d_model,
+                             dim_feedforward,
+                             false,
+                             false,
+                             nullptr,
+                             nullptr,
+                             nullptr);
+
+  const XPUTypeT* a_1 = reinterpret_cast<const XPUTypeT*>(NULL);
+  const XPUTypeT* b_1 = reinterpret_cast<const XPUTypeT*>(NULL);
+  const XPUTypeT* a_2 = reinterpret_cast<const XPUTypeT*>(NULL);
+  const XPUTypeT* b_2 = reinterpret_cast<const XPUTypeT*>(NULL);
+  XPUTypeT* c_1 = d_linear2_out_ptr;
+  XPUTypeT* c_2 = d_linear2_weight_ptr;
+  phi::XpuFcInfo info_d_dropout1;
+  phi::XpuFcInfo info_dw2;
+
+  std::tuple<phi::XpuFcInfo,
+             phi::XpuFcInfo,
+             const XPUTypeT*,
+             const XPUTypeT*,
+             const XPUTypeT*,
+             const XPUTypeT*>
+      fc_info = phi::MatmulGradFcInfo(xpu_ctx,
+                                      &RAII_GUARD,
+                                      linear2_fc_info,
+                                      false,
+                                      false,
+                                      dropout1_out_ptr,
+                                      linear2_weight_ptr,
+                                      d_dropout2_out_ptr);
+
+  std::tie(info_d_dropout1, info_dw2, a_1, b_1, a_2, b_2) = fc_info;
+
+  // if l3_total_size >= dim_feedforward * bsz_seq * sizeof(T), first transpos
+  if (l3_total_size >= dim_feedforward * bsz_seq * sizeof(T) &&
+      info_dw2.trans_x) {
+    r = xpu::transpose<XPUTypeT>(xpu_ctx,
+                                 dropout1_out_ptr,
+                                 big_tmp_l3_ptr,
+                                 {bsz_seq, dim_feedforward},
+                                 {1, 0});
+    PADDLE_ENFORCE_XDNN_SUCCESS(r, "transpose");
+    a_2 = big_tmp_l3_ptr;
+    info_dw2.trans_x = !info_dw2.trans_x;
+    info_dw2.stride_x = info_dw2.k;
+  }
+
+  phi::MatMulXPUFunction<XPUTypeT>(
+      xpu_ctx, a_1, b_1, c_1, info_d_dropout1, 1.0f, true);
+
+  phi::MatMulXPUFunction<XPUTypeT>(
+      xpu_ctx, a_2, b_2, c_2, info_dw2, 1.0f, true);
+
+  // dropout_grad1
+  DropoutGrad(xpu_ctx,
+              d_linear2_out_ptr,
+              dropout1_mask_ptr,
+              d_dropout1_out_ptr,
+              dropout_param1,
+              bsz_seq * dim_feedforward);
+
+  // act_grad
+  if (act_method == "gelu") {
+    r = xpu::gelu_grad(xpu_ctx,
+                       linear1_out_ptr,
+                       linear1_out_ptr,
+                       d_dropout1_out_ptr,
+                       d_act_out_ptr,
+                       bsz_seq * dim_feedforward);
+    PADDLE_ENFORCE_XDNN_SUCCESS(r, "gelu_grad");
+  } else if (act_method == "relu") {
+    r = xpu::relu_grad(xpu_ctx,
+                       linear1_out_ptr,
+                       linear1_out_ptr,
+                       d_dropout1_out_ptr,
+                       d_act_out_ptr,
+                       bsz_seq * dim_feedforward);
+    PADDLE_ENFORCE_XDNN_SUCCESS(r, "relu_grad");
+  } else {
+    PADDLE_THROW(phi::errors::Unimplemented(
+        "Currently only supports gelu or relu activation functions!"));
+  }
+
+  // linear1_grad
+  r = xpu::reduce_sum(xpu_ctx,
+                      d_act_out_ptr,
+                      d_linear1_bias_ptr,
+                      {bsz_seq, dim_feedforward},
+                      {0});
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "reduce_sum");
+
+  phi::XpuFcInfo linear1_fc_info;
+  linear1_fc_info.InitFcInfo(0,
+                             bsz_seq,
+                             dim_feedforward,
+                             d_model,
+                             false,
+                             false,
+                             nullptr,
+                             nullptr,
+                             nullptr);
+
+  a_1 = reinterpret_cast<const XPUTypeT*>(NULL);
+  b_1 = reinterpret_cast<const XPUTypeT*>(NULL);
+  a_2 = reinterpret_cast<const XPUTypeT*>(NULL);
+  b_2 = reinterpret_cast<const XPUTypeT*>(NULL);
+
+  c_1 = (pre_layer_norm == true ? d_linear1_out_ptr : d_x_ptr);
+  c_2 = d_linear1_weight_ptr;
+  phi::XpuFcInfo info_dx;
+  phi::XpuFcInfo info_dw1;
+
+  const XPUTypeT* linear1_x_ptr =
+      (pre_layer_norm == true
+           ? reinterpret_cast<const XPUTypeT*>(ln1_out->data<T>())
+           : x_ptr);
+
+  if (l3_total_size >= d_model * bsz_seq * sizeof(T) && info_dw1.trans_x) {
+    r = xpu::transpose<XPUTypeT>(
+        xpu_ctx, linear1_x_ptr, small_tmp_l3_ptr, {bsz_seq, d_model}, {1, 0});
+    PADDLE_ENFORCE_XDNN_SUCCESS(r, "transpose");
+    a_2 = small_tmp_l3_ptr;
+    info_dw1.trans_x = !info_dw1.trans_x;
+    info_dw1.stride_x = info_dw1.k;
+  }
+
+  fc_info = phi::MatmulGradFcInfo(xpu_ctx,
+                                  &RAII_GUARD,
+                                  linear1_fc_info,
+                                  false,
+                                  false,
+                                  linear1_x_ptr,
+                                  linear1_weight_ptr,
+                                  d_act_out_ptr);
+
+  std::tie(info_dx, info_dw1, a_1, b_1, a_2, b_2) = fc_info;
+
+  phi::MatMulXPUFunction<XPUTypeT>(xpu_ctx, a_1, b_1, c_1, info_dx, 1.0f, true);
+
+  phi::MatMulXPUFunction<XPUTypeT>(
+      xpu_ctx, a_2, b_2, c_2, info_dw1, 1.0f, true);
+
+  if (pre_layer_norm) {
+    r = xpu::layer_norm_grad(xpu_ctx,
+                             x_ptr,
+                             c_1,
+                             c_1,
+                             bsz_seq,
+                             d_model,
+                             epsilon,
+                             ln_scale_ptr,
+                             ln_mean_ptr,
+                             ln_variance_ptr,
+                             d_ln_scale_ptr,
+                             d_ln_bias_ptr);
+    PADDLE_ENFORCE_XDNN_SUCCESS(r, "layer_norm_grad");
+  }
+
+  r = xpu::add(xpu_ctx, c_1, d_residual_ptr, d_x_ptr, d_model * bsz_seq);
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "add");
+}
+
+template <typename T, typename Context>
+void FusedFeedForwardGradKernel(
+    const Context& dev_ctx,
+    const DenseTensor& out_grad,
+    const DenseTensor& x,
+    const DenseTensor& linear1_weight,
+    const DenseTensor& linear1_bias,
+    const DenseTensor& linear2_weight,
+    const DenseTensor& dropout1_mask,
+    const DenseTensor& dropout2_mask,
+    const DenseTensor& linear1_out,
+    const DenseTensor& dropout1_out,
+    const DenseTensor& dropout2_out,
+    const paddle::optional<DenseTensor>& ln1_scale,
+    const paddle::optional<DenseTensor>& ln1_bias,
+    const paddle::optional<DenseTensor>& ln1_out,
+    const paddle::optional<DenseTensor>& ln1_mean,
+    const paddle::optional<DenseTensor>& ln1_variance,
+    const paddle::optional<DenseTensor>& ln2_scale,
+    const paddle::optional<DenseTensor>& ln2_bias,
+    const paddle::optional<DenseTensor>& ln2_mean,
+    const paddle::optional<DenseTensor>& ln2_variance,
+    const paddle::optional<DenseTensor>& linear2_bias,
+    bool pre_layer_norm,
+    float ln1_epsilon,
+    float ln2_epsilon,
+    const std::string& act_method,
+    float dropout1_prob,
+    float dropout2_prob,
+    const std::string& dropout1_implementation,
+    const std::string& dropout2_implementation,
+    bool is_test,
+    bool dropout1_fix_seed,
+    bool dropout2_fix_seed,
+    int dropout1_seed_val,
+    int dropout2_seed_val,
+    bool add_residual,
+    int ring_id,
+    DenseTensor* x_grad,
+    DenseTensor* ln1_scale_grad,
+    DenseTensor* ln1_bias_grad,
+    DenseTensor* ln2_scale_grad,
+    DenseTensor* ln2_bias_grad,
+    DenseTensor* linear1_weight_grad,
+    DenseTensor* linear1_bias_grad,
+    DenseTensor* linear2_weight_grad,
+    DenseTensor* linear2_bias_grad) {
+  // inputs
+  auto* d_out = &out_grad;
+  auto* x_ptr = &x;
+
+  auto* dropout1_mask_ptr = &dropout1_mask;
+  auto* dropout2_mask_ptr = &dropout2_mask;
+  auto* linear1_out_ptr = &linear1_out;
+  auto* ln1_out_ptr = pre_layer_norm ? ln1_out.get_ptr() : nullptr;
+
+  auto* dropout1_out_ptr = &dropout1_out;
+  auto* dropout2_out_ptr = &dropout2_out;
+  auto* linear1_weight_ptr = &linear1_weight;
+  auto* linear2_weight_ptr = &linear2_weight;
+
+  const phi::DenseTensor* ln_mean = nullptr;
+  const phi::DenseTensor* ln_variance = nullptr;
+  const phi::DenseTensor* ln_scale = nullptr;
+
+  if (pre_layer_norm) {
+    ln_mean = ln1_mean.get_ptr();
+    ln_variance = ln1_variance.get_ptr();
+    ln_scale = ln1_scale.get_ptr();
+  } else {
+    ln_mean = ln2_mean.get_ptr();
+    ln_variance = ln2_variance.get_ptr();
+    ln_scale = ln2_scale.get_ptr();
+  }
+
+  // output
+  auto* d_x = x_grad;
+
+  phi::DenseTensor* d_ln_scale = nullptr;
+  phi::DenseTensor* d_ln_bias = nullptr;
+
+  if (pre_layer_norm) {
+    d_ln_scale = ln1_scale_grad;
+    d_ln_bias = ln1_bias_grad;
+  } else {
+    d_ln_scale = ln2_scale_grad;
+    d_ln_bias = ln2_bias_grad;
+  }
+
+  auto* d_linear1_weight = linear1_weight_grad;
+  auto* d_linear1_bias = linear1_bias_grad;
+  auto* d_linear2_weight = linear2_weight_grad;
+  auto* d_linear2_bias = linear2_bias_grad;
+
+  float epsilon = 0.0f;
+  if (pre_layer_norm) {
+    epsilon = ln1_epsilon;
+  } else {
+    epsilon = ln2_epsilon;
+  }
+
+  bool is_upscale_in_train_1 = dropout1_implementation == "upscale_in_train";
+  bool is_upscale_in_train_2 = dropout2_implementation == "upscale_in_train";
+
+  phi::XPUDropoutParam dropout_param1;
+  dropout_param1.initXPUDropoutParam(dropout1_prob,
+                                     is_upscale_in_train_1,
+                                     is_test,
+                                     dropout1_fix_seed,
+                                     nullptr,
+                                     dropout1_seed_val);
+  phi::XPUDropoutParam dropout_param2;
+  dropout_param2.initXPUDropoutParam(dropout2_prob,
+                                     is_upscale_in_train_2,
+                                     is_test,
+                                     dropout2_fix_seed,
+                                     nullptr,
+                                     dropout2_seed_val);
+
+  dev_ctx.template Alloc<float>(d_ln_scale);
+  dev_ctx.template Alloc<float>(d_ln_bias);
+  dev_ctx.template Alloc<T>(d_linear1_bias);
+  dev_ctx.template Alloc<T>(d_linear2_bias);
+  dev_ctx.template Alloc<T>(d_linear1_weight);
+  dev_ctx.template Alloc<T>(d_linear2_weight);
+
+  auto x_dim = x_ptr->dims();
+  auto mat_dim_x = phi::funcs::CreateMatrixDescriptor(
+      phi::RowMatrixFromVector(x_dim), 0, false);
+
+  auto linear1_weight_dim = linear1_weight_ptr->dims();
+  int d_model = linear1_weight_dim[0];
+  int dim_feedforward = linear1_weight_dim[linear1_weight_dim.size() - 1];
+  int bsz_seq = mat_dim_x.batch_size_ * mat_dim_x.height_;
+
+  FFNGrad<T, Context>(dev_ctx,
+                      d_out,
+                      x_ptr,
+                      dropout1_mask_ptr,
+                      dropout2_mask_ptr,
+                      linear1_out_ptr,
+                      ln1_out_ptr,
+                      dropout1_out_ptr,
+                      dropout2_out_ptr,
+                      linear1_weight_ptr,
+                      linear2_weight_ptr,
+                      ln_scale,
+                      ln_mean,
+                      ln_variance,
+                      d_x,
+                      d_linear1_weight,
+                      d_linear1_bias,
+                      d_linear2_weight,
+                      d_linear2_bias,
+                      d_ln_scale,
+                      d_ln_bias,
+                      bsz_seq,
+                      d_model,
+                      dim_feedforward,
+                      dropout_param1,
+                      dropout_param2,
+                      act_method,
+                      pre_layer_norm,
+                      epsilon,
+                      ring_id);
+}
+}  // namespace fusion
+}  // namespace phi
+
+PD_REGISTER_KERNEL(fused_feedward_grad,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::fusion::FusedFeedForwardGradKernel,
+                   float,
+                   phi::dtype::float16) {
+  kernel->OutputAt(1).SetDataType(phi::DataType::FLOAT32);
+  kernel->OutputAt(2).SetDataType(phi::DataType::FLOAT32);
+  kernel->OutputAt(3).SetDataType(phi::DataType::FLOAT32);
+  kernel->OutputAt(4).SetDataType(phi::DataType::FLOAT32);
+}
diff --git a/paddle/phi/kernels/fusion/xpu/fused_feedforward_xpu_kernel.cc b/paddle/phi/kernels/fusion/xpu/fused_feedforward_xpu_kernel.cc
new file mode 100644
index 0000000000000..35039ba571e57
--- /dev/null
+++ b/paddle/phi/kernels/fusion/xpu/fused_feedforward_xpu_kernel.cc
@@ -0,0 +1,390 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/backends/xpu/enforce_xpu.h"
+#include "paddle/phi/backends/xpu/xpu_context.h"
+#include "paddle/phi/core/ddim.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/blas/blas.h"
+#include "paddle/phi/kernels/impl/matmul_grad_kernel_impl.h"
+#include "paddle/phi/kernels/xpu/xpu_api_wrapper.h"
+#include "paddle/phi/kernels/xpu/xpu_fused_common_function.h"
+
+namespace phi {
+namespace fusion {
+
+template <typename T, typename Context>
+void FFN(const phi::XPUContext& dev_ctx,
+         const phi::DenseTensor* x,
+         const phi::DenseTensor* linear1_weight,
+         const phi::DenseTensor* linear1_bias,
+         const phi::DenseTensor* linear2_weight,
+         const phi::DenseTensor* linear2_bias,
+         const phi::DenseTensor* ln_scale,
+         const phi::DenseTensor* ln_bias,
+         phi::DenseTensor* out,
+         phi::DenseTensor* dropout1_mask,
+         phi::DenseTensor* dropout2_mask,
+         phi::DenseTensor* ln_mean,
+         phi::DenseTensor* ln_variance,
+         phi::DenseTensor* linear1_out,
+         phi::DenseTensor* ln1_out,
+         phi::DenseTensor* dropout1_out,
+         phi::DenseTensor* dropout2_out,
+         const int bsz_seq,
+         const int d_model,
+         const int dim_feedforward,
+         const std::string& act_method,
+         const bool pre_layer_norm,
+         const float epsilon1,
+         const float epsilon2,
+         const phi::XPUDropoutParam& dropout_param1,
+         const phi::XPUDropoutParam& dropout_param2,
+         int ring_id) {
+  using XPUTypeT = typename XPUTypeTrait<T>::Type;
+  xpu::Context* xpu_ctx = dev_ctx.x_context();
+  xpu::ctx_guard RAII_GUARD(xpu_ctx);
+
+  int r = xpu::SUCCESS;
+
+  const XPUTypeT* x_ptr = reinterpret_cast<const XPUTypeT*>(x->data<T>());
+  const XPUTypeT* residual_ptr = x_ptr;
+  const XPUTypeT* linear1_weight_ptr =
+      reinterpret_cast<const XPUTypeT*>(linear1_weight->data<T>());
+  const XPUTypeT* linear1_bias_ptr =
+      reinterpret_cast<const XPUTypeT*>(linear1_bias->data<T>());
+  const XPUTypeT* linear2_weight_ptr =
+      reinterpret_cast<const XPUTypeT*>(linear2_weight->data<T>());
+  const XPUTypeT* linear2_bias_ptr =
+      reinterpret_cast<const XPUTypeT*>(linear2_bias->data<T>());
+
+  const float* ln_scale_ptr = ln_scale->data<float>();
+
+  const float* ln_bias_ptr = ln_bias->data<float>();
+
+  // out
+  XPUTypeT* out_ptr = reinterpret_cast<XPUTypeT*>(out->data<T>());
+  XPUTypeT* linear1_out_ptr =
+      reinterpret_cast<XPUTypeT*>(linear1_out->data<T>());
+  XPUTypeT* dropout1_mask_ptr =
+      reinterpret_cast<XPUTypeT*>(dropout1_mask->data<T>());
+  XPUTypeT* dropout2_mask_ptr =
+      reinterpret_cast<XPUTypeT*>(dropout2_mask->data<T>());
+  float* ln_mean_ptr = ln_mean->data<float>();
+  float* ln_variance_ptr = ln_variance->data<float>();
+
+  XPUTypeT* dropout1_out_ptr =
+      reinterpret_cast<XPUTypeT*>(dropout1_out->data<T>());
+  XPUTypeT* dropout2_out_ptr =
+      reinterpret_cast<XPUTypeT*>(dropout2_out->data<T>());
+
+  size_t l3_total_size = xpu_ctx->_l3_mgr.get_size();
+  XPUTypeT* linear2_before_tmp_ptr = NULL;  // dim_feedforward * bsz_seq
+  XPUTypeT* linear2_after_tmp_ptr = NULL;   // d_model * bsz_seq
+  if (l3_total_size >= dim_feedforward * bsz_seq * sizeof(T)) {
+    XPUTypeT* l3_ptr = RAII_GUARD.alloc_l3<XPUTypeT>(dim_feedforward * bsz_seq);
+    PADDLE_ENFORCE_XDNN_NOT_NULL(l3_ptr);
+    linear2_before_tmp_ptr = linear2_after_tmp_ptr = l3_ptr;
+  } else if ((l3_total_size < dim_feedforward * bsz_seq * sizeof(T)) &&
+             (l3_total_size >= d_model * bsz_seq * sizeof(T))) {
+    XPUTypeT* l3_ptr = RAII_GUARD.alloc_l3<XPUTypeT>(d_model * bsz_seq);
+    PADDLE_ENFORCE_XDNN_NOT_NULL(l3_ptr);
+    linear2_after_tmp_ptr = l3_ptr;
+    linear2_before_tmp_ptr =
+        RAII_GUARD.alloc<XPUTypeT>(dim_feedforward * bsz_seq);
+    PADDLE_ENFORCE_XDNN_NOT_NULL(linear2_before_tmp_ptr);
+
+  } else {
+    XPUTypeT* gm_ptr = RAII_GUARD.alloc<XPUTypeT>(dim_feedforward * bsz_seq);
+    PADDLE_ENFORCE_XDNN_NOT_NULL(gm_ptr);
+    linear2_before_tmp_ptr = linear2_after_tmp_ptr = gm_ptr;
+  }
+
+  // layernorm
+  if (pre_layer_norm) {
+    XPUTypeT* ln1_out_ptr = reinterpret_cast<XPUTypeT*>(ln1_out->data<T>());
+    r = xpu::layer_norm(xpu_ctx,
+                        x_ptr,
+                        ln1_out_ptr,
+                        bsz_seq,
+                        d_model,
+                        epsilon1,
+                        ln_scale_ptr,
+                        ln_bias_ptr,
+                        ln_mean_ptr,
+                        ln_variance_ptr);
+    PADDLE_ENFORCE_XDNN_SUCCESS(r, "layer_norm ");
+    x_ptr = ln1_out_ptr;
+  }
+
+  // fc
+  phi::XpuFcInfo linear1_fc_info;
+  linear1_fc_info.InitFcInfo(0,
+                             bsz_seq,
+                             dim_feedforward,
+                             d_model,
+                             false,
+                             false,
+                             nullptr,
+                             nullptr,
+                             nullptr);
+  phi::MatMulXPUFunction<XPUTypeT>(xpu_ctx,
+                                   x_ptr,
+                                   linear1_weight_ptr,
+                                   linear2_before_tmp_ptr,
+                                   linear1_fc_info,
+                                   1.0f);
+
+  // bias
+  r = xpu::broadcast_add(xpu_ctx,
+                         linear2_before_tmp_ptr,
+                         linear1_bias_ptr,
+                         linear1_out_ptr,
+                         {bsz_seq, dim_feedforward},
+                         {dim_feedforward});
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast_add");
+
+  // act
+  if (act_method == "gelu") {
+    r = xpu::gelu(
+        xpu_ctx, linear1_out_ptr, linear2_before_tmp_ptr, linear1_out->numel());
+    PADDLE_ENFORCE_XDNN_SUCCESS(r, "gelu");
+  } else if (act_method == "relu") {
+    r = xpu::relu(
+        xpu_ctx, linear1_out_ptr, linear2_before_tmp_ptr, linear1_out->numel());
+    PADDLE_ENFORCE_XDNN_SUCCESS(r, "relu");
+  } else {
+    PADDLE_THROW(phi::errors::Unimplemented(
+        "Currently only supports gelu or relu activation functions!"));
+  }
+
+  // dropout1
+  phi::Dropout<XPUTypeT>(xpu_ctx,
+                         linear2_before_tmp_ptr,
+                         dropout1_mask_ptr,
+                         dropout1_out_ptr,
+                         dropout_param1,
+                         dropout1_out->numel());
+
+  // fc
+  phi::XpuFcInfo linear2_fc_info;
+  linear2_fc_info.InitFcInfo(0,
+                             bsz_seq,
+                             d_model,
+                             dim_feedforward,
+                             false,
+                             false,
+                             nullptr,
+                             nullptr,
+                             nullptr);
+  phi::MatMulXPUFunction<XPUTypeT>(xpu_ctx,
+                                   dropout1_out_ptr,
+                                   linear2_weight_ptr,
+                                   dropout2_out_ptr,
+                                   linear2_fc_info,
+                                   1.0f);
+
+  // bias
+  r = xpu::broadcast_add(xpu_ctx,
+                         dropout2_out_ptr,
+                         linear2_bias_ptr,
+                         dropout2_out_ptr,
+                         {bsz_seq, d_model},
+                         {d_model});
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast_add");
+
+  // dropout2
+  phi::Dropout<XPUTypeT>(xpu_ctx,
+                         dropout2_out_ptr,
+                         dropout2_mask_ptr,
+                         dropout2_out_ptr,
+                         dropout_param2,
+                         dropout2_out->numel());
+
+  // residual_ptr + dropout_out
+  XPUTypeT* residual_add_out_ptr = out_ptr;
+  if (pre_layer_norm == false) {
+    residual_add_out_ptr = dropout2_out_ptr;
+  }
+  r = xpu::broadcast_add(xpu_ctx,
+                         residual_ptr,
+                         dropout2_out_ptr,
+                         residual_add_out_ptr,
+                         {bsz_seq, d_model},
+                         {bsz_seq, d_model});
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast_add");
+
+  if (pre_layer_norm == false) {
+    r = xpu::layer_norm(xpu_ctx,
+                        residual_add_out_ptr,
+                        out_ptr,
+                        bsz_seq,
+                        d_model,
+                        epsilon2,
+                        ln_scale_ptr,
+                        ln_bias_ptr,
+                        ln_mean_ptr,
+                        ln_variance_ptr);
+    PADDLE_ENFORCE_XDNN_SUCCESS(r, "layer_norm");
+  }
+}
+
+template <typename T, typename Context>
+void FusedFeedForwardKernel(const Context& dev_ctx,
+                            const DenseTensor& x,
+                            const paddle::optional<DenseTensor>& dropout1_seed,
+                            const paddle::optional<DenseTensor>& dropout2_seed,
+                            const DenseTensor& linear1_weight,
+                            const paddle::optional<DenseTensor>& linear1_bias,
+                            const DenseTensor& linear2_weight,
+                            const paddle::optional<DenseTensor>& linear2_bias,
+                            const paddle::optional<DenseTensor>& ln1_scale,
+                            const paddle::optional<DenseTensor>& ln1_bias,
+                            const paddle::optional<DenseTensor>& ln2_scale,
+                            const paddle::optional<DenseTensor>& ln2_bias,
+                            bool pre_layer_norm,
+                            float ln1_epsilon,
+                            float ln2_epsilon,
+                            const std::string& act_method,
+                            float dropout1_prob,
+                            float dropout2_prob,
+                            const std::string& dropout1_implementation,
+                            const std::string& dropout2_implementation,
+                            bool is_test,
+                            bool dropout1_fix_seed,
+                            bool dropout2_fix_seed,
+                            int dropout1_seed_val,
+                            int dropout2_seed_val,
+                            bool add_residual,
+                            int ring_id,
+                            DenseTensor* out,
+                            DenseTensor* dropout1_mask,
+                            DenseTensor* dropout2_mask,
+                            DenseTensor* ln1_mean,
+                            DenseTensor* ln1_variance,
+                            DenseTensor* ln2_mean,
+                            DenseTensor* ln2_variance,
+                            DenseTensor* linear1_out,
+                            DenseTensor* ln1_out,
+                            DenseTensor* dropout1_out,
+                            DenseTensor* dropout2_out) {
+  auto* x_ptr = &x;
+  auto* linear1_weight_ptr = &linear1_weight;
+  auto* linear1_bias_ptr = linear1_bias.get_ptr();
+  auto* linear2_weight_ptr = &linear2_weight;
+  auto* linear2_bias_ptr = linear2_bias.get_ptr();
+
+  const phi::DenseTensor* ln_scale = nullptr;
+  const phi::DenseTensor* ln_bias = nullptr;
+  phi::DenseTensor* ln_mean = nullptr;
+  phi::DenseTensor* ln_variance = nullptr;
+
+  if (pre_layer_norm) {
+    ln_scale = ln1_scale.get_ptr();
+    ln_bias = ln1_bias.get_ptr();
+    ln_mean = ln1_mean;
+    ln_variance = ln1_variance;
+    dev_ctx.template Alloc<T>(ln1_out);
+  } else {
+    ln_scale = ln2_scale.get_ptr();
+    ln_bias = ln2_bias.get_ptr();
+    ln_mean = ln2_mean;
+    ln_variance = ln2_variance;
+  }
+
+  const float epsilon1 = ln1_epsilon;
+  const float epsilon2 = ln2_epsilon;
+
+  bool is_upscale_in_train_1 = dropout1_implementation == "upscale_in_train";
+  bool is_upscale_in_train_2 = dropout2_implementation == "upscale_in_train";
+
+  auto* dropout1_seed_ptr = dropout1_seed.get_ptr();
+  auto* dropout2_seed_ptr = dropout2_seed.get_ptr();
+  phi::XPUDropoutParam dropout_param1;
+  dropout_param1.initXPUDropoutParam(dropout1_prob,
+                                     is_upscale_in_train_1,
+                                     is_test,
+                                     dropout1_fix_seed,
+                                     dropout1_seed_ptr,
+                                     dropout1_seed_val);
+  phi::XPUDropoutParam dropout_param2;
+  dropout_param2.initXPUDropoutParam(dropout2_prob,
+                                     is_upscale_in_train_2,
+                                     is_test,
+                                     dropout2_fix_seed,
+                                     dropout2_seed_ptr,
+                                     dropout2_seed_val);
+
+  dev_ctx.template Alloc<float>(ln_mean);
+  dev_ctx.template Alloc<float>(ln_variance);
+
+  dev_ctx.template Alloc<T>(out);
+  dev_ctx.template Alloc<T>(dropout1_mask);
+  dev_ctx.template Alloc<T>(dropout2_mask);
+  dev_ctx.template Alloc<T>(dropout1_out);
+  dev_ctx.template Alloc<T>(dropout2_out);
+  dev_ctx.template Alloc<T>(linear1_out);
+
+  auto x_dim = x_ptr->dims();
+  auto mat_dim_x = phi::funcs::CreateMatrixDescriptor(
+      phi::RowMatrixFromVector(x_dim), 0, false);
+
+  auto dim = linear1_weight_ptr->dims();
+  int d_model = dim[0];
+  int dim_feedforward = dim[dim.size() - 1];
+  int bsz_seq = mat_dim_x.batch_size_ * mat_dim_x.height_;
+
+  phi::fusion::FFN<T, Context>(dev_ctx,
+                               x_ptr,
+                               linear1_weight_ptr,
+                               linear1_bias_ptr,
+                               linear2_weight_ptr,
+                               linear2_bias_ptr,
+                               ln_scale,
+                               ln_bias,
+                               out,
+                               dropout1_mask,
+                               dropout2_mask,
+                               ln_mean,
+                               ln_variance,
+                               linear1_out,
+                               ln1_out,
+                               dropout1_out,
+                               dropout2_out,
+                               bsz_seq,
+                               d_model,
+                               dim_feedforward,
+                               act_method,
+                               pre_layer_norm,
+                               epsilon1,
+                               epsilon2,
+                               dropout_param1,
+                               dropout_param2,
+                               ring_id);
+}
+}  // namespace fusion
+}  // namespace phi
+
+PD_REGISTER_KERNEL(fused_feedward,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::fusion::FusedFeedForwardKernel,
+                   float,
+                   phi::dtype::float16) {
+  kernel->OutputAt(3).SetDataType(phi::DataType::FLOAT32);
+  kernel->OutputAt(4).SetDataType(phi::DataType::FLOAT32);
+  kernel->OutputAt(5).SetDataType(phi::DataType::FLOAT32);
+  kernel->OutputAt(6).SetDataType(phi::DataType::FLOAT32);
+}
diff --git a/paddle/phi/ops/compat/fused_feedforward_sig.cc b/paddle/phi/ops/compat/fused_feedforward_sig.cc
new file mode 100644
index 0000000000000..1dd78288deaf8
--- /dev/null
+++ b/paddle/phi/ops/compat/fused_feedforward_sig.cc
@@ -0,0 +1,102 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/core/compat/op_utils.h"
+
+namespace phi {
+
+KernelSignature FeedForwardFuseOpArgumentMapping(
+    const ArgumentMappingContext& ctx) {
+  return KernelSignature("fused_feedforward",
+                         {"X",
+                          "Dropout1Seed",
+                          "Dropout2Seed",
+                          "Linear1Weight",
+                          "Linear1Bias",
+                          "Linear2Weight",
+                          "Linear2Bias",
+                          "Ln1Scale",
+                          "Ln1Bias",
+                          "Ln2Scale",
+                          "Ln2Bias"},
+                         {"pre_layer_norm",
+                          "ln1_epsilon",
+                          "ln2_epsilon",
+                          "act_method",
+                          "dropout1_rate",
+                          "dropout2_rate",
+                          "dropout1_implementation",
+                          "dropout2_implementation",
+                          "is_test",
+                          "dropout1_fix_seed",
+                          "dropout2_fix_seed",
+                          "dropout1_seed",
+                          "dropout2_seed",
+                          "add_residual",
+                          "ring_id"},
+                         {"Out",
+                          "Dropout1Mask",
+                          "Dropout2Mask",
+                          "Ln1Mean",
+                          "Ln1Variance",
+                          "Ln2Mean",
+                          "Ln2Variance",
+                          "Linear1Out",
+                          "Ln1Out",
+                          "Dropout1Out",
+                          "Dropout2Out"});
+}
+
+KernelSignature FeedForwardGradFuseOpArgumentMapping(
+    const ArgumentMappingContext& ctx) {
+  return KernelSignature("fused_feedforward_grad",
+                         {"Out@GRAD",      "X",
+                          "Linear1Weight", "Linear1Bias",
+                          "Linear2Weight", "Dropout1Mask",
+                          "Dropout2Mask",  "Linear1Out",
+                          "Dropout1Out",   "Dropout2Out",
+                          "Ln1Scale",      "Ln1Bias",
+                          "Ln1Out",        "Ln1Mean",
+                          "Ln1Variance",   "Ln2Scale",
+                          "Ln2Bias",       "Ln2Mean",
+                          "Ln2Variance",   "Linear2Bias"},
+                         {"pre_layer_norm",
+                          "ln1_epsilon",
+                          "ln2_epsilon",
+                          "act_method",
+                          "dropout1_rate",
+                          "dropout2_rate",
+                          "dropout1_implementation",
+                          "dropout2_implementation",
+                          "is_test",
+                          "dropout1_fix_seed",
+                          "dropout2_fix_seed",
+                          "dropout1_seed",
+                          "dropout2_seed",
+                          "add_residual",
+                          "ring_id"},
+                         {"X@GRAD",
+                          "Ln1Scale@GRAD",
+                          "Ln1Bias@GRAD",
+                          "Ln2Scale@GRAD",
+                          "Ln2Bias@GRAD",
+                          "Linear1Weight@GRAD",
+                          "Linear1Bias@GRAD",
+                          "Linear2Weight@GRAD",
+                          "Linear2Bias@GRAD"});
+}
+}  // namespace phi
+
+PD_REGISTER_ARG_MAPPING_FN(fused_feedforward,
+                           phi::FeedForwardFuseOpArgumentMapping);
+
+PD_REGISTER_ARG_MAPPING_FN(fused_feedforward_grad,
+                           phi::FeedForwardGradFuseOpArgumentMapping);

From 81fb7df23d0321aa9b10516d9136dee23ea2ec36 Mon Sep 17 00:00:00 2001
From: zqw_1997 <118182234+zhengqiwen1997@users.noreply.github.com>
Date: Mon, 24 Apr 2023 16:02:08 +0800
Subject: [PATCH 014/405] [Zero-Dim] Support output 0D for to_tensor. (#52741)

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* fix doc erros, test=allcase
---
 paddle/fluid/pybind/imperative.cc             |  2 +-
 python/paddle/audio/functional/window.py      |  9 ++-
 .../pp_utils/p2p_communication.py             |  8 +--
 .../unittests/parallel_dygraph_none_var.py    |  2 +-
 .../fluid/tests/unittests/test_cholesky_op.py |  2 +-
 .../fluid/tests/unittests/test_deg2rad.py     |  2 +-
 .../unittests/test_deprecated_decorator.py    |  2 +-
 .../fluid/tests/unittests/test_einsum_v2.py   |  2 +-
 .../fluid/tests/unittests/test_inplace.py     |  4 +-
 .../tests/unittests/test_jit_save_load.py     |  6 +-
 .../fluid/tests/unittests/test_lbfgs_v2.py    | 66 +++++++++----------
 .../fluid/tests/unittests/test_rad2deg.py     |  4 +-
 .../unittests/test_state_dict_convert.py      |  2 +-
 .../unittests/test_tensor_register_hook.py    |  6 +-
 .../fluid/tests/unittests/test_var_base.py    |  4 +-
 .../tests/unittests/test_zero_dim_tensor.py   | 27 ++++++++
 python/paddle/tensor/creation.py              | 24 +++----
 python/paddle/tensor/math.py                  | 40 +++++------
 test/auto_parallel/random_control_unittest.py |  2 +-
 .../test_distribution_bernoulli.py            | 36 +++++-----
 .../test_distribution_transform.py            | 14 ++--
 ...t_distribution_transformed_distribution.py |  2 +-
 test/distribution/test_kl.py                  |  4 +-
 .../test_cpu_cuda_to_tensor.py                |  4 +-
 test/dygraph_to_static/test_fallback.py       |  2 +-
 test/dygraph_to_static/test_to_tensor.py      | 12 ++++
 test/legacy_test/auto_parallel_gpt_model.py   |  2 +-
 test/legacy_test/test_audio_functions.py      |  6 +-
 test/quantization/imperative_test_utils.py    |  2 +-
 29 files changed, 169 insertions(+), 129 deletions(-)

diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc
index 44d4d070eafb1..e60211286ed37 100644
--- a/paddle/fluid/pybind/imperative.cc
+++ b/paddle/fluid/pybind/imperative.cc
@@ -1321,7 +1321,7 @@ void BindImperative(py::module *m_ptr) {
 
                 import paddle
 
-                x = paddle.to_tensor(1.0, stop_gradient=False)
+                x = paddle.to_tensor([1.0], stop_gradient=False)
                 detach_x = x.detach()
                 detach_x[:] = 10.0
                 print(x)  # Tensor(shape=[1], dtype=float32, place=CPUPlace, stop_gradient=False,
diff --git a/python/paddle/audio/functional/window.py b/python/paddle/audio/functional/window.py
index b2dd63973d3f2..eb84d6f188970 100644
--- a/python/paddle/audio/functional/window.py
+++ b/python/paddle/audio/functional/window.py
@@ -13,6 +13,8 @@
 import math
 from typing import List, Tuple, Union
 
+import numpy as np
+
 import paddle
 from paddle import Tensor
 
@@ -38,7 +40,12 @@ def get(self, name):
 
 @window_function_register.register()
 def _cat(x: List[Tensor], data_type: str) -> Tensor:
-    l = [paddle.to_tensor(_, data_type) for _ in x]
+    l = []
+    for t in x:
+        if np.isscalar(t) and not isinstance(t, str):
+            l.append(paddle.to_tensor([t], data_type))
+        else:
+            l.append(paddle.to_tensor(t, data_type))
     return paddle.concat(l)
 
 
diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
index 81385f9a0a4a1..9d3cb5bd17098 100644
--- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
+++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
@@ -117,7 +117,7 @@ def recv_meta(self, group):
 
     def _send_dims_shape_dtype(self, tensor, group):
         # send len(shape)
-        dims = paddle.to_tensor(len(tensor.shape))
+        dims = paddle.to_tensor([len(tensor.shape)])
         dst_rank = _hcg._get_p2p_next_rank()
 
         paddle.distributed.send(dims, dst=dst_rank, group=group)
@@ -127,11 +127,11 @@ def _send_dims_shape_dtype(self, tensor, group):
         paddle.distributed.send(shape, dst=dst_rank, group=group)
 
         # send dtype
-        dtype = paddle.to_tensor(paddle_2_number(tensor.dtype))
+        dtype = paddle.to_tensor([paddle_2_number(tensor.dtype)])
         paddle.distributed.send(dtype, dst=dst_rank, group=group)
 
         # send trainable
-        stop_grad = paddle.to_tensor(int(tensor.stop_gradient))
+        stop_grad = paddle.to_tensor([int(tensor.stop_gradient)])
         paddle.distributed.send(stop_grad, dst=dst_rank, group=group)
 
     def send_meta(self, tensor, group):
@@ -148,7 +148,7 @@ def send_meta(self, tensor, group):
             # send tensor type
             paddle.distributed.send(tensor_type, dst=dst_rank, group=group)
 
-            nums = paddle.to_tensor(len(tensor))
+            nums = paddle.to_tensor([len(tensor)])
             paddle.distributed.send(nums, dst=dst_rank, group=group)
 
             for d in tensor:
diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_none_var.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_none_var.py
index fa5d71dbfe2a9..f7720902946e2 100644
--- a/python/paddle/fluid/tests/unittests/parallel_dygraph_none_var.py
+++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_none_var.py
@@ -40,7 +40,7 @@ def __init__(self):
         self.step = 0
 
     def forward(self, x):
-        return paddle.to_tensor(0.0, dtype='float32')
+        return paddle.to_tensor([0.0], dtype='float32')
 
 
 def fake_sample_reader():
diff --git a/python/paddle/fluid/tests/unittests/test_cholesky_op.py b/python/paddle/fluid/tests/unittests/test_cholesky_op.py
index ba7a6fb631ad3..b1261c3d17b22 100644
--- a/python/paddle/fluid/tests/unittests/test_cholesky_op.py
+++ b/python/paddle/fluid/tests/unittests/test_cholesky_op.py
@@ -104,7 +104,7 @@ def test_dygraph(self):
         a = np.random.rand(3, 3)
         a_t = np.transpose(a, [1, 0])
         x_data = np.matmul(a, a_t) + 1e-03
-        x = paddle.to_tensor(x_data)
+        x = paddle.to_tensor([x_data])
         out = paddle.cholesky(x, upper=False)
 
 
diff --git a/python/paddle/fluid/tests/unittests/test_deg2rad.py b/python/paddle/fluid/tests/unittests/test_deg2rad.py
index 77dce311a939d..0f038e86f2522 100644
--- a/python/paddle/fluid/tests/unittests/test_deg2rad.py
+++ b/python/paddle/fluid/tests/unittests/test_deg2rad.py
@@ -74,7 +74,7 @@ def setUp(self):
     def test_dygraph(self):
         paddle.disable_static()
 
-        x2 = paddle.to_tensor(180)
+        x2 = paddle.to_tensor([180])
         result2 = paddle.deg2rad(x2)
         np.testing.assert_allclose(np.pi, result2.numpy(), rtol=1e-05)
 
diff --git a/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py b/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py
index e2ed84a57c5d0..4624897f0168e 100755
--- a/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py
+++ b/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py
@@ -111,7 +111,7 @@ def test_ops_elementwise_mul(self):
     def test_tensor_gradient(self):
         paddle.__version__ = '2.1.0'
 
-        x = paddle.to_tensor(5.0, stop_gradient=False)
+        x = paddle.to_tensor([5.0], stop_gradient=False)
         y = paddle.pow(x, 4.0)
         y.backward()
 
diff --git a/python/paddle/fluid/tests/unittests/test_einsum_v2.py b/python/paddle/fluid/tests/unittests/test_einsum_v2.py
index becadd85c02d8..434c59b5b804e 100644
--- a/python/paddle/fluid/tests/unittests/test_einsum_v2.py
+++ b/python/paddle/fluid/tests/unittests/test_einsum_v2.py
@@ -553,7 +553,7 @@ def test_shape(self):
             B = paddle.to_tensor(np.array([2.0, 3.0])).astype(paddle.bfloat16)
             B = B.cuda()
             C = paddle.einsum('i,i->', A, B)
-            D = paddle.to_tensor(8.0).astype(paddle.bfloat16)
+            D = paddle.to_tensor([8.0]).astype(paddle.bfloat16)
             self.assertEqual(C.item(), D.item())
 
 
diff --git a/python/paddle/fluid/tests/unittests/test_inplace.py b/python/paddle/fluid/tests/unittests/test_inplace.py
index c10d7d2c5c457..91a569d34c62b 100644
--- a/python/paddle/fluid/tests/unittests/test_inplace.py
+++ b/python/paddle/fluid/tests/unittests/test_inplace.py
@@ -259,11 +259,11 @@ def inplace_api_processing(self, var):
 
 class TestDygraphInplaceReshapeTensor(TestDygraphInplace):
     def non_inplace_api_processing(self, var):
-        shape = paddle.to_tensor(-1)
+        shape = paddle.to_tensor([-1])
         return paddle.reshape(var, shape)
 
     def inplace_api_processing(self, var):
-        shape = paddle.to_tensor(-1)
+        shape = paddle.to_tensor([-1])
         return paddle.reshape_(var, shape)
 
 
diff --git a/python/paddle/fluid/tests/unittests/test_jit_save_load.py b/python/paddle/fluid/tests/unittests/test_jit_save_load.py
index d04357e201cbb..7f58638e7ac7a 100644
--- a/python/paddle/fluid/tests/unittests/test_jit_save_load.py
+++ b/python/paddle/fluid/tests/unittests/test_jit_save_load.py
@@ -283,7 +283,7 @@ def __init__(self, in_size, out_size):
         super().__init__()
         self._linear_0 = Linear(in_size, out_size)
         self._linear_1 = Linear(in_size, out_size)
-        self._scale = paddle.to_tensor(9.9)
+        self._scale = paddle.to_tensor([9.9])
 
     @paddle.jit.to_static
     def forward(self, x):
@@ -1196,7 +1196,7 @@ def __init__(self, in_size, out_size):
         self._linear_1_0 = Linear(self.hidden, self.hidden)
         self._linear_1_1 = Linear(self.hidden, self.hidden)
         self._linear_2 = Linear(self.hidden, out_size)
-        self._scale = paddle.to_tensor(9.9)
+        self._scale = paddle.to_tensor([9.9])
 
     @paddle.jit.to_static
     def forward(self, x):
@@ -1319,7 +1319,7 @@ def __init__(self, in_size, out_size, load_path):
         self._linear_1_0 = Linear(out_size, in_size)
         self._linear_1_1 = Linear(out_size, in_size)
         self._linear_2 = Linear(out_size, out_size)
-        self._scale = paddle.to_tensor(9.9)
+        self._scale = paddle.to_tensor([9.9])
 
         # Load multiple times
         self._load_l1 = paddle.jit.load(load_path)
diff --git a/python/paddle/fluid/tests/unittests/test_lbfgs_v2.py b/python/paddle/fluid/tests/unittests/test_lbfgs_v2.py
index fa64c480ed970..9617938967cd3 100644
--- a/python/paddle/fluid/tests/unittests/test_lbfgs_v2.py
+++ b/python/paddle/fluid/tests/unittests/test_lbfgs_v2.py
@@ -208,64 +208,64 @@ def error_func():
 
     def test_line_search(self):
         def func1(x, alpha, d):
-            return paddle.to_tensor(x + alpha * d), paddle.to_tensor(0.0)
+            return paddle.to_tensor(x + alpha * d), paddle.to_tensor([0.0])
 
         def func2(x, alpha, d):
-            return paddle.to_tensor(x + alpha * d), paddle.to_tensor(1.0)
+            return paddle.to_tensor(x + alpha * d), paddle.to_tensor([1.0])
 
         def func3(x, alpha, d):
-            return paddle.to_tensor(x + alpha * d), paddle.to_tensor(-1.0)
+            return paddle.to_tensor(x + alpha * d), paddle.to_tensor([-1.0])
 
         _strong_wolfe(
             func1,
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(0.001),
-            paddle.to_tensor(0.0),
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(0.0),
-            paddle.to_tensor(0.0),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([0.001]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([0.0]),
             max_ls=0,
         )
 
         _strong_wolfe(
             func2,
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(-0.001),
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(1.0),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([-0.001]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
             max_ls=1,
         )
 
         _strong_wolfe(
             func3,
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(-0.001),
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(1.0),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([-0.001]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
             max_ls=1,
         )
 
         _cubic_interpolate(
-            paddle.to_tensor(2.0),
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(0.0),
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(2.0),
-            paddle.to_tensor(0.0),
+            paddle.to_tensor([2.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([2.0]),
+            paddle.to_tensor([0.0]),
             [0.1, 0.5],
         )
 
         _cubic_interpolate(
-            paddle.to_tensor(2.0),
-            paddle.to_tensor(0.0),
-            paddle.to_tensor(-3.0),
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(1.0),
-            paddle.to_tensor(-0.1),
+            paddle.to_tensor([2.0]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([-3.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([-0.1]),
             [0.1, 0.5],
         )
 
diff --git a/python/paddle/fluid/tests/unittests/test_rad2deg.py b/python/paddle/fluid/tests/unittests/test_rad2deg.py
index 8629d7dcd37f2..710d77f0d9fc1 100644
--- a/python/paddle/fluid/tests/unittests/test_rad2deg.py
+++ b/python/paddle/fluid/tests/unittests/test_rad2deg.py
@@ -73,7 +73,7 @@ def setUp(self):
     def test_dygraph(self):
         paddle.disable_static()
 
-        x2 = paddle.to_tensor(np.pi / 2)
+        x2 = paddle.to_tensor([np.pi / 2])
         result2 = paddle.rad2deg(x2)
         np.testing.assert_allclose(90, result2.numpy(), rtol=1e-05)
 
@@ -91,7 +91,7 @@ def setUp(self):
     def test_dygraph(self):
         paddle.disable_static()
 
-        x2 = paddle.to_tensor(1)
+        x2 = paddle.to_tensor([1])
         result2 = paddle.rad2deg(x2)
         np.testing.assert_allclose(180 / np.pi, result2.numpy(), rtol=1e-05)
 
diff --git a/python/paddle/fluid/tests/unittests/test_state_dict_convert.py b/python/paddle/fluid/tests/unittests/test_state_dict_convert.py
index a6b6975eed7a7..90bdd3c1949f5 100644
--- a/python/paddle/fluid/tests/unittests/test_state_dict_convert.py
+++ b/python/paddle/fluid/tests/unittests/test_state_dict_convert.py
@@ -88,7 +88,7 @@ class TestStateDictReturn(unittest.TestCase):
     def test_missing_keys_and_unexpected_keys(self):
         model1 = MyModel2()
         tmp_dict = {}
-        tmp_dict["unexpected_keys"] = paddle.to_tensor(1)
+        tmp_dict["unexpected_keys"] = paddle.to_tensor([1])
         missing_keys, unexpected_keys = model1.set_state_dict(tmp_dict)
         self.assertEqual(len(missing_keys), 2)
         self.assertEqual(missing_keys[0], "linear.weight")
diff --git a/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py b/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py
index 02934d07a8924..16b6d32ce404d 100644
--- a/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py
+++ b/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py
@@ -553,7 +553,7 @@ def test_register_backward_hook(self):
         global HOOK_INIT_VALUE
         global HOOK_IS_CALLED
         for device in self.devices:
-            x = paddle.to_tensor(5.0, stop_gradient=False)
+            x = paddle.to_tensor([5.0], stop_gradient=False)
             x._register_backward_hook(global_void_hook)
             for i in range(5):
                 y = paddle.pow(x, 4.0)
@@ -567,14 +567,14 @@ def test_register_backward_hook(self):
             HOOK_IS_CALLED = False
 
     def test_register_backward_hook_for_interior_var(self):
-        x = paddle.to_tensor(5.0, stop_gradient=False)
+        x = paddle.to_tensor([5.0], stop_gradient=False)
         y = paddle.pow(x, 4.0)
 
         with self.assertRaises(ValueError):
             y._register_backward_hook(global_void_hook)
 
     def test_register_backward_hook_for_var_without_gradient(self):
-        x = paddle.to_tensor(5.0)
+        x = paddle.to_tensor([5.0])
         y = paddle.pow(x, 4.0)
 
         with self.assertRaises(ValueError):
diff --git a/python/paddle/fluid/tests/unittests/test_var_base.py b/python/paddle/fluid/tests/unittests/test_var_base.py
index b533dc2b6e588..9d489c77374c2 100644
--- a/python/paddle/fluid/tests/unittests/test_var_base.py
+++ b/python/paddle/fluid/tests/unittests/test_var_base.py
@@ -114,7 +114,7 @@ def check_with_place(place):
                 )
                 np.testing.assert_array_equal(x.numpy(), [1.0])
                 self.assertEqual(x.dtype, core.VarDesc.VarType.FP32)
-                self.assertEqual(x.shape, [1])
+                self.assertEqual(x.shape, [])
                 self.assertEqual(x.stop_gradient, False)
                 self.assertEqual(x.type, core.VarDesc.VarType.LOD_TENSOR)
 
@@ -407,7 +407,7 @@ def test_leaf_tensor(self):
 
     def test_detach(self):
         with fluid.dygraph.guard():
-            x = paddle.to_tensor(1.0, dtype="float64", stop_gradient=False)
+            x = paddle.to_tensor([1.0], dtype="float64", stop_gradient=False)
             detach_x = x.detach()
             self.assertTrue(detach_x.stop_gradient, True)
 
diff --git a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
index d18f94e78da39..7ea98f7c889a3 100644
--- a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
+++ b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
@@ -2385,6 +2385,20 @@ def body(i, x):
         self.assertEqual(x.grad.shape, [])
         np.testing.assert_allclose(x.grad, np.array(1.0))
 
+    def test_to_tensor(self):
+        out1 = paddle.to_tensor(1)
+        out2 = paddle.to_tensor(2.5)
+
+        out1.retain_grads()
+        out1.backward()
+        out2.retain_grads()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out1, 1)
+        self.assertEqual(out2.shape, [])
+        self.assertEqual(out2, 2.5)
+
     def test_linalg_slogdet(self):
         # 2-D input
         x = paddle.randn([3, 3])
@@ -4355,6 +4369,19 @@ def test_broadcast_tensors(self):
         self.assertEqual(out1.shape, (2, 3))
         self.assertEqual(out2.shape, (2, 3))
 
+    @prog_scope()
+    def test_to_tensor(self):
+        out1 = paddle.to_tensor(1)
+        out2 = paddle.to_tensor(2.5)
+
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out1, out2])
+
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[0], 1)
+        self.assertEqual(res[1].shape, ())
+        self.assertEqual(res[1], 2.5)
+
     @prog_scope()
     def test_linalg_slogdet(self):
         # 2-D input
diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py
index d9b22ac045f8d..186eda03e74d8 100644
--- a/python/paddle/tensor/creation.py
+++ b/python/paddle/tensor/creation.py
@@ -555,7 +555,7 @@ def _handle_dtype(data, dtype):
             return data
 
         if np.isscalar(data) and not isinstance(data, str):
-            data = np.array([data])
+            data = np.array(data)
         elif isinstance(data, (list, tuple)):
             data = np.array(data)
             if data.dtype == np.object_:
@@ -649,7 +649,7 @@ def _to_tensor_static(data, dtype=None, stop_gradient=None):
 
         if not isinstance(data, np.ndarray):
             if np.isscalar(data) and not isinstance(data, str):
-                data = np.array([data])
+                data = np.array(data)
             elif isinstance(data, (list, tuple)):
                 data = np.array(data)
 
@@ -677,12 +677,6 @@ def _to_tensor_static(data, dtype=None, stop_gradient=None):
             and len(data.shape) > 0
             and any(isinstance(x, Variable) for x in data)
         ):
-            if not all(
-                [x.shape == (1,) for x in data if isinstance(x, Variable)]
-            ):
-                raise TypeError(
-                    "Unsupport paddle.to_tensor([Variable, Variable...]) with non-scalar variable."
-                )
             to_stack_list = [None] * data.shape[0]
             for idx, d in enumerate(data):
                 to_stack_list[idx] = _to_tensor_static(d, dtype, stop_gradient)
@@ -717,7 +711,7 @@ def to_tensor(data, dtype=None, place=None, stop_gradient=True):
                                 (0D-Tensor)
                     default_dtype
         Python Number ───────────────► paddle.Tensor
-                                        (1D-Tensor)
+                                        (0D-Tensor)
                     Keep dtype
         np.ndarray ───────────► paddle.Tensor
 
@@ -746,17 +740,17 @@ def to_tensor(data, dtype=None, place=None, stop_gradient=True):
         # <class 'paddle.Tensor'>
 
         paddle.to_tensor(1)
-        # Tensor(shape=[1], dtype=int64, place=CPUPlace, stop_gradient=True,
-        #        [1])
+        # Tensor(shape=[], dtype=int64, place=CPUPlace, stop_gradient=True,
+        #        1)
 
         x = paddle.to_tensor(1, stop_gradient=False)
         print(x)
-        # Tensor(shape=[1], dtype=int64, place=CPUPlace, stop_gradient=False,
-        #        [1])
+        # Tensor(shape=[], dtype=int64, place=CPUPlace, stop_gradient=False,
+        #        1)
 
         paddle.to_tensor(x)  # A new tensor will be created with default stop_gradient=True
-        # Tensor(shape=[1], dtype=int64, place=CPUPlace, stop_gradient=True,
-        #        [1])
+        # Tensor(shape=[], dtype=int64, place=CPUPlace, stop_gradient=True,
+        #        1)
 
         paddle.to_tensor([[0.1, 0.2], [0.3, 0.4]], place=paddle.CPUPlace(), stop_gradient=False)
         # Tensor(shape=[2, 2], dtype=float32, place=CPUPlace, stop_gradient=False,
diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py
index efa8cbfc54e02..6b07f57e33a17 100644
--- a/python/paddle/tensor/math.py
+++ b/python/paddle/tensor/math.py
@@ -4421,8 +4421,8 @@ def rad2deg(x, name=None):
             x3 = paddle.to_tensor(1)
             result3 = paddle.rad2deg(x3)
             print(result3)
-            # Tensor(shape=[1], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
-            #         [57.29578018])
+            # Tensor(shape=[], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
+            #        57.29578018)
     """
     rad2deg_scale = 180 / np.pi
     if in_dygraph_mode():
@@ -4485,8 +4485,8 @@ def deg2rad(x, name=None):
             x2 = paddle.to_tensor(180)
             result2 = paddle.deg2rad(x2)
             print(result2)
-            # Tensor(shape=[1], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
-            #         [3.14159274])
+            # Tensor(shape=[], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
+            #        3.14159274)
     """
     deg2rad_scale = np.pi / 180.0
     if in_dygraph_mode():
@@ -4545,8 +4545,8 @@ def gcd(x, y, name=None):
             x1 = paddle.to_tensor(12)
             x2 = paddle.to_tensor(20)
             paddle.gcd(x1, x2)
-            # Tensor(shape=[1], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
-            #        [4])
+            # Tensor(shape=[], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
+            #        4)
 
             x3 = paddle.arange(6)
             paddle.gcd(x3, x2)
@@ -4555,17 +4555,17 @@ def gcd(x, y, name=None):
 
             x4 = paddle.to_tensor(0)
             paddle.gcd(x4, x2)
-            # Tensor(shape=[1], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
-            #        [20])
+            # Tensor(shape=[], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
+            #        20)
 
             paddle.gcd(x4, x4)
-            # Tensor(shape=[1], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
-            #        [0])
+            # Tensor(shape=[], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
+            #        0)
 
             x5 = paddle.to_tensor(-20)
             paddle.gcd(x1, x5)
-            # Tensor(shape=[1], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
-            #        [4])
+            # Tensor(shape=[], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
+            #        4)
     """
     shape = paddle.broadcast_shape(x.shape, y.shape)
     x = paddle.broadcast_to(x, shape)
@@ -4630,8 +4630,8 @@ def lcm(x, y, name=None):
             x1 = paddle.to_tensor(12)
             x2 = paddle.to_tensor(20)
             paddle.lcm(x1, x2)
-            # Tensor(shape=[1], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
-            #        [60])
+            # Tensor(shape=[], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
+            #        60)
 
             x3 = paddle.arange(6)
             paddle.lcm(x3, x2)
@@ -4640,17 +4640,17 @@ def lcm(x, y, name=None):
 
             x4 = paddle.to_tensor(0)
             paddle.lcm(x4, x2)
-            # Tensor(shape=[1], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
-            #        [0])
+            # Tensor(shape=[], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
+            #        0)
 
             paddle.lcm(x4, x4)
-            # Tensor(shape=[1], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
-            #        [0])
+            # Tensor(shape=[], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
+            #        0)
 
             x5 = paddle.to_tensor(-20)
             paddle.lcm(x1, x5)
-            # Tensor(shape=[1], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
-            #        [60])
+            # Tensor(shape=[], dtype=int64, place=CUDAPlace(0), stop_gradient=True,
+            #        60)
     """
     d = paddle.gcd(x, y)
     # paddle.mod will raise an error when any element of y is 0. To avoid
diff --git a/test/auto_parallel/random_control_unittest.py b/test/auto_parallel/random_control_unittest.py
index 52e6e216074fd..f55b57b9f1b1e 100644
--- a/test/auto_parallel/random_control_unittest.py
+++ b/test/auto_parallel/random_control_unittest.py
@@ -82,7 +82,7 @@ def compare_mask_between_ranks(
     ):
 
         for np_mask in [mask_np_list[i] for i in comapre_idx]:
-            mask_tensor_local = paddle.to_tensor(np_mask.astype("float32"))
+            mask_tensor_local = paddle.to_tensor([np_mask.astype("float32")])
             if rank == 0:
                 mask_tensor_remote = paddle.ones_like(mask_tensor_local)
                 dy_broadcast_helper(mask_tensor_remote)
diff --git a/test/distribution/test_distribution_bernoulli.py b/test/distribution/test_distribution_bernoulli.py
index 2229880b7a6bf..490bd9aa54d7a 100644
--- a/test/distribution/test_distribution_bernoulli.py
+++ b/test/distribution/test_distribution_bernoulli.py
@@ -184,10 +184,10 @@ def init_dynamic_data(self, probs, default_dtype, dtype):
         ('probs_00', 0.0, 'float64', 'float32'),
         ('probs_03', 0.3, 'float64', 'float32'),
         ('probs_10', 1.0, 'float64', 'float32'),
-        ('probs_tensor_03_32', paddle.to_tensor(0.3), 'float32', 'float32'),
+        ('probs_tensor_03_32', paddle.to_tensor([0.3]), 'float32', 'float32'),
         (
             'probs_tensor_03_64',
-            paddle.to_tensor(0.3, dtype='float64'),
+            paddle.to_tensor([0.3], dtype='float64'),
             'float64',
             'float64',
         ),
@@ -257,11 +257,11 @@ def test_variance(self):
             ),
             (
                 paddle.to_tensor(
-                    0.0,
+                    [0.0],
                 ),
             ),
-            (paddle.to_tensor(1.0),),
-            (paddle.to_tensor(0.0, dtype='float64'),),
+            (paddle.to_tensor([1.0]),),
+            (paddle.to_tensor([0.0], dtype='float64'),),
         ]
     )
     def test_log_prob(self, value):
@@ -291,9 +291,9 @@ def test_log_prob(self, value):
                     ]
                 ),
             ),
-            (paddle.to_tensor(0.0),),
-            (paddle.to_tensor(1.0),),
-            (paddle.to_tensor(0.0, dtype='float64'),),
+            (paddle.to_tensor([0.0]),),
+            (paddle.to_tensor([1.0]),),
+            (paddle.to_tensor([0.0], dtype='float64'),),
         ]
     )
     def test_prob(self, value):
@@ -323,11 +323,11 @@ def test_prob(self, value):
                     ]
                 ),
             ),
-            (paddle.to_tensor(0.0),),
-            (paddle.to_tensor(0.3),),
-            (paddle.to_tensor(0.7),),
-            (paddle.to_tensor(1.0),),
-            (paddle.to_tensor(0.0, dtype='float64'),),
+            (paddle.to_tensor([0.0]),),
+            (paddle.to_tensor([0.3]),),
+            (paddle.to_tensor([0.7]),),
+            (paddle.to_tensor([1.0]),),
+            (paddle.to_tensor([0.0], dtype='float64'),),
         ]
     )
     def test_cdf(self, value):
@@ -359,7 +359,7 @@ def test_entropy(self):
 
     def test_kl_divergence(self):
         with paddle.fluid.dygraph.guard(self.place):
-            other_probs = paddle.to_tensor(0.9, dtype=self.dtype)
+            other_probs = paddle.to_tensor([0.9], dtype=self.dtype)
 
             rv_paddle_other = Bernoulli(other_probs)
             rv_np_other = BernoulliNumpy(other_probs)
@@ -422,7 +422,7 @@ def test_kl_divergence(self):
         # 1-D probs
         (
             'probs_1d_1d_32',
-            paddle.to_tensor(0.3),
+            paddle.to_tensor([0.3]),
             'float32',
             'float32',
             [
@@ -432,7 +432,7 @@ def test_kl_divergence(self):
         ),
         (
             'probs_1d_1d_64',
-            paddle.to_tensor(0.3, dtype='float64'),
+            paddle.to_tensor([0.3], dtype='float64'),
             'float64',
             'float64',
             paddle.to_tensor(
@@ -444,7 +444,7 @@ def test_kl_divergence(self):
         ),
         (
             'probs_1d_2d',
-            paddle.to_tensor(0.3),
+            paddle.to_tensor([0.3]),
             'float32',
             'float32',
             [100, 2],
@@ -452,7 +452,7 @@ def test_kl_divergence(self):
         ),
         (
             'probs_1d_3d',
-            paddle.to_tensor(0.3),
+            paddle.to_tensor([0.3]),
             'float32',
             'float32',
             [100, 2, 3],
diff --git a/test/distribution/test_distribution_transform.py b/test/distribution/test_distribution_transform.py
index 4bc90d8f792e2..63ecd99d77baa 100644
--- a/test/distribution/test_distribution_transform.py
+++ b/test/distribution/test_distribution_transform.py
@@ -523,7 +523,7 @@ def test_codomain(self, input, expected):
                 transform.ChainTransform(
                     (
                         transform.AffineTransform(
-                            paddle.to_tensor(0.0), paddle.to_tensor(1.0)
+                            paddle.to_tensor([0.0]), paddle.to_tensor([1.0])
                         ),
                         transform.ExpTransform(),
                     )
@@ -560,7 +560,7 @@ def test_forward(self, chain, input, expected):
                 transform.ChainTransform(
                     (
                         transform.AffineTransform(
-                            paddle.to_tensor(0.0), paddle.to_tensor(-1.0)
+                            paddle.to_tensor([0.0]), paddle.to_tensor([-1.0])
                         ),
                         transform.ExpTransform(),
                     )
@@ -595,9 +595,9 @@ def test_inverse(self, chain, input, expected):
                 transform.ChainTransform(
                     (
                         transform.AffineTransform(
-                            paddle.to_tensor(0.0), paddle.to_tensor(-1.0)
+                            paddle.to_tensor([0.0]), paddle.to_tensor([-1.0])
                         ),
-                        transform.PowerTransform(paddle.to_tensor(2.0)),
+                        transform.PowerTransform(paddle.to_tensor([2.0])),
                     )
                 ),
                 np.array([1.0, 2.0, 3.0]),
@@ -619,7 +619,7 @@ def test_forward_log_det_jacobian(self, chain, input, expected):
                 transform.ChainTransform(
                     (
                         transform.AffineTransform(
-                            paddle.to_tensor(0.0), paddle.to_tensor(-1.0)
+                            paddle.to_tensor([0.0]), paddle.to_tensor([-1.0])
                         ),
                         transform.ExpTransform(),
                     )
@@ -638,7 +638,7 @@ def test_forward_shape(self, chain, shape, expected_shape):
                 transform.ChainTransform(
                     (
                         transform.AffineTransform(
-                            paddle.to_tensor(0.0), paddle.to_tensor(-1.0)
+                            paddle.to_tensor([0.0]), paddle.to_tensor([-1.0])
                         ),
                         transform.ExpTransform(),
                     )
@@ -743,7 +743,7 @@ def test_inverse_shape(self, shape, expected_shape):
 @param.place(config.DEVICES)
 class TestPowerTransform(unittest.TestCase):
     def setUp(self):
-        self._t = transform.PowerTransform(paddle.to_tensor(2.0))
+        self._t = transform.PowerTransform(paddle.to_tensor([2.0]))
 
     def test_init(self):
         with self.assertRaises(TypeError):
diff --git a/test/distribution/test_distribution_transformed_distribution.py b/test/distribution/test_distribution_transformed_distribution.py
index 57264b5f8972a..09b5bce4ecc0a 100644
--- a/test/distribution/test_distribution_transformed_distribution.py
+++ b/test/distribution/test_distribution_transformed_distribution.py
@@ -41,7 +41,7 @@ def _np_sum_rightmost(self, value, n):
         return np.sum(value, tuple(range(-n, 0))) if n > 0 else value
 
     def test_log_prob(self):
-        value = paddle.to_tensor(0.5)
+        value = paddle.to_tensor([0.5])
         np.testing.assert_allclose(
             self.simple_log_prob(value, self.base, self.transforms),
             self._t.log_prob(value),
diff --git a/test/distribution/test_kl.py b/test/distribution/test_kl.py
index e4b5c51fa75a4..d75f17208a8b9 100644
--- a/test/distribution/test_kl.py
+++ b/test/distribution/test_kl.py
@@ -139,8 +139,8 @@ def test_dispatch_with_unregister(self):
         ),
         (
             'test-same-dist',
-            mock.Exponential(paddle.to_tensor(1.0)),
-            mock.Exponential(paddle.to_tensor(1.0)),
+            mock.Exponential(paddle.to_tensor([1.0])),
+            mock.Exponential(paddle.to_tensor([1.0])),
         ),
     ],
 )
diff --git a/test/dygraph_to_static/test_cpu_cuda_to_tensor.py b/test/dygraph_to_static/test_cpu_cuda_to_tensor.py
index 24f0bd84556a2..f8d15971a7bc0 100644
--- a/test/dygraph_to_static/test_cpu_cuda_to_tensor.py
+++ b/test/dygraph_to_static/test_cpu_cuda_to_tensor.py
@@ -35,8 +35,8 @@ def func(x):
 class TestToTensor(unittest.TestCase):
     def test_to_tensor_with_variable_list(self):
         def func(x):
-            ones = paddle.to_tensor([1])
-            twos = paddle.to_tensor([2])
+            ones = paddle.to_tensor(1)
+            twos = paddle.to_tensor(2)
             x = paddle.to_tensor([ones, twos, 3, 4])
             return x
 
diff --git a/test/dygraph_to_static/test_fallback.py b/test/dygraph_to_static/test_fallback.py
index 6da8602e6043f..e4dc0114054ad 100644
--- a/test/dygraph_to_static/test_fallback.py
+++ b/test/dygraph_to_static/test_fallback.py
@@ -52,7 +52,7 @@ def forward(self, x):
 
 class TestFallback(unittest.TestCase):
     def setUp(self):
-        self.x = paddle.to_tensor(2).astype('int')
+        self.x = paddle.to_tensor([2]).astype('int')
 
     def tearDown(self):
         pass
diff --git a/test/dygraph_to_static/test_to_tensor.py b/test/dygraph_to_static/test_to_tensor.py
index b0131263c4e69..05cd5ec78f2c8 100644
--- a/test/dygraph_to_static/test_to_tensor.py
+++ b/test/dygraph_to_static/test_to_tensor.py
@@ -84,6 +84,12 @@ def case6(x):
     return a
 
 
+def case7(x):
+    a = paddle.to_tensor(10.0)
+
+    return a
+
+
 class TestToTensorReturnVal(unittest.TestCase):
     def test_to_tensor_badreturn(self):
         paddle.disable_static()
@@ -131,6 +137,12 @@ def test_to_tensor_badreturn(self):
         self.assertTrue(a.stop_gradient == b.stop_gradient)
         self.assertTrue(a.place._equals(b.place))
 
+        a = paddle.jit.to_static(case7)(x)
+        b = case7(x)
+        self.assertTrue(a.dtype == b.dtype)
+        self.assertTrue(a.stop_gradient == b.stop_gradient)
+        self.assertTrue(a.place._equals(b.place))
+
 
 class TestStatic(unittest.TestCase):
     def test_static(self):
diff --git a/test/legacy_test/auto_parallel_gpt_model.py b/test/legacy_test/auto_parallel_gpt_model.py
index 5aef64e412ad1..7cf48cf1c475b 100644
--- a/test/legacy_test/auto_parallel_gpt_model.py
+++ b/test/legacy_test/auto_parallel_gpt_model.py
@@ -233,7 +233,7 @@ def core_attn(self, q, k, v, attn_mask):
         product = paddle.matmul(x=q, y=k, transpose_y=True)
         product = paddle.multiply(
             product,
-            paddle.to_tensor(self.head_dim**-0.5, dtype=product.dtype),
+            paddle.to_tensor([self.head_dim**-0.5], dtype=product.dtype),
         )
         if attn_mask is not None:
             product = product + attn_mask
diff --git a/test/legacy_test/test_audio_functions.py b/test/legacy_test/test_audio_functions.py
index 8400bd4ecb40e..47adbdd490501 100644
--- a/test/legacy_test/test_audio_functions.py
+++ b/test/legacy_test/test_audio_functions.py
@@ -60,7 +60,7 @@ def get_wav_data(dtype: str, num_channels: int, num_frames: int):
     def test_audio_function(self, val: float, htk_flag: bool):
         mel_paddle = paddle.audio.functional.hz_to_mel(val, htk_flag)
         mel_paddle_tensor = paddle.audio.functional.hz_to_mel(
-            paddle.to_tensor(val), htk_flag
+            paddle.to_tensor([val]), htk_flag
         )
         mel_librosa = librosa.hz_to_mel(val, htk_flag)
         np.testing.assert_almost_equal(mel_paddle, mel_librosa, decimal=5)
@@ -70,7 +70,7 @@ def test_audio_function(self, val: float, htk_flag: bool):
 
         hz_paddle = paddle.audio.functional.mel_to_hz(val, htk_flag)
         hz_paddle_tensor = paddle.audio.functional.mel_to_hz(
-            paddle.to_tensor(val), htk_flag
+            paddle.to_tensor([val]), htk_flag
         )
         hz_librosa = librosa.mel_to_hz(val, htk_flag)
         np.testing.assert_almost_equal(hz_paddle, hz_librosa, decimal=4)
@@ -79,7 +79,7 @@ def test_audio_function(self, val: float, htk_flag: bool):
         )
 
         decibel_paddle = paddle.audio.functional.power_to_db(
-            paddle.to_tensor(val)
+            paddle.to_tensor([val])
         )
         decibel_librosa = librosa.power_to_db(val)
         np.testing.assert_almost_equal(
diff --git a/test/quantization/imperative_test_utils.py b/test/quantization/imperative_test_utils.py
index 9e9c2cca447a5..36e931091543f 100644
--- a/test/quantization/imperative_test_utils.py
+++ b/test/quantization/imperative_test_utils.py
@@ -165,7 +165,7 @@ def forward(self, inputs):
         x = self.features(x)
 
         x = paddle.flatten(x, 1)
-        x = self.add(x, paddle.to_tensor(0.0))  # For CI
+        x = self.add(x, paddle.to_tensor([0.0]))  # For CI
         x = self.fc(x)
         return x
 

From b06ec0c759cf79aa92d6124ac6fcde315785574f Mon Sep 17 00:00:00 2001
From: HongyuJia <jiahongyu@baidu.com>
Date: Mon, 24 Apr 2023 16:02:21 +0800
Subject: [PATCH 015/405] [CppExtension Unittest] Add unit test of
 vector<Tensor> (#53040)

---
 .../tests/cpp_extension/custom_extension.cc     | 11 +++++++++++
 .../cpp_extension/test_cpp_extension_jit.py     | 15 +++++++++++++++
 .../cpp_extension/test_cpp_extension_setup.py   | 17 +++++++++++++++++
 3 files changed, 43 insertions(+)

diff --git a/python/paddle/fluid/tests/cpp_extension/custom_extension.cc b/python/paddle/fluid/tests/cpp_extension/custom_extension.cc
index 2fc5c42a80d75..67653bffb3bdb 100644
--- a/python/paddle/fluid/tests/cpp_extension/custom_extension.cc
+++ b/python/paddle/fluid/tests/cpp_extension/custom_extension.cc
@@ -26,6 +26,16 @@ paddle::Tensor custom_add(const paddle::Tensor& x, const paddle::Tensor& y) {
   return x.exp() + y.exp();
 }
 
+std::vector<paddle::Tensor> custom_tensor(
+    const std::vector<paddle::Tensor>& inputs) {
+  std::vector<paddle::Tensor> out;
+  out.reserve(inputs.size());
+  for (const auto& input : inputs) {
+    out.push_back(input + 1.0);
+  }
+  return out;
+}
+
 paddle::Tensor nullable_tensor(bool return_none = false) {
   paddle::Tensor t;
   if (!return_none) {
@@ -45,6 +55,7 @@ paddle::optional<paddle::Tensor> optional_tensor(bool return_option = false) {
 PYBIND11_MODULE(custom_cpp_extension, m) {
   m.def("custom_add", &custom_add, "exp(x) + exp(y)");
   m.def("custom_sub", &custom_sub, "exp(x) - exp(y)");
+  m.def("custom_tensor", &custom_tensor, "x + 1");
   m.def("nullable_tensor", &nullable_tensor, "returned Tensor might be None");
   m.def(
       "optional_tensor", &optional_tensor, "returned Tensor might be optional");
diff --git a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py b/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py
index bc6f8113afd91..5723df1b585b0 100644
--- a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py
+++ b/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py
@@ -68,6 +68,7 @@ def tearDown(self):
     def test_cpp_extension(self):
         self._test_extension_function()
         self._test_extension_class()
+        self._test_vector_tensor()
         self._test_nullable_tensor()
         self._test_optional_tensor()
         if paddle.is_compiled_with_cuda():
@@ -109,6 +110,20 @@ def _test_extension_class(self):
                 atol=1e-5,
             )
 
+    def _test_vector_tensor(self):
+        for dtype in self.dtypes:
+            np_inputs = [
+                np.random.uniform(-1, 1, [4, 8]).astype(dtype) for _ in range(3)
+            ]
+            inputs = [paddle.to_tensor(np_x, dtype=dtype) for np_x in np_inputs]
+
+            out = custom_cpp_extension.custom_tensor(inputs)
+            target_out = [x + 1.0 for x in inputs]
+            for i in range(3):
+                np.testing.assert_allclose(
+                    out[i].numpy(), target_out[i].numpy(), atol=1e-5
+                )
+
     def _test_nullable_tensor(self):
         x = custom_cpp_extension.nullable_tensor(True)
         assert x is None, "Return None when input parameter return_none = True"
diff --git a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py b/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py
index 53dffde432095..15d0cb77d03d5 100644
--- a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py
+++ b/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py
@@ -148,6 +148,7 @@ def test_cpp_extension(self):
         # Extension
         self._test_extension_function_plain()
         self._test_extension_function_mixed()
+        self._test_vector_tensor()
         self._test_extension_class()
         self._test_nullable_tensor()
         self._test_optional_tensor()
@@ -218,6 +219,22 @@ def _test_extension_class(self):
                 atol=1e-5,
             )
 
+    def _test_vector_tensor(self):
+        import custom_cpp_extension
+
+        for dtype in self.dtypes:
+            np_inputs = [
+                np.random.uniform(-1, 1, [4, 8]).astype(dtype) for _ in range(3)
+            ]
+            inputs = [paddle.to_tensor(np_x, dtype=dtype) for np_x in np_inputs]
+
+            out = custom_cpp_extension.custom_tensor(inputs)
+            target_out = [x + 1 for x in inputs]
+            for i in range(3):
+                np.testing.assert_allclose(
+                    out[i].numpy(), target_out[i].numpy(), atol=1e-5
+                )
+
     def _test_nullable_tensor(self):
         import custom_cpp_extension
 

From 987fb2d88e8731544a777342f75ffcc2cdcc0359 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?=
 <83450930+Liyulingyue@users.noreply.github.com>
Date: Mon, 24 Apr 2023 16:11:59 +0800
Subject: [PATCH 016/405] rm mlu (#53194)

---
 paddle/fluid/imperative/amp_auto_cast.cc                  | 4 ++--
 paddle/fluid/operators/collective/c_comm_init_op.cc       | 4 ++--
 .../fluid/operators/generator/get_expected_kernel_func.cc | 2 +-
 paddle/fluid/operators/reduce_ops/reduce_op.h             | 2 +-
 paddle/fluid/operators/softmax_op.cc                      | 4 ++--
 paddle/fluid/platform/device_context.h                    | 1 -
 paddle/fluid/platform/profiler/dump/nodetree.proto        | 2 --
 paddle/fluid/platform/profiler/profiler.h                 | 2 +-
 paddle/fluid/pybind/imperative.cc                         | 7 +++----
 paddle/phi/api/profiler/trace_event.h                     | 2 --
 paddle/phi/common/backend.h                               | 1 -
 paddle/phi/common/place.h                                 | 1 -
 python/paddle/amp/auto_cast.py                            | 2 +-
 python/paddle/amp/grad_scaler.py                          | 2 +-
 python/paddle/distributed/launch/main.py                  | 2 +-
 python/paddle/distributed/spawn.py                        | 8 +++-----
 python/paddle/profiler/profiler.py                        | 8 ++------
 17 files changed, 20 insertions(+), 34 deletions(-)

diff --git a/paddle/fluid/imperative/amp_auto_cast.cc b/paddle/fluid/imperative/amp_auto_cast.cc
index 2689a4eafa442..bf6bdf382ce44 100644
--- a/paddle/fluid/imperative/amp_auto_cast.cc
+++ b/paddle/fluid/imperative/amp_auto_cast.cc
@@ -57,7 +57,7 @@ OpSupportedInfos(const std::string& place,
                     0,
                     platform::errors::InvalidArgument(
                         "The argument `place` should be 'GPU', 'CPU', 'XPU', "
-                        "'NPU', 'MLU', but got '%s'.",
+                        "'NPU', but got '%s'.",
                         place));
 
   std::unordered_set<std::string> all_ops;
@@ -148,7 +148,7 @@ AmpOperators::AmpOperators()
       OpSupportedInfos("GPU", paddle::framework::proto::VarType::BF16));
   unsupported_bf16_ops_->insert(unsupported_ops_gpu_bf16.begin(),
                                 unsupported_ops_gpu_bf16.end());
-// NOTE: GPU/NPU/XPU/MLU is compiled seperatly.
+// NOTE: GPU/NPU/XPU is compiled seperatly.
 #elif defined(PADDLE_WITH_XPU)
   auto unsupported_ops_xpu_fp16 = std::get<2>(
       OpSupportedInfos("XPU", paddle::framework::proto::VarType::FP16));
diff --git a/paddle/fluid/operators/collective/c_comm_init_op.cc b/paddle/fluid/operators/collective/c_comm_init_op.cc
index 9a34a3a2f3779..b32857a27b2d2 100644
--- a/paddle/fluid/operators/collective/c_comm_init_op.cc
+++ b/paddle/fluid/operators/collective/c_comm_init_op.cc
@@ -57,14 +57,14 @@ class CCommInitOp : public framework::OperatorBase {
     using CommContext = platform::BKCLCommContext;
 #else
     PADDLE_THROW(platform::errors::PreconditionNotMet(
-        "PaddlePaddle should be compiled with GPU or XPU or MLU."));
+        "PaddlePaddle should be compiled with GPU or XPU."));
 #endif
 
     PADDLE_ENFORCE_EQ(
         platform::is_gpu_place(place) || platform::is_xpu_place(place),
         true,
         platform::errors::PreconditionNotMet(
-            "CCommInitOp can run on gpu or xpu or mlu place only."));
+            "CCommInitOp can run on gpu or xpu place only."));
 
 #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) || \
     defined(PADDLE_WITH_XPU_BKCL)
diff --git a/paddle/fluid/operators/generator/get_expected_kernel_func.cc b/paddle/fluid/operators/generator/get_expected_kernel_func.cc
index 931bbc1fecc7d..558b0c400a4ca 100644
--- a/paddle/fluid/operators/generator/get_expected_kernel_func.cc
+++ b/paddle/fluid/operators/generator/get_expected_kernel_func.cc
@@ -80,7 +80,7 @@ phi::KernelKey GetReduceExpectedKernelType(
             platform::is_custom_place(ctx.GetPlace()),
         true,
         platform::errors::InvalidArgument(
-            "float16 can only be used on GPU or NPU or MLU or XPU place"));
+            "float16 can only be used on GPU or NPU or XPU place"));
   }
   return phi::KernelKey(input_data_type, ctx.GetPlace());
 }
diff --git a/paddle/fluid/operators/reduce_ops/reduce_op.h b/paddle/fluid/operators/reduce_ops/reduce_op.h
index 1a3925e4422d0..3c2b0b948bb22 100644
--- a/paddle/fluid/operators/reduce_ops/reduce_op.h
+++ b/paddle/fluid/operators/reduce_ops/reduce_op.h
@@ -629,7 +629,7 @@ class ReduceBaseOp : public framework::OperatorWithKernel {
               platform::is_custom_place(ctx.GetPlace()),
           true,
           platform::errors::InvalidArgument(
-              "float16 can only be used on GPU or NPU or MLU or XPU place"));
+              "float16 can only be used on GPU or NPU or XPU place"));
     }
     return phi::KernelKey(input_data_type, ctx.GetPlace());
   }
diff --git a/paddle/fluid/operators/softmax_op.cc b/paddle/fluid/operators/softmax_op.cc
index 4f440ec495d42..ab5816965f05c 100644
--- a/paddle/fluid/operators/softmax_op.cc
+++ b/paddle/fluid/operators/softmax_op.cc
@@ -48,7 +48,7 @@ class SoftmaxOp : public framework::OperatorWithKernel {
               platform::is_custom_place(ctx.GetPlace()),
           true,
           platform::errors::InvalidArgument(
-              "float16 can only be used on GPU/NPU/XPU/MLU and custom place"));
+              "float16 can only be used on GPU/NPU/XPU and custom place"));
     }
     return phi::KernelKey(
         ctx.GetPlace(), layout_, phi::TransToPhiDataType(input_data_type));
@@ -132,7 +132,7 @@ class SoftmaxOpGrad : public framework::OperatorWithKernel {
             platform::is_xpu_place(ctx.GetPlace()) ||
             platform::is_custom_place(ctx.GetPlace())))
         PADDLE_THROW(platform::errors::InvalidArgument(
-            "float16 can only be used on GPU/NPU/XPU/MLU and custom place"));
+            "float16 can only be used on GPU/NPU/XPU and custom place"));
     }
     return phi::KernelKey(
         ctx.GetPlace(), layout_, phi::TransToPhiDataType(input_data_type));
diff --git a/paddle/fluid/platform/device_context.h b/paddle/fluid/platform/device_context.h
index d2866a9aa1165..96ddef2c60efe 100644
--- a/paddle/fluid/platform/device_context.h
+++ b/paddle/fluid/platform/device_context.h
@@ -96,7 +96,6 @@ enum DeviceType {
   NPU = 2,
   XPU = 3,
   IPU = 4,
-  MLU = 5,
   CUSTOM_DEVICE = 6,
 
   MAX_DEVICE_TYPES = 7,
diff --git a/paddle/fluid/platform/profiler/dump/nodetree.proto b/paddle/fluid/platform/profiler/dump/nodetree.proto
index dc7b5046b079d..cfb424b020269 100644
--- a/paddle/fluid/platform/profiler/dump/nodetree.proto
+++ b/paddle/fluid/platform/profiler/dump/nodetree.proto
@@ -46,8 +46,6 @@ enum TracerEventTypeProto {
   PythonOp = 13;
   // Used to mark python level userdefined
   PythonUserDefined = 14;
-  // Used to mark mlu runtime record returned by cnpapi
-  MluRuntime = 15;
 };
 
 enum TracerMemEventTypeProto {
diff --git a/paddle/fluid/platform/profiler/profiler.h b/paddle/fluid/platform/profiler/profiler.h
index 2a7100b0a64ea..b486e7543d96c 100644
--- a/paddle/fluid/platform/profiler/profiler.h
+++ b/paddle/fluid/platform/profiler/profiler.h
@@ -39,7 +39,7 @@ static constexpr uint32_t kProfileCustomDeviceOptionBit = 3;
 void SynchronizeDevice();
 
 struct ProfilerOptions {
-  uint32_t trace_switch = 0;  // bit 0: cpu, bit 1: gpu, bit 2: mlu
+  uint32_t trace_switch = 0;  // bit 0: cpu, bit 1: gpu
   uint32_t trace_level = FLAGS_host_trace_level;
 };
 
diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc
index e60211286ed37..d6a5a8b8dfc87 100644
--- a/paddle/fluid/pybind/imperative.cc
+++ b/paddle/fluid/pybind/imperative.cc
@@ -155,7 +155,7 @@ static const platform::Place PyObjectToPlace(const py::object &place_obj) {
     PADDLE_THROW(platform::errors::InvalidArgument(
         "Place should be one of "
         "Place/CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/NPUPlace/IPUPlace/"
-        "MLUPlace/CustomPlace"));
+        "CustomPlace"));
   }
 }
 
@@ -209,8 +209,7 @@ static void InitVarBaseAndTensor(imperative::VarBase *self,
   } else {
     PADDLE_THROW(platform::errors::InvalidArgument(
         "Place should be one of "
-        "CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/NPUPlace/IPUPlace/"
-        "MLUPlace"));
+        "CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/NPUPlace/IPUPlace/"));
   }
   self->SetDataType(framework::TransToProtoVarType(tensor->dtype()));
 }
@@ -2214,7 +2213,7 @@ void BindImperative(py::module *m_ptr) {
             } else {
               PADDLE_THROW(platform::errors::InvalidArgument(
                   "Incompatible Place Type: supports XPUPlace, CUDAPlace, "
-                  "CPUPlace, NPUPlace, IPUPlace, MLUPlace"
+                  "CPUPlace, NPUPlace, IPUPlace"
                   "and CUDAPinnedPlace, "
                   "but got Unknown Type!"));
             }
diff --git a/paddle/phi/api/profiler/trace_event.h b/paddle/phi/api/profiler/trace_event.h
index 1b049a0f105cb..e526953d5c8e0 100644
--- a/paddle/phi/api/profiler/trace_event.h
+++ b/paddle/phi/api/profiler/trace_event.h
@@ -51,8 +51,6 @@ enum class TracerEventType {
   PythonOp = 13,
   // Used to mark python level userdefined
   PythonUserDefined = 14,
-  // Used to mark mlu runtime record returned by cnpapi
-  MluRuntime = 15,
   // A flag to denote the number of current types
   NumTypes
 };
diff --git a/paddle/phi/common/backend.h b/paddle/phi/common/backend.h
index 904038a4f5602..b7f30797ca78e 100644
--- a/paddle/phi/common/backend.h
+++ b/paddle/phi/common/backend.h
@@ -53,7 +53,6 @@ enum class Backend : uint8_t {
   // various acceleration devices' backends
   XPU,  // XPU currently does not exist at the same time as CUDA
   NPU,  // NPU currently does not exist at the same time as CUDA
-  MLU,  // MLU currently does not exist at the same time as CUDA
   IPU,
 
   // paddle kernel primitives backend
diff --git a/paddle/phi/common/place.h b/paddle/phi/common/place.h
index e65235cfa3d69..543a79977eb7e 100644
--- a/paddle/phi/common/place.h
+++ b/paddle/phi/common/place.h
@@ -34,7 +34,6 @@ enum class AllocationType : int8_t {
   NPU = 5,
   NPUPINNED = 6,
   IPU = 7,
-  MLU = 8,
   CUSTOM = 9,
 };
 
diff --git a/python/paddle/amp/auto_cast.py b/python/paddle/amp/auto_cast.py
index 1f82533edbfb3..ae9c957df68fa 100644
--- a/python/paddle/amp/auto_cast.py
+++ b/python/paddle/amp/auto_cast.py
@@ -349,7 +349,7 @@ def amp_guard(
         or tracer._expected_place.is_custom_place()
     ):
         warnings.warn(
-            'amp_guard can only be enabled on CUDAPlace, XPUPlace, MLUPlace, NPUPlace, and CustomPlace, current place is %s, so it makes no effect.'
+            'amp_guard can only be enabled on CUDAPlace, XPUPlace, NPUPlace, and CustomPlace, current place is %s, so it makes no effect.'
             % tracer._expected_place
         )
         enable = False
diff --git a/python/paddle/amp/grad_scaler.py b/python/paddle/amp/grad_scaler.py
index 5c2d033d33633..0f6d9f21a32c6 100644
--- a/python/paddle/amp/grad_scaler.py
+++ b/python/paddle/amp/grad_scaler.py
@@ -108,7 +108,7 @@ def __init__(
             or tracer._expected_place.is_custom_place()
         ):
             warnings.warn(
-                'AmpScaler can only be enabled on CUDAPlace, XPUPlace, MLUPlace and CustomPlace, current place is %s, so it makes no effect.'
+                'AmpScaler can only be enabled on CUDAPlace, XPUPlace and CustomPlace, current place is %s, so it makes no effect.'
                 % tracer._expected_place
             )
             enable = False
diff --git a/python/paddle/distributed/launch/main.py b/python/paddle/distributed/launch/main.py
index da113e72c35c5..02099c743933e 100644
--- a/python/paddle/distributed/launch/main.py
+++ b/python/paddle/distributed/launch/main.py
@@ -52,7 +52,7 @@ def launch():
 
         - ``--job_id``: The job unique id, it affects the log files' name. e.g., ``--job_id=job1``. Default ``--job_id=default``.
 
-        - ``--devices``: The selected accelerate devices on nodes, can be gpu/xpu/npu/mlu etc.. e.g., ``--devices=0,1,2,3`` will launch four training processes each bound to one device.
+        - ``--devices``: The selected accelerate devices on nodes, can be gpu/xpu/npu etc.. e.g., ``--devices=0,1,2,3`` will launch four training processes each bound to one device.
 
         - ``training_script``: The full path to the single GPU training program/script to be launched in parallel, followed by all the arguments for the training script. e.g., ``training.py``
 
diff --git a/python/paddle/distributed/spawn.py b/python/paddle/distributed/spawn.py
index 713ba7d118f0a..62e6eb0b14228 100644
--- a/python/paddle/distributed/spawn.py
+++ b/python/paddle/distributed/spawn.py
@@ -428,9 +428,9 @@ def spawn(func, args=(), nprocs=-1, join=True, daemon=False, **options):
     Start multiple processes with ``spawn`` method for parallel training.
 
     .. note::
-        ``spawn`` now only supports GPU or XPU or MLU collective mode. The collective mode
-        of GPU and XPU and MLU cannot be started at the same time, so the option `gpus` and
-        `xpus` and 'mlus' cannot be configured at the same time.
+        ``spawn`` now only supports GPU or XPU collective mode. The collective mode
+        of GPU and XPU cannot be started at the same time, so the option `gpus` and
+        `xpus` cannot be configured at the same time.
 
     Args:
         func (function): The target function is called by spawned process.
@@ -457,8 +457,6 @@ def spawn(func, args=(), nprocs=-1, join=True, daemon=False, **options):
             selected gpus, such as "0,1,2,3". Default: None;
             (3) xpus (string): The training process will run on the
             selected xpus, such as "0,1,2,3". Default: None;
-            (4) mlus (string): The training process will run on the
-            selected mlus, such as "0,1,2,3". Default: None;
             (5) ips (string): Paddle cluster nodes ips, such as
             "192.168.0.16,192.168.0.17". Default: "127.0.0.1" .
 
diff --git a/python/paddle/profiler/profiler.py b/python/paddle/profiler/profiler.py
index 570580a3d2cd4..065721a274750 100644
--- a/python/paddle/profiler/profiler.py
+++ b/python/paddle/profiler/profiler.py
@@ -98,19 +98,16 @@ class ProfilerState(Enum):
 
 class ProfilerTarget(Enum):
     r"""
-    ProfilerTarget is used to specify target device for :ref:`profiling <api_paddle_profiler_Profiler>` . Only CPU, GPU and MLU are supported currently.
+    ProfilerTarget is used to specify target device for :ref:`profiling <api_paddle_profiler_Profiler>` . Only CPU and GPU are supported currently.
 
     The meaning of each ProfilerState is as following
 
     - **ProfilerTarget.CPU** : Profile events on CPU.
 
     - **ProfilerTarget.GPU** : Profile events on GPU.
-
-    - **ProfilerTarget.MLU** : Profile events on MLU.
     """
     CPU = 0
     GPU = 1
-    MLU = 2
     CUSTOM_DEVICE = 3
 
 
@@ -335,7 +332,6 @@ def _get_supported_targets() -> Iterable[ProfilerTarget]:
     if _Profiler.is_cnpapi_supported():
         return [
             ProfilerTarget.CPU,
-            ProfilerTarget.MLU,
             ProfilerTarget.CUSTOM_DEVICE,
         ]
     return [ProfilerTarget.CPU, ProfilerTarget.CUSTOM_DEVICE]
@@ -346,7 +342,7 @@ class Profiler:
     Profiler context manager, user interface to manage profiling process to start, stop, export profiling data and print summary table.
 
     Args:
-        targets (list, optional): specify target devices to profile, and all existing and supported devices will be chosen by default. Currently supported values, :ref:`ProfilerTarget.CPU <api_paddle_profiler_ProfilerTarget>` , :ref:`ProfilerTarget.GPU <api_paddle_profiler_ProfilerTarget>` and :ref:`ProfilerTarget.MLU <api_paddle_profiler_ProfilerTarget>` .
+        targets (list, optional): specify target devices to profile, and all existing and supported devices will be chosen by default. Currently supported values, :ref:`ProfilerTarget.CPU <api_paddle_profiler_ProfilerTarget>` and :ref:`ProfilerTarget.GPU <api_paddle_profiler_ProfilerTarget>`  .
         scheduler (Callable|tuple, optional): If it is a callable object, it takes a step number as parameter and return the corresponding :ref:`ProfilerState <api_paddle_profiler_ProfilerState>`. This callable object can be generated by :ref:`make_scheduler <api_paddle_profiler_make_scheduler>` function.
             If not provided (None), the default scheduler will keep tracing until the profiler exits. If it is a tuple, it has two values start_batch and end_batch,
             which means profiling range [start_batch, end_batch).

From 6a8d98e0bbca6b9c45b75bf7fe156330c0a995f7 Mon Sep 17 00:00:00 2001
From: Siming Dai <908660116@qq.com>
Date: Mon, 24 Apr 2023 16:13:10 +0800
Subject: [PATCH 017/405] Add weighted sample (#52013)

Add paddle.geometric.weighted_sample_neighbors API
---
 paddle/phi/api/yaml/ops.yaml                  |   9 +
 paddle/phi/infermeta/multiary.cc              |  47 ++
 paddle/phi/infermeta/multiary.h               |  11 +
 .../cpu/weighted_sample_neighbors_kernel.cc   | 255 +++++++++
 paddle/phi/kernels/funcs/block_radix_topk.cuh | 349 ++++++++++++
 paddle/phi/kernels/funcs/random.cuh           |  80 +++
 .../gpu/weighted_sample_neighbors_kernel.cu   | 535 ++++++++++++++++++
 .../weighted_sample_neighbors_kernel.h        |  35 ++
 .../test_weighted_sample_neighbors.py         | 217 +++++++
 python/paddle/geometric/__init__.py           |   2 +
 python/paddle/geometric/sampling/__init__.py  |   1 +
 python/paddle/geometric/sampling/neighbors.py | 152 ++++-
 12 files changed, 1691 insertions(+), 2 deletions(-)
 create mode 100644 paddle/phi/kernels/cpu/weighted_sample_neighbors_kernel.cc
 create mode 100644 paddle/phi/kernels/funcs/block_radix_topk.cuh
 create mode 100644 paddle/phi/kernels/funcs/random.cuh
 create mode 100644 paddle/phi/kernels/gpu/weighted_sample_neighbors_kernel.cu
 create mode 100644 paddle/phi/kernels/weighted_sample_neighbors_kernel.h
 create mode 100644 python/paddle/fluid/tests/unittests/test_weighted_sample_neighbors.py

diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index b07bf0ecb99a5..1541d1890a07a 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -2144,6 +2144,15 @@
   intermediate: warprnntgrad
   backward : warprnnt_grad
 
+- op : weighted_sample_neighbors
+  args : (Tensor row, Tensor colptr, Tensor edge_weight, Tensor input_nodes, Tensor eids, int sample_size, bool return_eids)
+  output : Tensor(out_neighbors), Tensor(out_count), Tensor(out_eids)
+  infer_meta :
+    func : WeightedSampleNeighborsInferMeta
+  kernel :
+    func : weighted_sample_neighbors
+  optional: eids
+
 - op : where
   args : (Tensor condition, Tensor x, Tensor y)
   output : Tensor
diff --git a/paddle/phi/infermeta/multiary.cc b/paddle/phi/infermeta/multiary.cc
index 45769cdcb591f..5a8e38e21fd72 100644
--- a/paddle/phi/infermeta/multiary.cc
+++ b/paddle/phi/infermeta/multiary.cc
@@ -3249,5 +3249,52 @@ void MoeInferMeta(const MetaTensor& x,
   out->set_layout(x.layout());
 }
 
+void WeightedSampleNeighborsInferMeta(const MetaTensor& row,
+                                      const MetaTensor& col_ptr,
+                                      const MetaTensor& edge_weight,
+                                      const MetaTensor& x,
+                                      const MetaTensor& eids,
+                                      int sample_size,
+                                      bool return_eids,
+                                      MetaTensor* out,
+                                      MetaTensor* out_count,
+                                      MetaTensor* out_eids) {
+  // GSN: GraphSampleNeighbors
+  auto GSNShapeCheck = [](const phi::DDim& dims, std::string tensor_name) {
+    if (dims.size() == 2) {
+      PADDLE_ENFORCE_EQ(
+          dims[1],
+          1,
+          phi::errors::InvalidArgument("The last dim of %s should be 1 when it "
+                                       "is 2D, but we get %d",
+                                       tensor_name,
+                                       dims[1]));
+    } else {
+      PADDLE_ENFORCE_EQ(
+          dims.size(),
+          1,
+          phi::errors::InvalidArgument(
+              "The %s should be 1D, when it is not 2D, but we get %d",
+              tensor_name,
+              dims.size()));
+    }
+  };
+
+  GSNShapeCheck(row.dims(), "row");
+  GSNShapeCheck(col_ptr.dims(), "colptr");
+  GSNShapeCheck(edge_weight.dims(), "edge_weight");
+  GSNShapeCheck(x.dims(), "input_nodes");
+  if (return_eids) {
+    GSNShapeCheck(eids.dims(), "eids");
+    out_eids->set_dims({-1});
+    out_eids->set_dtype(row.dtype());
+  }
+
+  out->set_dims({-1});
+  out->set_dtype(row.dtype());
+  out_count->set_dims({-1});
+  out_count->set_dtype(DataType::INT32);
+}
+
 }  // namespace phi
 PD_REGISTER_INFER_META_FN(batch_norm_infer, phi::BatchNormInferInferMeta);
diff --git a/paddle/phi/infermeta/multiary.h b/paddle/phi/infermeta/multiary.h
index f094ea90d9a9d..993e6c21ff6ff 100644
--- a/paddle/phi/infermeta/multiary.h
+++ b/paddle/phi/infermeta/multiary.h
@@ -550,6 +550,17 @@ void WarprnntInferMeta(const MetaTensor& input,
                        MetaTensor* loss,
                        MetaTensor* warpctcgrad);
 
+void WeightedSampleNeighborsInferMeta(const MetaTensor& row,
+                                      const MetaTensor& col_ptr,
+                                      const MetaTensor& edge_weight,
+                                      const MetaTensor& x,
+                                      const MetaTensor& eids,
+                                      int sample_size,
+                                      bool return_eids,
+                                      MetaTensor* out,
+                                      MetaTensor* out_count,
+                                      MetaTensor* out_eids);
+
 void WhereInferMeta(const MetaTensor& condition,
                     const MetaTensor& x,
                     const MetaTensor& y,
diff --git a/paddle/phi/kernels/cpu/weighted_sample_neighbors_kernel.cc b/paddle/phi/kernels/cpu/weighted_sample_neighbors_kernel.cc
new file mode 100644
index 0000000000000..cc2b6cdbdf2fa
--- /dev/null
+++ b/paddle/phi/kernels/cpu/weighted_sample_neighbors_kernel.cc
@@ -0,0 +1,255 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/weighted_sample_neighbors_kernel.h"
+
+#include <cmath>
+#include <queue>
+#include <vector>
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+
+template <typename T>
+struct GraphWeightedNode {
+  T node_id;
+  float weight_key;
+  T eid;
+  GraphWeightedNode() {
+    node_id = 0;
+    weight_key = 0;
+    eid = 0;
+  }
+  GraphWeightedNode(T node_id, float weight_key, T eid = 0)
+      : node_id(node_id), weight_key(weight_key), eid(eid) {}
+  void operator=(const GraphWeightedNode<T>& other) {
+    node_id = other.node_id;
+    weight_key = other.weight_key;
+    eid = other.eid;
+  }
+  friend bool operator>(const GraphWeightedNode<T>& n1,
+                        const GraphWeightedNode<T>& n2) {
+    return n1.weight_key > n2.weight_key;
+  }
+};
+
+template <typename T>
+void SampleWeightedNeighbors(
+    std::vector<T>& out_src,  // NOLINT
+    const std::vector<float>& out_weight,
+    std::vector<T>& out_eids,  // NOLINT
+    int sample_size,
+    std::mt19937& rng,                                         // NOLINT
+    std::uniform_real_distribution<float>& dice_distribution,  // NOLINT
+    bool return_eids) {
+  std::priority_queue<phi::GraphWeightedNode<T>,
+                      std::vector<phi::GraphWeightedNode<T>>,
+                      std::greater<phi::GraphWeightedNode<T>>>
+      min_heap;
+  for (size_t i = 0; i < out_src.size(); i++) {
+    float weight_key = log2(dice_distribution(rng)) * (1 / out_weight[i]);
+    if (static_cast<int>(i) < sample_size) {
+      if (!return_eids) {
+        min_heap.push(phi::GraphWeightedNode<T>(out_src[i], weight_key));
+      } else {
+        min_heap.push(
+            phi::GraphWeightedNode<T>(out_src[i], weight_key, out_eids[i]));
+      }
+    } else {
+      const phi::GraphWeightedNode<T>& small = min_heap.top();
+      phi::GraphWeightedNode<T> cmp;
+      if (!return_eids) {
+        cmp = GraphWeightedNode<T>(out_src[i], weight_key);
+      } else {
+        cmp = GraphWeightedNode<T>(out_src[i], weight_key, out_eids[i]);
+      }
+      bool flag = cmp > small;
+      if (flag) {
+        min_heap.pop();
+        min_heap.push(cmp);
+      }
+    }
+  }
+
+  int cnt = 0;
+  while (!min_heap.empty()) {
+    const phi::GraphWeightedNode<T>& tmp = min_heap.top();
+    out_src[cnt] = tmp.node_id;
+    if (return_eids) {
+      out_eids[cnt] = tmp.eid;
+    }
+    cnt++;
+    min_heap.pop();
+  }
+}
+
+template <typename T>
+void SampleNeighbors(const T* row,
+                     const T* col_ptr,
+                     const float* edge_weight,
+                     const T* eids,
+                     const T* input,
+                     std::vector<T>* output,
+                     std::vector<int>* output_count,
+                     std::vector<T>* output_eids,
+                     int sample_size,
+                     int bs,
+                     bool return_eids) {
+  std::vector<std::vector<T>> out_src_vec;
+  std::vector<std::vector<float>> out_weight_vec;
+  std::vector<std::vector<T>> out_eids_vec;
+  // `sample_cumsum_sizes` record the start position and end position
+  // after sampling.
+  std::vector<int> sample_cumsum_sizes(bs + 1);
+  // `total_neighbors` the size of output after sample.
+  int total_neighbors = 0;
+  sample_cumsum_sizes[0] = total_neighbors;
+  for (int i = 0; i < bs; i++) {
+    T node = input[i];
+    int cap = col_ptr[node + 1] - col_ptr[node];
+    int k = cap > sample_size ? sample_size : cap;
+    total_neighbors += k;
+    sample_cumsum_sizes[i + 1] = total_neighbors;
+    std::vector<T> out_src;
+    out_src.resize(cap);
+    out_src_vec.emplace_back(out_src);
+    std::vector<float> out_weight;
+    out_weight.resize(cap);
+    out_weight_vec.emplace_back(out_weight);
+    if (return_eids) {
+      std::vector<T> out_eids;
+      out_eids.resize(cap);
+      out_eids_vec.emplace_back(out_eids);
+    }
+  }
+
+  output_count->resize(bs);
+  output->resize(total_neighbors);
+  if (return_eids) {
+    output_eids->resize(total_neighbors);
+  }
+
+  std::random_device rd;
+  std::mt19937 rng{rd()};
+  std::uniform_real_distribution<float> dice_distribution(0, 1);
+
+#ifdef PADDLE_WITH_MKLML
+#pragma omp parallel for
+#endif
+  // Sample the neighbors in parallelism.
+  for (int i = 0; i < bs; i++) {
+    T node = input[i];
+    T begin = col_ptr[node], end = col_ptr[node + 1];
+    int cap = end - begin;
+    if (sample_size < cap) {  // sample_size < neighbor_len
+      std::copy(row + begin, row + end, out_src_vec[i].begin());
+      std::copy(
+          edge_weight + begin, edge_weight + end, out_weight_vec[i].begin());
+      if (return_eids) {
+        std::copy(eids + begin, eids + end, out_eids_vec[i].begin());
+      }
+      SampleWeightedNeighbors(out_src_vec[i],
+                              out_weight_vec[i],
+                              out_eids_vec[i],
+                              sample_size,
+                              rng,
+                              dice_distribution,
+                              return_eids);
+      *(output_count->data() + i) = sample_size;
+    } else {  // sample_size >= neighbor_len, directly copy
+      std::copy(row + begin, row + end, out_src_vec[i].begin());
+      if (return_eids) {
+        std::copy(eids + begin, eids + end, out_eids_vec[i].begin());
+      }
+      *(output_count->data() + i) = cap;
+    }
+  }
+
+#ifdef PADDLE_WITH_MKLML
+#pragma omp parallel for
+#endif
+  // Copy the results parallelism
+  for (int i = 0; i < bs; i++) {
+    int k = sample_cumsum_sizes[i + 1] - sample_cumsum_sizes[i];
+    std::copy(out_src_vec[i].begin(),
+              out_src_vec[i].begin() + k,
+              output->data() + sample_cumsum_sizes[i]);
+    if (return_eids) {
+      std::copy(out_eids_vec[i].begin(),
+                out_eids_vec[i].begin() + k,
+                output_eids->data() + sample_cumsum_sizes[i]);
+    }
+  }
+}
+
+template <typename T, typename Context>
+void WeightedSampleNeighborsKernel(const Context& dev_ctx,
+                                   const DenseTensor& row,
+                                   const DenseTensor& col_ptr,
+                                   const DenseTensor& edge_weight,
+                                   const DenseTensor& x,
+                                   const paddle::optional<DenseTensor>& eids,
+                                   int sample_size,
+                                   bool return_eids,
+                                   DenseTensor* out,
+                                   DenseTensor* out_count,
+                                   DenseTensor* out_eids) {
+  const T* row_data = row.data<T>();
+  const T* col_ptr_data = col_ptr.data<T>();
+  const float* weights_data = edge_weight.data<float>();
+  const T* x_data = x.data<T>();
+  const T* eids_data =
+      (eids.get_ptr() == nullptr ? nullptr : eids.get_ptr()->data<T>());
+  int bs = x.dims()[0];
+
+  std::vector<T> output;
+  std::vector<int> output_count;
+  std::vector<T> output_eids;
+
+  SampleNeighbors<T>(row_data,
+                     col_ptr_data,
+                     weights_data,
+                     eids_data,
+                     x_data,
+                     &output,
+                     &output_count,
+                     &output_eids,
+                     sample_size,
+                     bs,
+                     return_eids);
+
+  if (return_eids) {
+    out_eids->Resize({static_cast<int>(output_eids.size())});
+    T* out_eids_data = dev_ctx.template Alloc<T>(out_eids);
+    std::copy(output_eids.begin(), output_eids.end(), out_eids_data);
+  }
+
+  out->Resize({static_cast<int>(output.size())});
+  T* out_data = dev_ctx.template Alloc<T>(out);
+  std::copy(output.begin(), output.end(), out_data);
+  out_count->Resize({bs});
+  int* out_count_data = dev_ctx.template Alloc<int>(out_count);
+  std::copy(output_count.begin(), output_count.end(), out_count_data);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(weighted_sample_neighbors,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::WeightedSampleNeighborsKernel,
+                   int,
+                   int64_t) {}
diff --git a/paddle/phi/kernels/funcs/block_radix_topk.cuh b/paddle/phi/kernels/funcs/block_radix_topk.cuh
new file mode 100644
index 0000000000000..320d8ad8fc4f3
--- /dev/null
+++ b/paddle/phi/kernels/funcs/block_radix_topk.cuh
@@ -0,0 +1,349 @@
+/*
+ * Copyright (c) 2019-2022, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#ifdef PADDLE_WITH_CUDA
+#include <cub/block/block_load.cuh>
+#include <cub/block/block_scan.cuh>
+#include <cub/block/block_store.cuh>
+#include <cub/util_ptx.cuh>
+#include <cub/util_type.cuh>
+
+namespace paddle {
+namespace framework {
+
+template<
+    typename KeyT,
+    int BLOCK_SIZE,
+    bool GREATER = true,
+    int RADIX_BITS = 8>
+class BlockRadixTopKGlobalMemory {
+  static_assert(cub::PowerOfTwo<RADIX_BITS>::VALUE && (RADIX_BITS <= (sizeof(KeyT) * 8)),
+                "RADIX_BITS should be power of 2, and <= (sizeof(KeyT) * 8)");
+  static_assert(cub::PowerOfTwo<BLOCK_SIZE>::VALUE, "BLOCK_SIZE should be power of 2");
+  using KeyTraits = cub::Traits<KeyT>;
+  using UnsignedBits = typename KeyTraits::UnsignedBits;
+  using BlockScanT = cub::BlockScan<int, BLOCK_SIZE>;
+  static constexpr int RADIX_SIZE = (1 << RADIX_BITS);
+  static constexpr int SCAN_ITEMS_PER_THREAD = (RADIX_SIZE + BLOCK_SIZE - 1) / BLOCK_SIZE;
+  using BinBlockLoad = cub::BlockLoad<int, BLOCK_SIZE, SCAN_ITEMS_PER_THREAD>;
+  using BinBlockStore = cub::BlockStore<int, BLOCK_SIZE, SCAN_ITEMS_PER_THREAD>;
+  struct _TempStorage {
+    typename BlockScanT::TempStorage scan_storage;
+    union {
+      typename BinBlockLoad::TempStorage load_storage;
+      typename BinBlockStore::TempStorage store_storage;
+    } load_store;
+    union {
+      int shared_bins[RADIX_SIZE];
+    };
+    int share_target_k;
+    int share_bucket_id;
+  };
+
+ public:
+  struct TempStorage : cub::Uninitialized<_TempStorage> {
+  };
+  __device__ __forceinline__ BlockRadixTopKGlobalMemory(TempStorage &temp_storage)
+      : temp_storage_{temp_storage.Alias()}, tid_(threadIdx.x){};
+  __device__ __forceinline__ void radixTopKGetThreshold(const KeyT *data, int k, int size, KeyT &topK, bool &topk_is_unique) {
+    assert(k < size && k > 0);
+    int target_k = k;
+    UnsignedBits key_pattern = 0;
+    int digit_pos = sizeof(KeyT) * 8 - RADIX_BITS;
+    for (; digit_pos >= 0; digit_pos -= RADIX_BITS) {
+      UpdateSharedBins(data, size, digit_pos, key_pattern);
+      InclusiveScanBins();
+      UpdateTopK(digit_pos, target_k, key_pattern);
+      if (target_k == 0) break;
+    }
+    if (target_k == 0) {
+      key_pattern -= 1;
+      topk_is_unique = true;
+    } else {
+      topk_is_unique = false;
+    }
+    if (GREATER) key_pattern = ~key_pattern;
+    UnsignedBits topK_unsigned = KeyTraits::TwiddleOut(key_pattern);
+    topK = reinterpret_cast<KeyT &>(topK_unsigned);
+  }
+
+ private:
+  __device__ __forceinline__ void UpdateSharedBins(const KeyT *key, int size, int digit_pos, UnsignedBits key_pattern) {
+    for (int id = tid_; id < RADIX_SIZE; id += BLOCK_SIZE) {
+      temp_storage_.shared_bins[id] = 0;
+    }
+    cub::CTA_SYNC();
+    UnsignedBits key_mask = ((UnsignedBits)(-1)) << ((UnsignedBits)(digit_pos + RADIX_BITS));
+#pragma unroll
+    for (int idx = tid_; idx < size; idx += BLOCK_SIZE) {
+      KeyT key_data = key[idx];
+      UnsignedBits twiddled_data = KeyTraits::TwiddleIn(reinterpret_cast<UnsignedBits &>(key_data));
+      if (GREATER) twiddled_data = ~twiddled_data;
+      UnsignedBits digit_in_radix = cub::BFE<UnsignedBits>(twiddled_data, digit_pos, RADIX_BITS);
+      if ((twiddled_data & key_mask) == (key_pattern & key_mask)) {
+        atomicAdd(&temp_storage_.shared_bins[digit_in_radix], 1);
+      }
+    }
+    cub::CTA_SYNC();
+  }
+  __device__ __forceinline__ void InclusiveScanBins() {
+    int items[SCAN_ITEMS_PER_THREAD];
+    BinBlockLoad(temp_storage_.load_store.load_storage).Load(temp_storage_.shared_bins, items, RADIX_SIZE, 0);
+    cub::CTA_SYNC();
+    BlockScanT(temp_storage_.scan_storage).InclusiveSum(items, items);
+    cub::CTA_SYNC();
+    BinBlockStore(temp_storage_.load_store.store_storage).Store(temp_storage_.shared_bins, items, RADIX_SIZE);
+    cub::CTA_SYNC();
+  }
+  __device__ __forceinline__ void UpdateTopK(int digit_pos,
+                                             int &target_k,
+                                             UnsignedBits &target_pattern) {
+    for (int idx = tid_; (idx < RADIX_SIZE); idx += BLOCK_SIZE) {
+      int prev_count = (idx == 0) ? 0 : temp_storage_.shared_bins[idx - 1];
+      int cur_count = temp_storage_.shared_bins[idx];
+      if (prev_count <= target_k && cur_count > target_k) {
+        temp_storage_.share_target_k = target_k - prev_count;
+        temp_storage_.share_bucket_id = idx;
+      }
+    }
+    cub::CTA_SYNC();
+    target_k = temp_storage_.share_target_k;
+    int target_bucket_id = temp_storage_.share_bucket_id;
+    UnsignedBits key_segment = ((UnsignedBits) target_bucket_id) << ((UnsignedBits) digit_pos);
+    target_pattern |= key_segment;
+  }
+  _TempStorage &temp_storage_;
+  int tid_;
+};
+
+template<
+    typename KeyT,
+    int BLOCK_SIZE,
+    int ITEMS_PER_THREAD,
+    bool GREATER = true,
+    typename ValueT = cub::NullType,
+    int RADIX_BITS = 8>
+class BlockRadixTopKRegister {
+  static_assert(cub::PowerOfTwo<RADIX_BITS>::VALUE && (RADIX_BITS <= (sizeof(KeyT) * 8)),
+                "RADIX_BITS should be power of 2, and <= (sizeof(KeyT) * 8)");
+  static_assert(cub::PowerOfTwo<BLOCK_SIZE>::VALUE, "BLOCK_SIZE should be power of 2");
+  using KeyTraits = cub::Traits<KeyT>;
+  using UnsignedBits = typename KeyTraits::UnsignedBits;
+  using BlockScanT = cub::BlockScan<int, BLOCK_SIZE>;
+  static constexpr int RADIX_SIZE = (1 << RADIX_BITS);
+  static constexpr bool KEYS_ONLY = std::is_same<ValueT, cub::NullType>::value;
+  static constexpr int SCAN_ITEMS_PER_THREAD = (RADIX_SIZE + BLOCK_SIZE - 1) / BLOCK_SIZE;
+  using BinBlockLoad = cub::BlockLoad<int, BLOCK_SIZE, SCAN_ITEMS_PER_THREAD>;
+  using BinBlockStore = cub::BlockStore<int, BLOCK_SIZE, SCAN_ITEMS_PER_THREAD>;
+  using BlockExchangeKey = cub::BlockExchange<KeyT, BLOCK_SIZE, ITEMS_PER_THREAD>;
+  using BlockExchangeValue = cub::BlockExchange<ValueT, BLOCK_SIZE, ITEMS_PER_THREAD>;
+
+  using _ExchangeKeyTempStorage = typename BlockExchangeKey::TempStorage;
+  using _ExchangeValueTempStorage = typename BlockExchangeValue::TempStorage;
+  typedef union ExchangeKeyTempStorageType {
+    _ExchangeKeyTempStorage key_storage;
+  } ExchKeyTempStorageType;
+  typedef union ExchangeKeyValueTempStorageType {
+    _ExchangeKeyTempStorage key_storage;
+    _ExchangeValueTempStorage value_storage;
+  } ExchKeyValueTempStorageType;
+  using _ExchangeType = typename std::conditional<KEYS_ONLY, ExchKeyTempStorageType, ExchKeyValueTempStorageType>::type;
+
+  struct _TempStorage {
+    typename BlockScanT::TempStorage scan_storage;
+    union {
+      typename BinBlockLoad::TempStorage load_storage;
+      typename BinBlockStore::TempStorage store_storage;
+    } load_store;
+    union {
+      int shared_bins[RADIX_SIZE];
+      _ExchangeType exchange_storage;
+    };
+    int share_target_k;
+    int share_bucket_id;
+    int share_prev_count;
+  };
+
+ public:
+  struct TempStorage : cub::Uninitialized<_TempStorage> {
+  };
+  __device__ __forceinline__ BlockRadixTopKRegister(TempStorage &temp_storage)
+      : temp_storage_{temp_storage.Alias()}, tid_(threadIdx.x){};
+  __device__ __forceinline__ void radixTopKToStriped(KeyT (&keys)[ITEMS_PER_THREAD],
+                                                     const int k, const int valid_count) {
+    TopKGenRank(keys, k, valid_count);
+    int is_valid[ITEMS_PER_THREAD];
+    GenValidArray(is_valid, k);
+    BlockExchangeKey{temp_storage_.exchange_storage.key_storage}.ScatterToStripedFlagged(keys, keys, ranks_, is_valid);
+    cub::CTA_SYNC();
+  }
+  __device__ __forceinline__ void radixTopKToStriped(KeyT (&keys)[ITEMS_PER_THREAD], ValueT (&values)[ITEMS_PER_THREAD],
+                                                     const int k, const int valid_count) {
+    TopKGenRank(keys, k, valid_count);
+    int is_valid[ITEMS_PER_THREAD];
+    GenValidArray(is_valid, k);
+    BlockExchangeKey{temp_storage_.exchange_storage.key_storage}.ScatterToStripedFlagged(keys, keys, ranks_, is_valid);
+    cub::CTA_SYNC();
+    BlockExchangeValue{temp_storage_.exchange_storage.value_storage}.ScatterToStripedFlagged(values, values, ranks_, is_valid);
+    cub::CTA_SYNC();
+  }
+
+ private:
+  __device__ __forceinline__ void TopKGenRank(KeyT (&keys)[ITEMS_PER_THREAD], const int k, const int valid_count) {
+    assert(k <= BLOCK_SIZE * ITEMS_PER_THREAD);
+    assert(k <= valid_count);
+    if (k == valid_count) return;
+    UnsignedBits(&unsigned_keys)[ITEMS_PER_THREAD] = reinterpret_cast<UnsignedBits(&)[ITEMS_PER_THREAD]>(keys);
+    search_mask_ = 0;
+    top_k_mask_ = 0;
+
+#pragma unroll
+    for (unsigned int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) {
+      int idx = KEY * BLOCK_SIZE + tid_;
+      unsigned_keys[KEY] = KeyTraits::TwiddleIn(unsigned_keys[KEY]);
+      if (GREATER) unsigned_keys[KEY] = ~unsigned_keys[KEY];
+      if (idx < valid_count) search_mask_ |= (1U << KEY);
+    }
+
+    int target_k = k;
+    int prefix_k = 0;
+
+    for (int digit_pos = sizeof(KeyT) * 8 - RADIX_BITS; digit_pos >= 0; digit_pos -= RADIX_BITS) {
+      UpdateSharedBins(unsigned_keys, digit_pos, prefix_k);
+      InclusiveScanBins();
+      UpdateTopK(unsigned_keys, digit_pos, target_k, prefix_k, digit_pos == 0);
+      if (target_k == 0) break;
+    }
+
+#pragma unroll
+    for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) {
+      if (GREATER) unsigned_keys[KEY] = ~unsigned_keys[KEY];
+      unsigned_keys[KEY] = KeyTraits::TwiddleOut(unsigned_keys[KEY]);
+    }
+  }
+  __device__ __forceinline__ void GenValidArray(int (&is_valid)[ITEMS_PER_THREAD], int k) {
+#pragma unroll
+    for (unsigned int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) {
+      if ((top_k_mask_ & (1U << KEY)) && ranks_[KEY] < k) {
+        is_valid[KEY] = 1;
+      } else {
+        is_valid[KEY] = 0;
+      }
+    }
+  }
+  __device__ __forceinline__ void UpdateSharedBins(UnsignedBits (&unsigned_keys)[ITEMS_PER_THREAD],
+                                                   int digit_pos, int prefix_k) {
+    for (int id = tid_; id < RADIX_SIZE; id += BLOCK_SIZE) {
+      temp_storage_.shared_bins[id] = 0;
+    }
+    cub::CTA_SYNC();
+//#define USE_MATCH
+#ifdef USE_MATCH
+    int lane_mask = cub::LaneMaskLt();
+#pragma unroll
+    for (unsigned int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) {
+      bool is_search = search_mask_ & (1U << KEY);
+      int bucket_idx = -1;
+      if (is_search) {
+        UnsignedBits digit_in_radix = cub::BFE<UnsignedBits>(unsigned_keys[KEY], digit_pos, RADIX_BITS);
+        bucket_idx = (int) digit_in_radix;
+      }
+      int warp_match_mask = __match_any_sync(0xffffffff, bucket_idx);
+      int same_count = __popc(warp_match_mask);
+      int idx_in_same_bucket = __popc(warp_match_mask & lane_mask);
+      int same_bucket_root_lane = __ffs(warp_match_mask) - 1;
+      int same_bucket_start_idx;
+      if (idx_in_same_bucket == 0 && is_search) {
+        same_bucket_start_idx = atomicAdd(&temp_storage_.shared_bins[bucket_idx], same_count);
+      }
+      same_bucket_start_idx = __shfl_sync(0xffffffff, same_bucket_start_idx, same_bucket_root_lane, 32);
+      if (is_search) {
+        ranks_[KEY] = same_bucket_start_idx + idx_in_same_bucket + prefix_k;
+      }
+    }
+#else
+#pragma unroll
+    for (unsigned int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) {
+      bool is_search = search_mask_ & (1U << KEY);
+      int bucket_idx = -1;
+      if (is_search) {
+        UnsignedBits digit_in_radix = cub::BFE<UnsignedBits>(unsigned_keys[KEY], digit_pos, RADIX_BITS);
+        bucket_idx = (int) digit_in_radix;
+        ranks_[KEY] = atomicAdd(&temp_storage_.shared_bins[bucket_idx], 1) + prefix_k;
+      }
+    }
+#endif
+    cub::CTA_SYNC();
+  }
+  __device__ __forceinline__ void InclusiveScanBins() {
+    int items[SCAN_ITEMS_PER_THREAD];
+    BinBlockLoad(temp_storage_.load_store.load_storage).Load(temp_storage_.shared_bins, items, RADIX_SIZE, 0);
+    cub::CTA_SYNC();
+    BlockScanT(temp_storage_.scan_storage).InclusiveSum(items, items);
+    cub::CTA_SYNC();
+    BinBlockStore(temp_storage_.load_store.store_storage).Store(temp_storage_.shared_bins, items, RADIX_SIZE);
+    cub::CTA_SYNC();
+  }
+  __device__ __forceinline__ void UpdateTopK(UnsignedBits (&unsigned_keys)[ITEMS_PER_THREAD],
+                                             int digit_pos,
+                                             int &target_k,
+                                             int &prefix_k,
+                                             bool mark_equal) {
+    for (int idx = tid_; (idx < RADIX_SIZE); idx += BLOCK_SIZE) {
+      int prev_count = (idx == 0) ? 0 : temp_storage_.shared_bins[idx - 1];
+      int cur_count = temp_storage_.shared_bins[idx];
+      if (prev_count <= target_k && cur_count > target_k) {
+        temp_storage_.share_target_k = target_k - prev_count;
+        temp_storage_.share_bucket_id = idx;
+        temp_storage_.share_prev_count = prev_count;
+      }
+    }
+    cub::CTA_SYNC();
+    target_k = temp_storage_.share_target_k;
+    prefix_k += temp_storage_.share_prev_count;
+    int target_bucket_id = temp_storage_.share_bucket_id;
+#pragma unroll
+    for (unsigned int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++) {
+      if (search_mask_ & (1U << KEY)) {
+        UnsignedBits digit_in_radix = cub::BFE<UnsignedBits>(unsigned_keys[KEY], digit_pos, RADIX_BITS);
+        if (digit_in_radix < target_bucket_id) {
+          top_k_mask_ |= (1U << KEY);
+          search_mask_ &= ~(1U << KEY);
+        } else if (digit_in_radix > target_bucket_id) {
+          search_mask_ &= ~(1U << KEY);
+        } else {
+          if (mark_equal) top_k_mask_ |= (1U << KEY);
+        }
+        if (digit_in_radix <= target_bucket_id) {
+          int prev_count = (digit_in_radix == 0) ? 0 : temp_storage_.shared_bins[digit_in_radix - 1];
+          ranks_[KEY] += prev_count;
+        }
+      }
+    }
+    cub::CTA_SYNC();
+  }
+
+  _TempStorage &temp_storage_;
+  int tid_;
+  int ranks_[ITEMS_PER_THREAD];
+  unsigned int search_mask_;
+  unsigned int top_k_mask_;
+};
+
+};  // end namespace framework
+};  // end namespace paddle
+#endif
diff --git a/paddle/phi/kernels/funcs/random.cuh b/paddle/phi/kernels/funcs/random.cuh
new file mode 100644
index 0000000000000..502b7e85ee97f
--- /dev/null
+++ b/paddle/phi/kernels/funcs/random.cuh
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2019-2022, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#ifdef __NVCC__
+#include <cuda_runtime_api.h>  // NOLINT
+#endif
+
+class RandomNumGen {
+ public:
+  __host__ __device__ __forceinline__ RandomNumGen(int gid, unsigned long long seed) {
+    next_random = seed + gid;
+    next_random ^= next_random >> 33U;
+    next_random *= 0xff51afd7ed558ccdUL;
+    next_random ^= next_random >> 33U;
+    next_random *= 0xc4ceb9fe1a85ec53UL;
+    next_random ^= next_random >> 33U;
+  }
+  __host__ __device__ __forceinline__ ~RandomNumGen() = default;
+  __host__ __device__ __forceinline__ void SetSeed(int seed) {
+    next_random = seed;
+    NextValue();
+  }
+  __host__ __device__ __forceinline__ unsigned long long SaveState() const {
+    return next_random;
+  }
+  __host__ __device__ __forceinline__ void LoadState(unsigned long long state) {
+    next_random = state;
+  }
+  __host__ __device__ __forceinline__ int Random() {
+    int ret_value = (int) (next_random & 0x7fffffffULL);
+    NextValue();
+    return ret_value;
+  }
+  __host__ __device__ __forceinline__ int RandomMod(int mod) {
+    return Random() % mod;
+  }
+  __host__ __device__ __forceinline__ int64_t Random64() {
+    int64_t ret_value = (next_random & 0x7FFFFFFFFFFFFFFFLL);
+    NextValue();
+    return ret_value;
+  }
+  __host__ __device__ __forceinline__ int64_t RandomMod64(int64_t mod) {
+    return Random64() % mod;
+  }
+  __host__ __device__ __forceinline__ float RandomUniformFloat(float max = 1.0f, float min = 0.0f) {
+    int value = (int) (next_random & 0xffffff);
+    auto ret_value = (float) value;
+    ret_value /= 0xffffffL;
+    ret_value *= (max - min);
+    ret_value += min;
+    NextValue();
+    return ret_value;
+  }
+  __host__ __device__ __forceinline__ bool RandomBool(float true_prob) {
+    float value = RandomUniformFloat();
+    return value <= true_prob;
+  }
+  __host__ __device__ __forceinline__ void NextValue() {
+    //next_random = next_random * (unsigned long long)0xc4ceb9fe1a85ec53UL + generator_id;
+    //next_random = next_random * (unsigned long long)25214903917ULL + 11;
+    next_random = next_random * (unsigned long long) 13173779397737131ULL + 1023456798976543201ULL;
+  }
+
+ private:
+  unsigned long long next_random = 1;
+};
diff --git a/paddle/phi/kernels/gpu/weighted_sample_neighbors_kernel.cu b/paddle/phi/kernels/gpu/weighted_sample_neighbors_kernel.cu
new file mode 100644
index 0000000000000..d4e0ca632e04d
--- /dev/null
+++ b/paddle/phi/kernels/gpu/weighted_sample_neighbors_kernel.cu
@@ -0,0 +1,535 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <thrust/copy.h>
+#include <thrust/device_vector.h>
+#include <thrust/reduce.h>
+#include <thrust/scan.h>
+#include <thrust/sequence.h>
+#include <thrust/transform.h>
+
+#ifdef PADDLE_WITH_CUDA
+#include <cuda_runtime.h>
+#include <curand_kernel.h>
+#include "cub/cub.cuh"
+#endif
+
+#include "math.h"  // NOLINT
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/common/memory_utils.h"
+#include "paddle/phi/core/hostdevice.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/block_radix_topk.cuh"
+#include "paddle/phi/kernels/funcs/random.cuh"
+#include "paddle/phi/kernels/weighted_sample_neighbors_kernel.h"
+#define SAMPLE_SIZE_THRESHOLD 1024
+
+namespace phi {
+
+#ifdef PADDLE_WITH_CUDA
+__device__ __forceinline__ float GenKeyFromWeight(
+    const float weight,
+    RandomNumGen& rng) {  // NOLINT
+  rng.NextValue();
+  float u = -rng.RandomUniformFloat(1.0f, 0.5f);
+  long long random_num2 = 0;  // NOLINT
+  int seed_count = -1;
+  do {
+    random_num2 = rng.Random64();
+    seed_count++;
+  } while (!random_num2);
+  int one_bit = __clzll(random_num2) + seed_count * 64;
+  u *= exp2f(-one_bit);
+  float logk = (log1pf(u) / logf(2.0)) * (1 / weight);
+  return logk;
+}
+#endif
+
+template <typename T, bool NeedNeighbor = false>
+__global__ void GetSampleCountAndNeighborCountKernel(const T* col_ptr,
+                                                     const T* input_nodes,
+                                                     int* actual_size,
+                                                     int* neighbor_count,
+                                                     int sample_size,
+                                                     int n) {
+  int i = threadIdx.x + blockIdx.x * blockDim.x;
+  if (i >= n) return;
+  T nid = input_nodes[i];
+  int neighbor_size = static_cast<int>(col_ptr[nid + 1] - col_ptr[nid]);
+  // sample_size < 0 means sample all.
+  int k = neighbor_size;
+  if (sample_size >= 0) {
+    k = min(neighbor_size, sample_size);
+  }
+  actual_size[i] = k;
+  if (NeedNeighbor) {
+    neighbor_count[i] = (neighbor_size <= sample_size) ? 0 : neighbor_size;
+  }
+}
+
+#ifdef PADDLE_WITH_CUDA
+template <typename T, unsigned int BLOCK_SIZE>
+__launch_bounds__(BLOCK_SIZE) __global__
+    void WeightedSampleLargeKernel(T* sample_output,
+                                   const int* sample_offset,
+                                   const int* target_neighbor_offset,
+                                   float* weight_keys_buf,
+                                   const T* input_nodes,
+                                   int input_node_count,
+                                   const T* in_rows,
+                                   const T* col_ptr,
+                                   const float* edge_weight,
+                                   const T* eids,
+                                   int max_sample_count,
+                                   unsigned long long random_seed,  // NOLINT
+                                   T* out_eids,
+                                   bool return_eids) {
+  int i = blockIdx.x;
+  if (i >= input_node_count) return;
+  int gidx = threadIdx.x + blockIdx.x * BLOCK_SIZE;
+  T nid = input_nodes[i];
+  T start = col_ptr[nid + 1];
+  T end = col_ptr[nid];
+  int neighbor_count = static_cast<int>(end - start);
+
+  float* weight_keys_local_buff = weight_keys_buf + target_neighbor_offset[i];
+  int offset = sample_offset[i];
+  if (neighbor_count <= max_sample_count) {
+    for (int j = threadIdx.x; j < neighbor_count; j += BLOCK_SIZE) {
+      sample_output[offset + j] = in_rows[start + j];
+      if (return_eids) {
+        out_eids[offset + j] = eids[start + j];
+      }
+    }
+  } else {
+    RandomNumGen rng(gidx, random_seed);
+    for (int j = threadIdx.x; j < neighbor_count; j += BLOCK_SIZE) {
+      float thread_weight = edge_weight[start + j];
+      weight_keys_local_buff[j] =
+          static_cast<float>(GenKeyFromWeight(thread_weight, rng));
+    }
+    __syncthreads();
+
+    float topk_val;
+    bool topk_is_unique;
+
+    using BlockRadixSelectT =
+        paddle::framework::BlockRadixTopKGlobalMemory<float, BLOCK_SIZE, true>;
+    __shared__ typename BlockRadixSelectT::TempStorage share_storage;
+
+    BlockRadixSelectT{share_storage}.radixTopKGetThreshold(
+        weight_keys_local_buff,
+        max_sample_count,
+        neighbor_count,
+        topk_val,
+        topk_is_unique);
+    __shared__ int cnt;
+
+    if (threadIdx.x == 0) {
+      cnt = 0;
+    }
+    __syncthreads();
+
+    // We use atomicAdd 1 operations instead of binaryScan to calculate the
+    // write index, since we do not need to keep the relative positions of
+    // element.
+
+    if (topk_is_unique) {
+      for (int j = threadIdx.x; j < neighbor_count; j += BLOCK_SIZE) {
+        float key = weight_keys_local_buff[j];
+        bool has_topk = (key >= topk_val);
+
+        if (has_topk) {
+          int write_index = atomicAdd(&cnt, 1);
+          sample_output[offset + write_index] = in_rows[start + j];
+          if (return_eids) {
+            out_eids[offset + write_index] = eids[start + j];
+          }
+        }
+      }
+    } else {
+      for (int j = threadIdx.x; j < neighbor_count; j += BLOCK_SIZE) {
+        float key = weight_keys_local_buff[j];
+        bool has_topk = (key > topk_val);
+
+        if (has_topk) {
+          int write_index = atomicAdd(&cnt, 1);
+          sample_output[offset + write_index] = in_rows[start + j];
+          if (return_eids) {
+            out_eids[offset + write_index] = eids[start + j];
+          }
+        }
+      }
+      __syncthreads();
+
+      for (int j = threadIdx.x; j < neighbor_count; j += BLOCK_SIZE) {
+        float key = weight_keys_local_buff[j];
+        bool has_topk = (key == topk_val);
+        if (has_topk) {
+          int write_index = atomicAdd(&cnt, 1);
+          if (write_index >= max_sample_count) {
+            break;
+          }
+          sample_output[offset + write_index] = in_rows[start + j];
+          if (return_eids) {
+            out_eids[offset + write_index] = eids[start + j];
+          }
+        }
+      }
+    }
+  }
+}
+#endif
+
+template <typename T>
+__global__ void SampleAllKernel(T* sample_output,
+                                const int* sample_offset,
+                                const T* input_nodes,
+                                int input_node_count,
+                                const T* in_rows,
+                                const T* col_ptr,
+                                const T* eids,
+                                T* out_eids,
+                                bool return_eids) {
+  int i = blockIdx.x;
+  if (i >= input_node_count) return;
+  T nid = input_nodes[i];
+  T start = col_ptr[nid + 1];
+  T end = col_ptr[nid];
+  int neighbor_count = static_cast<int>(end - start);
+  if (neighbor_count <= 0) return;
+  int offset = sample_offset[i];
+  for (int j = threadIdx.x; j < neighbor_count; j += blockDim.x) {
+    sample_output[offset + j] = in_rows[start + j];
+    if (return_eids) {
+      out_eids[offset + j] = eids[start + j];
+    }
+  }
+}
+
+// A-RES algorithm
+#ifdef PADDLE_WITH_CUDA
+template <typename T, unsigned int ITEMS_PER_THREAD, unsigned int BLOCK_SIZE>
+__launch_bounds__(BLOCK_SIZE) __global__
+    void WeightedSampleKernel(T* sample_output,
+                              const int* sample_offset,
+                              const T* input_nodes,
+                              int input_node_count,
+                              const T* in_rows,
+                              const T* col_ptr,
+                              const float* edge_weight,
+                              const T* eids,
+                              int max_sample_count,
+                              unsigned long long random_seed,  // NOLINT
+                              T* out_eids,
+                              bool return_eids) {
+  int i = blockIdx.x;
+  if (i >= input_node_count) return;
+  int gidx = threadIdx.x + blockIdx.x * BLOCK_SIZE;
+  T nid = input_nodes[i];
+  T start = col_ptr[nid];
+  T end = col_ptr[nid + 1];
+  int neighbor_count = static_cast<int>(end - start);
+  int offset = sample_offset[i];
+
+  if (neighbor_count <= max_sample_count) {
+    for (int j = threadIdx.x; j < neighbor_count; j += BLOCK_SIZE) {
+      sample_output[offset + j] = in_rows[start + j];
+      if (return_eids) {
+        out_eids[offset + j] = eids[start + j];
+      }
+    }
+  } else {
+    RandomNumGen rng(gidx, random_seed);
+    float weight_keys[ITEMS_PER_THREAD];
+    int neighbor_idxs[ITEMS_PER_THREAD];
+    using BlockRadixTopKT = paddle::framework::
+        BlockRadixTopKRegister<float, BLOCK_SIZE, ITEMS_PER_THREAD, true, int>;
+    __shared__ typename BlockRadixTopKT::TempStorage sort_tmp_storage;
+
+    const int tx = threadIdx.x;
+#pragma unroll
+    for (int j = 0; j < ITEMS_PER_THREAD; j++) {
+      int idx = BLOCK_SIZE * j + tx;
+      if (idx < neighbor_count) {
+        float thread_weight = edge_weight[start + idx];
+        weight_keys[j] = GenKeyFromWeight(thread_weight, rng);
+        neighbor_idxs[j] = idx;
+      }
+    }
+    const int valid_count = (neighbor_count < (BLOCK_SIZE * ITEMS_PER_THREAD))
+                                ? neighbor_count
+                                : (BLOCK_SIZE * ITEMS_PER_THREAD);
+    BlockRadixTopKT{sort_tmp_storage}.radixTopKToStriped(
+        weight_keys, neighbor_idxs, max_sample_count, valid_count);
+    __syncthreads();
+    const int stride = BLOCK_SIZE * ITEMS_PER_THREAD - max_sample_count;
+
+    for (int idx_offset = ITEMS_PER_THREAD * BLOCK_SIZE;
+         idx_offset < neighbor_count;
+         idx_offset += stride) {
+#pragma unroll
+      for (int j = 0; j < ITEMS_PER_THREAD; j++) {
+        int local_idx = BLOCK_SIZE * j + tx - max_sample_count;
+        int target_idx = idx_offset + local_idx;
+        if (local_idx >= 0 && target_idx < neighbor_count) {
+          float thread_weight = edge_weight[start + target_idx];
+          weight_keys[j] = GenKeyFromWeight(thread_weight, rng);
+          neighbor_idxs[j] = target_idx;
+        }
+      }
+      const int iter_valid_count =
+          ((neighbor_count - idx_offset) >= stride)
+              ? (BLOCK_SIZE * ITEMS_PER_THREAD)
+              : (max_sample_count + neighbor_count - idx_offset);
+      BlockRadixTopKT{sort_tmp_storage}.radixTopKToStriped(
+          weight_keys, neighbor_idxs, max_sample_count, iter_valid_count);
+      __syncthreads();
+    }
+#pragma unroll
+    for (int j = 0; j < ITEMS_PER_THREAD; j++) {
+      int idx = j * BLOCK_SIZE + tx;
+      if (idx < max_sample_count) {
+        sample_output[offset + idx] = in_rows[start + neighbor_idxs[j]];
+        if (return_eids) {
+          out_eids[offset + idx] = eids[start + neighbor_idxs[j]];
+        }
+      }
+    }
+  }
+}
+#endif
+
+template <typename T, typename Context>
+void WeightedSampleNeighborsKernel(const Context& dev_ctx,
+                                   const DenseTensor& row,
+                                   const DenseTensor& col_ptr,
+                                   const DenseTensor& edge_weight,
+                                   const DenseTensor& x,
+                                   const paddle::optional<DenseTensor>& eids,
+                                   int sample_size,
+                                   bool return_eids,
+                                   DenseTensor* out,
+                                   DenseTensor* out_count,
+                                   DenseTensor* out_eids) {
+  auto* row_data = row.data<T>();
+  auto* col_ptr_data = col_ptr.data<T>();
+  auto* weights_data = edge_weight.data<float>();
+  auto* x_data = x.data<T>();
+  auto* eids_data =
+      (eids.get_ptr() == nullptr ? nullptr : eids.get_ptr()->data<T>());
+  int bs = x.dims()[0];
+
+  thread_local std::random_device rd;
+  thread_local std::mt19937 gen(rd());
+  thread_local std::uniform_int_distribution<unsigned long long>  // NOLINT
+      distrib;
+  unsigned long long random_seed = distrib(gen);  // NOLINT
+  const bool need_neighbor_count = sample_size > SAMPLE_SIZE_THRESHOLD;
+
+  out_count->Resize({bs});
+  int* out_count_data =
+      dev_ctx.template Alloc<int>(out_count);  // finally copy sample_count
+  int* neighbor_count_ptr = nullptr;
+  std::shared_ptr<phi::Allocation> neighbor_count;
+  auto sample_count = phi::memory_utils::Alloc(
+      dev_ctx.GetPlace(),
+      (bs + 1) * sizeof(int),
+      phi::Stream(reinterpret_cast<phi::StreamId>(dev_ctx.stream())));
+  int* sample_count_ptr = reinterpret_cast<int*>(sample_count->ptr());
+
+  int grid_size = (bs + 127) / 128;
+  if (need_neighbor_count) {
+    neighbor_count = phi::memory_utils::AllocShared(
+        dev_ctx.GetPlace(),
+        (bs + 1) * sizeof(int),
+        phi::Stream(reinterpret_cast<phi::StreamId>(dev_ctx.stream())));
+    neighbor_count_ptr = reinterpret_cast<int*>(neighbor_count->ptr());
+    GetSampleCountAndNeighborCountKernel<T, true>
+        <<<grid_size, 128, 0, dev_ctx.stream()>>>(col_ptr_data,
+                                                  x_data,
+                                                  sample_count_ptr,
+                                                  neighbor_count_ptr,
+                                                  sample_size,
+                                                  bs);
+  } else {
+    GetSampleCountAndNeighborCountKernel<T, false>
+        <<<grid_size, 128, 0, dev_ctx.stream()>>>(
+            col_ptr_data, x_data, sample_count_ptr, nullptr, sample_size, bs);
+  }
+
+  auto sample_offset = phi::memory_utils::Alloc(
+      dev_ctx.GetPlace(),
+      (bs + 1) * sizeof(int),
+      phi::Stream(reinterpret_cast<phi::StreamId>(dev_ctx.stream())));
+  int* sample_offset_ptr = reinterpret_cast<int*>(sample_offset->ptr());
+
+#ifdef PADDLE_WITH_CUDA
+  const auto& exec_policy = thrust::cuda::par.on(dev_ctx.stream());
+#else
+  const auto& exec_policy = thrust::hip::par.on(dev_ctx.stream());
+#endif
+  thrust::exclusive_scan(exec_policy,
+                         sample_count_ptr,
+                         sample_count_ptr + bs + 1,
+                         sample_offset_ptr);
+  int total_sample_size = 0;
+#ifdef PADDLE_WITH_CUDA
+  cudaMemcpyAsync(&total_sample_size,
+                  sample_offset_ptr + bs,
+                  sizeof(int),
+                  cudaMemcpyDeviceToHost,
+                  dev_ctx.stream());
+  cudaMemcpyAsync(out_count_data,
+                  sample_count_ptr,
+                  sizeof(int) * bs,
+                  cudaMemcpyDeviceToDevice,
+                  dev_ctx.stream());
+  cudaStreamSynchronize(dev_ctx.stream());
+#else
+  hipMemcpyAsync(&total_sample_size,
+                 sample_offset_ptr + bs,
+                 sizeof(int),
+                 hipMemcpyDeviceToHost,
+                 dev_ctx.stream());
+  hipMemcpyAsync(out_count_data,
+                 sample_count_ptr,
+                 sizeof(int) * bs,
+                 hipMemcpyDeviceToDevice,
+                 dev_ctx.stream());
+  hipStreamSynchronize(dev_ctx.stream());
+#endif
+
+  out->Resize({static_cast<int>(total_sample_size)});
+  T* out_data = dev_ctx.template Alloc<T>(out);
+  T* out_eids_data = nullptr;
+  if (return_eids) {
+    out_eids->Resize({static_cast<int>(total_sample_size)});
+    out_eids_data = dev_ctx.template Alloc<T>(out_eids);
+  }
+
+  // large sample size
+#ifdef PADDLE_WITH_CUDA
+  if (sample_size > SAMPLE_SIZE_THRESHOLD) {
+    thrust::exclusive_scan(exec_policy,
+                           neighbor_count_ptr,
+                           neighbor_count_ptr + bs + 1,
+                           neighbor_count_ptr);
+    int* neighbor_offset = neighbor_count_ptr;
+    int target_neighbor_counts;
+    cudaMemcpyAsync(&target_neighbor_counts,
+                    neighbor_offset + bs,
+                    sizeof(int),
+                    cudaMemcpyDeviceToHost,
+                    dev_ctx.stream());
+    cudaStreamSynchronize(dev_ctx.stream());
+
+    auto tmh_weights = phi::memory_utils::Alloc(
+        dev_ctx.GetPlace(),
+        target_neighbor_counts * sizeof(float),
+        phi::Stream(reinterpret_cast<phi::StreamId>(dev_ctx.stream())));
+    float* target_weights_keys_buf_ptr =
+        reinterpret_cast<float*>(tmh_weights->ptr());
+    constexpr int BLOCK_SIZE = 256;
+    WeightedSampleLargeKernel<T, BLOCK_SIZE>
+        <<<bs, BLOCK_SIZE, 0, dev_ctx.stream()>>>(out_data,
+                                                  sample_offset_ptr,
+                                                  neighbor_offset,
+                                                  target_weights_keys_buf_ptr,
+                                                  x_data,
+                                                  bs,
+                                                  row_data,
+                                                  col_ptr_data,
+                                                  weights_data,
+                                                  eids_data,
+                                                  sample_size,
+                                                  random_seed,
+                                                  out_eids_data,
+                                                  return_eids);
+    cudaStreamSynchronize(dev_ctx.stream());
+  } else if (sample_size <= 0) {
+    SampleAllKernel<T><<<bs, 64, 0, dev_ctx.stream()>>>(out_data,
+                                                        sample_offset_ptr,
+                                                        x_data,
+                                                        bs,
+                                                        row_data,
+                                                        col_ptr_data,
+                                                        eids_data,
+                                                        out_eids_data,
+                                                        return_eids);
+    cudaStreamSynchronize(dev_ctx.stream());
+  } else {  // sample_size < sample_count_threshold
+    using WeightedSampleFuncType = void (*)(T*,
+                                            const int*,
+                                            const T*,
+                                            int,
+                                            const T*,
+                                            const T*,
+                                            const float*,
+                                            const T*,
+                                            int,
+                                            unsigned long long,  // NOLINT
+                                            T*,
+                                            bool);
+    static const WeightedSampleFuncType func_array[7] = {
+        WeightedSampleKernel<T, 4, 128>,
+        WeightedSampleKernel<T, 6, 128>,
+        WeightedSampleKernel<T, 4, 256>,
+        WeightedSampleKernel<T, 5, 256>,
+        WeightedSampleKernel<T, 6, 256>,
+        WeightedSampleKernel<T, 8, 256>,
+        WeightedSampleKernel<T, 8, 512>,
+    };
+    const int block_sizes[7] = {128, 128, 256, 256, 256, 256, 512};
+    auto choose_func_idx = [](int sample_size) {
+      if (sample_size <= 128) {
+        return 0;
+      }
+      if (sample_size <= 384) {
+        return (sample_size - 129) / 64 + 4;
+      }
+      if (sample_size <= 512) {
+        return 5;
+      } else {
+        return 6;
+      }
+    };
+    int func_idx = choose_func_idx(sample_size);
+    int block_size = block_sizes[func_idx];
+    func_array[func_idx]<<<bs, block_size, 0, dev_ctx.stream()>>>(
+        out_data,
+        sample_offset_ptr,
+        x_data,
+        bs,
+        row_data,
+        col_ptr_data,
+        weights_data,
+        eids_data,
+        sample_size,
+        random_seed,
+        out_eids_data,
+        return_eids);
+    cudaStreamSynchronize(dev_ctx.stream());
+  }
+#endif
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(weighted_sample_neighbors,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::WeightedSampleNeighborsKernel,
+                   int,
+                   int64_t) {}
diff --git a/paddle/phi/kernels/weighted_sample_neighbors_kernel.h b/paddle/phi/kernels/weighted_sample_neighbors_kernel.h
new file mode 100644
index 0000000000000..2a0402f9fc494
--- /dev/null
+++ b/paddle/phi/kernels/weighted_sample_neighbors_kernel.h
@@ -0,0 +1,35 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void GraphWeightedSampleNeighborsKernel(
+    const Context& dev_ctx,
+    const DenseTensor& row,
+    const DenseTensor& col_ptr,
+    const DenseTensor& edge_weight,
+    const DenseTensor& x,
+    const paddle::optional<DenseTensor>& eids,
+    int sample_size,
+    bool return_eids,
+    DenseTensor* out,
+    DenseTensor* out_count,
+    DenseTensor* out_eids);
+
+}  // namespace phi
diff --git a/python/paddle/fluid/tests/unittests/test_weighted_sample_neighbors.py b/python/paddle/fluid/tests/unittests/test_weighted_sample_neighbors.py
new file mode 100644
index 0000000000000..8be782b9adf29
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_weighted_sample_neighbors.py
@@ -0,0 +1,217 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+import paddle
+
+
+class TestWeightedSampleNeighbors(unittest.TestCase):
+    def setUp(self):
+        num_nodes = 20
+        edges = np.random.randint(num_nodes, size=(100, 2))
+        edges = np.unique(edges, axis=0)
+        self.edges_id = np.arange(0, len(edges)).astype("int64")
+        sorted_edges = edges[np.argsort(edges[:, 1])]
+
+        # Calculate dst index cumsum counts, also means colptr
+        dst_count = np.zeros(num_nodes)
+        dst_src_dict = {}
+        for dst in range(0, num_nodes):
+            true_index = sorted_edges[:, 1] == dst
+            dst_count[dst] = np.sum(true_index)
+            dst_src_dict[dst] = sorted_edges[:, 0][true_index]
+        dst_count = dst_count.astype("int64")
+        colptr = np.cumsum(dst_count)
+        colptr = np.insert(colptr, 0, 0)
+
+        self.row = sorted_edges[:, 0].astype("int64")
+        self.colptr = colptr.astype("int64")
+        self.nodes = np.unique(np.random.randint(num_nodes, size=5)).astype(
+            "int64"
+        )
+        self.weight = np.ones(self.row.shape[0]).astype("float32")
+        self.sample_size = 5
+        self.dst_src_dict = dst_src_dict
+
+    def test_sample_result(self):
+        paddle.disable_static()
+        row = paddle.to_tensor(self.row)
+        colptr = paddle.to_tensor(self.colptr)
+        nodes = paddle.to_tensor(self.nodes)
+        weight = paddle.to_tensor(self.weight)
+
+        out_neighbors, out_count = paddle.geometric.weighted_sample_neighbors(
+            row, colptr, weight, nodes, sample_size=self.sample_size
+        )
+        out_count_cumsum = paddle.cumsum(out_count)
+        for i in range(len(out_count)):
+            if i == 0:
+                neighbors = out_neighbors[0 : out_count_cumsum[i]]
+            else:
+                neighbors = out_neighbors[
+                    out_count_cumsum[i - 1] : out_count_cumsum[i]
+                ]
+            # Ensure the correct sample size.
+            self.assertTrue(
+                out_count[i] == self.sample_size
+                or out_count[i] == len(self.dst_src_dict[self.nodes[i]])
+            )
+            # Ensure no repetitive sample neighbors.
+            self.assertTrue(
+                neighbors.shape[0] == paddle.unique(neighbors).shape[0]
+            )
+            # Ensure the correct sample neighbors.
+            in_neighbors = np.isin(
+                neighbors.numpy(), self.dst_src_dict[self.nodes[i]]
+            )
+            self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0])
+
+    def test_sample_result_static(self):
+        paddle.enable_static()
+        with paddle.static.program_guard(paddle.static.Program()):
+            row = paddle.static.data(
+                name="row", shape=self.row.shape, dtype=self.row.dtype
+            )
+            colptr = paddle.static.data(
+                name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype
+            )
+            weight = paddle.static.data(
+                name="weight", shape=self.weight.shape, dtype=self.weight.dtype
+            )
+            nodes = paddle.static.data(
+                name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype
+            )
+
+            (
+                out_neighbors,
+                out_count,
+            ) = paddle.geometric.weighted_sample_neighbors(
+                row, colptr, weight, nodes, sample_size=self.sample_size
+            )
+            exe = paddle.static.Executor(paddle.CPUPlace())
+            ret = exe.run(
+                feed={
+                    'row': self.row,
+                    'colptr': self.colptr,
+                    'weight': self.weight,
+                    'nodes': self.nodes,
+                },
+                fetch_list=[out_neighbors, out_count],
+            )
+            out_neighbors, out_count = ret
+            out_count_cumsum = np.cumsum(out_count)
+            out_neighbors = np.split(out_neighbors, out_count_cumsum)[:-1]
+            for neighbors, node, count in zip(
+                out_neighbors, self.nodes, out_count
+            ):
+                self.assertTrue(
+                    count == self.sample_size
+                    or count == len(self.dst_src_dict[node])
+                )
+                self.assertTrue(
+                    neighbors.shape[0] == np.unique(neighbors).shape[0]
+                )
+                in_neighbors = np.isin(neighbors, self.dst_src_dict[node])
+                self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0])
+
+    def test_raise_errors(self):
+        paddle.disable_static()
+        row = paddle.to_tensor(self.row)
+        colptr = paddle.to_tensor(self.colptr)
+        weight = paddle.to_tensor(self.weight)
+        nodes = paddle.to_tensor(self.nodes)
+
+        def check_eid_error():
+            paddle.geometric.weighted_sample_neighbors(
+                row,
+                colptr,
+                weight,
+                nodes,
+                sample_size=self.sample_size,
+                return_eids=True,
+            )
+
+        self.assertRaises(ValueError, check_eid_error)
+
+    def test_sample_result_with_eids(self):
+        paddle.disable_static()
+        row = paddle.to_tensor(self.row)
+        colptr = paddle.to_tensor(self.colptr)
+        weight = paddle.to_tensor(self.weight)
+        nodes = paddle.to_tensor(self.nodes)
+        eids = paddle.to_tensor(self.edges_id)
+
+        (
+            out_neighbors,
+            out_count,
+            out_eids,
+        ) = paddle.geometric.weighted_sample_neighbors(
+            row,
+            colptr,
+            weight,
+            nodes,
+            eids=eids,
+            sample_size=self.sample_size,
+            return_eids=True,
+        )
+
+        paddle.enable_static()
+        with paddle.static.program_guard(paddle.static.Program()):
+            row = paddle.static.data(
+                name="row", shape=self.row.shape, dtype=self.row.dtype
+            )
+            colptr = paddle.static.data(
+                name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype
+            )
+            weight = paddle.static.data(
+                name="weight", shape=self.weight.shape, dtype=self.weight.dtype
+            )
+            nodes = paddle.static.data(
+                name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype
+            )
+            eids = paddle.static.data(
+                name="eids", shape=self.edges_id.shape, dtype=self.nodes.dtype
+            )
+
+            (
+                out_neighbors,
+                out_count,
+                out_eids,
+            ) = paddle.geometric.weighted_sample_neighbors(
+                row,
+                colptr,
+                weight,
+                nodes,
+                sample_size=self.sample_size,
+                eids=eids,
+                return_eids=True,
+            )
+            exe = paddle.static.Executor(paddle.CPUPlace())
+            ret = exe.run(
+                feed={
+                    'row': self.row,
+                    'colptr': self.colptr,
+                    'weight': self.weight,
+                    'nodes': self.nodes,
+                    'eids': self.edges_id,
+                },
+                fetch_list=[out_neighbors, out_count, out_eids],
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/python/paddle/geometric/__init__.py b/python/paddle/geometric/__init__.py
index 9618bc57a203e..6c132a529bc37 100644
--- a/python/paddle/geometric/__init__.py
+++ b/python/paddle/geometric/__init__.py
@@ -22,6 +22,7 @@
 from .reindex import reindex_graph  # noqa: F401
 from .reindex import reindex_heter_graph  # noqa: F401
 from .sampling import sample_neighbors  # noqa: F401
+from .sampling import weighted_sample_neighbors  # noqa: F401
 
 __all__ = [
     'send_u_recv',
@@ -34,4 +35,5 @@
     'reindex_graph',
     'reindex_heter_graph',
     'sample_neighbors',
+    'weighted_sample_neighbors',
 ]
diff --git a/python/paddle/geometric/sampling/__init__.py b/python/paddle/geometric/sampling/__init__.py
index 2e5b24fdd60b7..ee7bacfc9047f 100644
--- a/python/paddle/geometric/sampling/__init__.py
+++ b/python/paddle/geometric/sampling/__init__.py
@@ -13,5 +13,6 @@
 # limitations under the License.
 
 from .neighbors import sample_neighbors  # noqa: F401
+from .neighbors import weighted_sample_neighbors  # noqa: F401
 
 __all__ = []
diff --git a/python/paddle/geometric/sampling/neighbors.py b/python/paddle/geometric/sampling/neighbors.py
index 093fd39617af3..c8d907c078bad 100644
--- a/python/paddle/geometric/sampling/neighbors.py
+++ b/python/paddle/geometric/sampling/neighbors.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from paddle import _legacy_C_ops
+from paddle import _C_ops, _legacy_C_ops
 from paddle.fluid.data_feeder import check_variable_and_dtype
-from paddle.fluid.framework import _non_static_mode
+from paddle.fluid.framework import _non_static_mode, in_dygraph_mode
 from paddle.fluid.layer_helper import LayerHelper
 
 __all__ = []
@@ -170,3 +170,151 @@ def sample_neighbors(
     if return_eids:
         return out_neighbors, out_count, out_eids
     return out_neighbors, out_count
+
+
+def weighted_sample_neighbors(
+    row,
+    colptr,
+    edge_weight,
+    input_nodes,
+    sample_size=-1,
+    eids=None,
+    return_eids=False,
+    name=None,
+):
+    """
+    Graph Weighted Sample Neighbors API.
+
+    This API is mainly used in Graph Learning domain, and the main purpose is to
+    provide high performance of graph weighted-sampling method. For example, we get the
+    CSC(Compressed Sparse Column) format of the input graph edges as `row` and
+    `colptr`, so as to convert graph data into a suitable format for sampling, and the
+    input `edge_weight` should also match the CSC format. Besides, `input_nodes` means
+    the nodes we need to sample neighbors, and `sample_sizes` means the number of neighbors
+    and number of layers we want to sample. This API will finally return the weighted sampled
+    neighbors, and the probability of being selected as a neighbor is related to its weight,
+    with higher weight and higher probability.
+
+    Args:
+        row (Tensor): One of the components of the CSC format of the input graph, and
+                      the shape should be [num_edges, 1] or [num_edges]. The available
+                      data type is int32, int64.
+        colptr (Tensor): One of the components of the CSC format of the input graph,
+                         and the shape should be [num_nodes + 1, 1] or [num_nodes + 1].
+                         The data type should be the same with `row`.
+        edge_weight (Tensor): The edge weight of the CSC format graph edges. And the shape
+                              should be [num_edges, 1] or [num_edges]. The available data
+                              type is float32.
+        input_nodes (Tensor): The input nodes we need to sample neighbors for, and the
+                              data type should be the same with `row`.
+        sample_size (int, optional): The number of neighbors we need to sample. Default value is -1,
+                           which means returning all the neighbors of the input nodes.
+        eids (Tensor, optional): The eid information of the input graph. If return_eids is True,
+                            then `eids` should not be None. The data type should be the
+                            same with `row`. Default is None.
+        return_eids (bool, optional): Whether to return eid information of sample edges. Default is False.
+        name (str, optional): Name for the operation (optional, default is None).
+                              For more information, please refer to :ref:`api_guide_Name`.
+
+    Returns:
+        - out_neighbors (Tensor), the sample neighbors of the input nodes.
+
+        - out_count (Tensor), the number of sampling neighbors of each input node, and the shape
+          should be the same with `input_nodes`.
+
+        - out_eids (Tensor), if `return_eids` is True, we will return the eid information of the
+          sample edges.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+
+            # edges: (3, 0), (7, 0), (0, 1), (9, 1), (1, 2), (4, 3), (2, 4),
+            #        (9, 5), (3, 5), (9, 6), (1, 6), (9, 8), (7, 8)
+            row = [3, 7, 0, 9, 1, 4, 2, 9, 3, 9, 1, 9, 7]
+            colptr = [0, 2, 4, 5, 6, 7, 9, 11, 11, 13, 13]
+            weight = [0.1, 0.5, 0.2, 0.5, 0.9, 1.9, 2.0, 2.1, 0.01, 0.9, 0,12, 0.59, 0.67]
+            nodes = [0, 8, 1, 2]
+            sample_size = 2
+            row = paddle.to_tensor(row, dtype="int64")
+            colptr = paddle.to_tensor(colptr, dtype="int64")
+            weight = paddle.to_tensor(weight, dtype="float32")
+            nodes = paddle.to_tensor(nodes, dtype="int64")
+            out_neighbors, out_count = paddle.geometric.weighted_sample_neighbors(row, colptr, weight, nodes, sample_size=sample_size)
+
+    """
+
+    if return_eids:
+        if eids is None:
+            raise ValueError(
+                "`eids` should not be None if `return_eids` is True."
+            )
+
+    if in_dygraph_mode():
+        (
+            out_neighbors,
+            out_count,
+            out_eids,
+        ) = _C_ops.weighted_sample_neighbors(
+            row,
+            colptr,
+            edge_weight,
+            input_nodes,
+            eids,
+            sample_size,
+            return_eids,
+        )
+        if return_eids:
+            return out_neighbors, out_count, out_eids
+        return out_neighbors, out_count
+
+    check_variable_and_dtype(
+        row, "row", ("int32", "int64"), "weighted_sample_neighbors"
+    )
+    check_variable_and_dtype(
+        colptr, "colptr", ("int32", "int64"), "weighted_sample_neighbors"
+    )
+    check_variable_and_dtype(
+        edge_weight,
+        "edge_weight",
+        ("float32"),
+        "weighted_sample_neighbors",
+    )
+    check_variable_and_dtype(
+        input_nodes,
+        "input_nodes",
+        ("int32", "int64"),
+        "weighted_sample_neighbors",
+    )
+    if return_eids:
+        check_variable_and_dtype(
+            eids, "eids", ("int32", "int64"), "weighted_sample_neighbors"
+        )
+
+    helper = LayerHelper("weighted_sample_neighbors", **locals())
+    out_neighbors = helper.create_variable_for_type_inference(dtype=row.dtype)
+    out_count = helper.create_variable_for_type_inference(dtype=row.dtype)
+    out_eids = helper.create_variable_for_type_inference(dtype=row.dtype)
+    helper.append_op(
+        type="weighted_sample_neighbors",
+        inputs={
+            "row": row,
+            "colptr": colptr,
+            "edge_weight": edge_weight,
+            "input_nodes": input_nodes,
+            "eids": eids if return_eids else None,
+        },
+        outputs={
+            "out_neighbors": out_neighbors,
+            "out_count": out_count,
+            "out_eids": out_eids,
+        },
+        attrs={
+            "sample_size": sample_size,
+            "return_eids": return_eids,
+        },
+    )
+    if return_eids:
+        return out_neighbors, out_count, out_eids
+    return out_neighbors, out_count

From fb5fab079c3225ba469198b2439d2ab3b9b80002 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?=
 <83450930+Liyulingyue@users.noreply.github.com>
Date: Mon, 24 Apr 2023 16:13:17 +0800
Subject: [PATCH 018/405] Update sharding_optimizer.py (#53164)

---
 .../distributed/fleet/meta_optimizers/sharding_optimizer.py    | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py
index 25f9928c826a9..88ad5cbfdc32c 100755
--- a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py
+++ b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py
@@ -755,9 +755,6 @@ def _init_pipeline_comm(self, startup_block):
                 sync=False,
             )
 
-        if core.is_compiled_with_custom_device('npu'):
-            return
-
         # GPU
         for pair in self.pipeline_pair:
             pair_key = pair[0] * 1000 + pair[1]

From f2c595a6dc6598afbb93b0c5eaa51377e440a251 Mon Sep 17 00:00:00 2001
From: Wang Xin <xinwang614@gmail.com>
Date: Mon, 24 Apr 2023 16:14:05 +0800
Subject: [PATCH 019/405] remove ASCEND* keyword in doc and cmakelists (#53132)

* remove ASCEND* keyword in doc

* fix PR-CI-Codestyle-Check error
---
 .../fluid/tests/unittests/CMakeLists.txt      |  2 --
 .../tests/unittests/collective/CMakeLists.txt |  2 +-
 .../tests/unittests/collective/README.md      |  2 +-
 .../unittests/collective/fleet/testslist.csv  | 28 +++++++++----------
 .../tests/unittests/collective/testslist.csv  |  2 +-
 5 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt
index 70e3e8a550e56..670a48faf5c9b 100755
--- a/python/paddle/fluid/tests/unittests/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt
@@ -38,8 +38,6 @@ list(APPEND MIXED_DIST_TEST_OPS test_fleetrun)
 list(APPEND MIXED_DIST_TEST_OPS test_fleet_run_random_port)
 list(APPEND MIXED_DIST_TEST_OPS test_fleet_launch_async)
 list(APPEND MIXED_DIST_TEST_OPS test_fleet_launch_cloud)
-list(APPEND MIXED_DIST_TEST_OPS test_fleet_launch_ascend)
-list(APPEND MIXED_DIST_TEST_OPS test_ascend_group)
 list(APPEND MIXED_DIST_TEST_OPS test_fleet_launch_nproc)
 
 list(APPEND MIXED_DIST_TEST_OPS test_fleet_base)
diff --git a/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt b/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt
index 08c72f79055af..1aec76f705236 100644
--- a/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt
@@ -215,7 +215,7 @@ if((WITH_GPU OR WITH_ROCM) AND (LINUX))
     test_collective_reduce_scatter_api ENVS
     "http_proxy=;https_proxy=;PYTHONPATH=..:${PADDLE_BINARY_DIR}/python")
   set_tests_properties(test_collective_reduce_scatter_api
-                       PROPERTIES TIMEOUT "150" LABELS "RUN_TYPE=DIST")
+                       PROPERTIES TIMEOUT "120" LABELS "RUN_TYPE=DIST")
 endif()
 if((WITH_GPU OR WITH_ROCM) AND (LINUX))
   py_test_modules(
diff --git a/python/paddle/fluid/tests/unittests/collective/README.md b/python/paddle/fluid/tests/unittests/collective/README.md
index a06cf6dc39640..2bf9491814184 100644
--- a/python/paddle/fluid/tests/unittests/collective/README.md
+++ b/python/paddle/fluid/tests/unittests/collective/README.md
@@ -7,7 +7,7 @@
     the properties are the following:
 * `name`: the test's name
 * `os`: The supported operator system, ignoring case. If the test run in multiple operator systems, use ";" to split systems, for example, `apple;linux` means the test runs on both Apple and Linux. The supported values are `linux`,`win32` and `apple`. If the value is empty, this means the test runs on all opertaor systems.
-* `arch`: the device's architecture. similar to `os`, multiple valuse ars splited by ";" and ignoring case. The supported architectures are `gpu`, `xpu`, `ASCEND`, `ASCEND_CL` and `rocm`.
+* `arch`: the device's architecture. similar to `os`, multiple valuse ars splited by ";" and ignoring case. The supported architectures are `gpu`, `xpu` and `rocm`.
 * `timeout`: timeout of a unittest, whose unit is second. Blank means default.
 * `run_type`: run_type of a unittest. Supported values are `NIGHTLY`, `EXCLUSIVE`, `CINN`, `DIST`, `GPUPS`, `INFER`, `EXCLUSIVE:NIGHTLY`, `DIST:NIGHTLY`，which are case-insensitive.
 * `launcher`: the test launcher.Supported values are test_runner.py, dist_test.sh and custom scripts' name. Blank means test_runner.py.
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv b/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv
index 9ac8fdf65729e..459a3e39df8e8 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv
@@ -1,5 +1,5 @@
 name,os,arch,timeout,run_type,launcher,num_port,run_serial,envs,conditions
-test_fleet_sharding_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,350,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_sharding_meta_optimizer,,GPU;XPU,350,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_static_mp_layers,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_dgc_op,,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC
 test_dgc_optimizer,,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC
@@ -7,14 +7,14 @@ test_parallel_margin_cross_entropy,,GPU,120,DIST,../../dist_test.sh,2,,http_prox
 test_parallel_dygraph_transformer,,GPU,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL;${NCCL_VERSION} VERSION_GREATER_EQUAL 2212
 test_parallel_dygraph_transformer,,ROCM,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_fp16_allreduce_meta_optimizer,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_rnn_dp,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_rnn_dp,,GPU;XPU,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_mp_layers,,GPU,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL
 test_tcp_store,LINUX;APPLE,,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_dygraph_sharding_stage3_for_eager,,,350,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_communicator_half_async,,,120,DIST,test_runner.py,2,,FLAGS_communicator_send_queue_size=1;FLAGS_communicator_max_merge_var_num=1;http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL
 test_parallel_dygraph_pipeline_parallel,,GPU,500,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_pipeline_parallel_with_virtual_stage,,GPU,500,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_localsgd_meta_optimizer,LINUX,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_localsgd_meta_optimizer,LINUX,GPU;XPU,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_class_center_sample,,GPU,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL
 test_pipeline,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_utils,LINUX;APPLE,,120,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
@@ -22,40 +22,40 @@ test_static_model_parallel,,,240,DIST,../../dist_test.sh,2,,http_proxy=;https_pr
 test_parallel_dygraph_no_sync,,GPU,300,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL
 test_dygraph_sharding_stage2,,,200,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_control_flow,,,350,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_lars_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_lars_meta_optimizer,,GPU;XPU,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_hybrid_parallel_inference_helper,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_rolemaker_new,,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_dist_mnist_gradient_merge,LINUX;WIN32,GPU;ROCM,360,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_recv_save_op,,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_communicator_sync,,,,,test_runner.py,2,,FLAGS_communicator_send_queue_size=1;FLAGS_communicator_max_merge_var_num=1;http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_pipeline_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_gradient_merge_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_pipeline_meta_optimizer,,GPU;XPU,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_gradient_merge_meta_optimizer,,GPU;XPU,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_amp_init,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_meta_optimizer_base,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_raw_program_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_raw_program_meta_optimizer,,GPU;XPU,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_sharding_parallel,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_tensor_parallel,,,200,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_dygraph_group_sharded_api_for_eager,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_distributed_strategy,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_dgc_meta_optimizer,,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC
 test_parallel_dygraph_unused_variables,,,350,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_lamb_meta_optimizer,LINUX,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_lamb_meta_optimizer,LINUX,GPU;XPU,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_dgc_momentum_op,,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC
 test_parallel_dygraph_no_sync_gradient_check,,,60,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_pipeline_meta_optimizer_with_recompute,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_hybrid_meta_optimizer,LINUX;WIN32,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_pipeline_meta_optimizer_with_recompute,,GPU;XPU,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_hybrid_meta_optimizer,LINUX;WIN32,GPU;XPU,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_qat,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_sparse_embedding,,GPU,200,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL;${NCCL_VERSION} VERSION_GREATER_EQUAL 2212
 test_parallel_dygraph_sparse_embedding,,ROCM,200,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_amp_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_amp_meta_optimizer,,GPU;XPU,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_sparse_embedding_over_height,,GPU,150,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL;${NCCL_VERSION} VERSION_GREATER_EQUAL 2212
 test_parallel_dygraph_sparse_embedding_over_height,,ROCM,350,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_distributed_strategy,LINUX;APPLE,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_auto_parallel_parallelizer,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_recompute_meta_optimizer,LINUX;WIN32,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_recompute_meta_optimizer,LINUX;WIN32,GPU;XPU,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_private_function,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_new_group,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,test_new_group.sh,2,,http_proxy=;https_proxy=,
-test_c_comm_init_op,LINUX,GPU;XPU;ASCEND;ASCEND_CL,120,DIST,test_c_comm_init_op.sh,2,,http_proxy=;https_proxy=,
+test_new_group,,GPU;XPU,,DIST,test_new_group.sh,2,,http_proxy=;https_proxy=,
+test_c_comm_init_op,LINUX,GPU;XPU,120,DIST,test_c_comm_init_op.sh,2,,http_proxy=;https_proxy=,
 test_fused_attention_pass_with_mp,LINUX,GPU,120,DIST,test_fused_attention_pass_with_mp.sh,2,,http_proxy=;https_proxy=,
 test_ir_pass_pipeline,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_mnist,,GPU;ROCM,200,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
diff --git a/python/paddle/fluid/tests/unittests/collective/testslist.csv b/python/paddle/fluid/tests/unittests/collective/testslist.csv
index cf2b6c6757b30..7d8e32998232e 100644
--- a/python/paddle/fluid/tests/unittests/collective/testslist.csv
+++ b/python/paddle/fluid/tests/unittests/collective/testslist.csv
@@ -45,7 +45,7 @@ test_communication_stream_reduce_scatter_api,linux,gpu;rocm,120,DIST,,2,,PYTHONP
 test_communication_stream_scatter_api,linux,gpu;rocm,120,DIST,,2,,PYTHONPATH=..;http_proxy=;https_proxy=,
 test_communication_stream_sendrecv_api,linux,gpu;rocm,120,DIST,,2,,PYTHONPATH=..;http_proxy=;https_proxy=,
 test_eager_dist_api,linux,gpu;rocm,120,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=..,
-test_gen_nccl_id_op,,gpu;rocm;ASCEND;ASCEND_CL,,DIST,../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=..,
+test_gen_nccl_id_op,,gpu;rocm,,DIST,../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=..,
 test_new_group_api,linux,gpu;rocm,120,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=..,
 test_world_size_and_rank,linux,rocm;gpu,120,DIST,test_world_size_and_rank.sh,2,,http_proxy=;https_proxy=,
 test_mpi_comm,linux,,,DIST,test_mpi_comm.sh,2,,http_proxy=;https_proxy=,WITH_MPI

From a85e038a22df9246476f9818c1275c464d5393f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?=
 <83450930+Liyulingyue@users.noreply.github.com>
Date: Mon, 24 Apr 2023 16:16:04 +0800
Subject: [PATCH 020/405] rm is_npu_place (#53105)

---
 paddle/fluid/eager/amp_auto_cast.h                  |  1 -
 paddle/fluid/eager/amp_utils.h                      |  1 -
 paddle/fluid/eager/eager_amp_auto_cast.h            |  1 -
 paddle/fluid/framework/data_device_transform.cc     |  2 +-
 .../new_executor/interpreter/data_transfer.cc       |  4 +---
 .../new_executor/interpreter/interpreter_util.cc    |  5 ++---
 .../new_executor/interpreter/stream_analyzer.cc     |  7 ++-----
 paddle/fluid/framework/parallel_executor.cc         |  2 +-
 paddle/fluid/framework/tensor_util.cc               | 13 +++----------
 paddle/fluid/imperative/amp_auto_cast.cc            |  2 --
 paddle/fluid/imperative/reducer.cc                  |  3 ---
 paddle/fluid/imperative/tracer.cc                   |  8 --------
 paddle/fluid/inference/api/analysis_predictor.cc    |  6 ------
 .../fluid/inference/api/details/zero_copy_tensor.cc |  2 --
 paddle/fluid/operators/array_operator.h             |  1 -
 paddle/fluid/operators/beam_search_decode_op.h      |  6 ++----
 .../operators/controlflow/conditional_block_op.h    |  1 -
 .../fluid/operators/controlflow/while_op_helper.cc  |  1 -
 .../operators/generator/get_expected_kernel_func.cc |  1 -
 paddle/fluid/operators/reader/buffered_reader.cc    |  2 --
 paddle/fluid/operators/reduce_ops/reduce_op.h       |  1 -
 paddle/fluid/operators/softmax_op.cc                |  2 --
 paddle/fluid/operators/tile_op_functor.h            |  6 ++----
 paddle/fluid/platform/device_context.cc             |  2 --
 paddle/fluid/platform/place.cc                      |  8 --------
 paddle/fluid/platform/place.h                       |  1 -
 paddle/fluid/pybind/eager.cc                        |  2 --
 paddle/fluid/pybind/eager_math_op_patch.cc          |  2 --
 paddle/fluid/pybind/imperative.cc                   |  2 --
 paddle/fluid/pybind/place.cc                        |  2 --
 paddle/fluid/pybind/tensor_py.h                     |  4 +---
 31 files changed, 15 insertions(+), 86 deletions(-)

diff --git a/paddle/fluid/eager/amp_auto_cast.h b/paddle/fluid/eager/amp_auto_cast.h
index 899216b70d20f..ab06e3d33e542 100644
--- a/paddle/fluid/eager/amp_auto_cast.h
+++ b/paddle/fluid/eager/amp_auto_cast.h
@@ -27,7 +27,6 @@ static inline bool NeedCast(const paddle::Tensor& tensor,
   if (paddle::platform::is_gpu_place(place) ||
       paddle::platform::is_cuda_pinned_place(place) ||
       paddle::platform::is_xpu_place(place) ||
-      paddle::platform::is_npu_place(place) ||
       paddle::platform::is_npu_pinned_place(place) ||
       paddle::platform::is_custom_place(place)) {
     // CudaPinndePlace is added for varbase created by dataloader
diff --git a/paddle/fluid/eager/amp_utils.h b/paddle/fluid/eager/amp_utils.h
index 95c77cf8e32df..ac9edc569df9f 100644
--- a/paddle/fluid/eager/amp_utils.h
+++ b/paddle/fluid/eager/amp_utils.h
@@ -96,7 +96,6 @@ inline phi::DataType GetDtypeWithPlace(
       is_right_place = (paddle::platform::is_gpu_place(place) ||
                         paddle::platform::is_cuda_pinned_place(place) ||
                         paddle::platform::is_xpu_place(place) ||
-                        paddle::platform::is_npu_place(place) ||
                         paddle::platform::is_npu_pinned_place(place) ||
                         paddle::platform::is_custom_place(place));
       if (is_right_place) {
diff --git a/paddle/fluid/eager/eager_amp_auto_cast.h b/paddle/fluid/eager/eager_amp_auto_cast.h
index c8088eec0f7ab..b4311cddc7e3e 100644
--- a/paddle/fluid/eager/eager_amp_auto_cast.h
+++ b/paddle/fluid/eager/eager_amp_auto_cast.h
@@ -27,7 +27,6 @@ static inline bool NeedCast(const paddle::Tensor& tensor,
   if (paddle::platform::is_gpu_place(place) ||
       paddle::platform::is_cuda_pinned_place(place) ||
       paddle::platform::is_xpu_place(place) ||
-      paddle::platform::is_npu_place(place) ||
       paddle::platform::is_npu_pinned_place(place) ||
       paddle::platform::is_custom_place(place) ||
       paddle::platform::is_cpu_place(place)) {
diff --git a/paddle/fluid/framework/data_device_transform.cc b/paddle/fluid/framework/data_device_transform.cc
index c8c92e95ea3a5..f40710f50bc28 100644
--- a/paddle/fluid/framework/data_device_transform.cc
+++ b/paddle/fluid/framework/data_device_transform.cc
@@ -29,7 +29,7 @@ void TransDataDevice(const phi::DenseTensor &in,
                                     "supported between CPU and CUDA."));
 
   // NOTE(zhiqiu): Special case for CPU->NPU, avoid stream sync.
-  if (platform::is_cpu_place(in.place()) && platform::is_npu_place(dst_place)) {
+  if (platform::is_cpu_place(in.place())) {
     paddle::framework::TensorCopy(
         in,
         dst_place,
diff --git a/paddle/fluid/framework/new_executor/interpreter/data_transfer.cc b/paddle/fluid/framework/new_executor/interpreter/data_transfer.cc
index 0db7313bafc1f..6d0615a4b7666 100644
--- a/paddle/fluid/framework/new_executor/interpreter/data_transfer.cc
+++ b/paddle/fluid/framework/new_executor/interpreter/data_transfer.cc
@@ -227,8 +227,7 @@ void DataTranferHelper::RunAndConstructOpFuncNode(
 
   // NOTE(winter-wang): in npu and custom device, D2H kernel is asynchronous.
   // need to explicit synchronization.
-  if ((platform::is_npu_place(place) || platform::is_custom_place(place)) &&
-      op_type == kMemcpyD2H) {
+  if ((platform::is_custom_place(place)) && op_type == kMemcpyD2H) {
     dev_ctx->Wait();
   }
 
@@ -419,7 +418,6 @@ std::shared_ptr<OperatorBase> TransferDevice(const std::string& var_name,
   if (IsSupportedHeterPlace(dst_place)) {
     op_type = kMemcpyH2D;
     int dst_place_type = platform::is_gpu_place(dst_place)      ? 0
-                         : platform::is_npu_place(dst_place)    ? 1
                          : platform::is_ipu_place(dst_place)    ? 3
                          : platform::is_xpu_place(dst_place)    ? 2
                          : platform::is_custom_place(dst_place) ? 6
diff --git a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
index 513b5f5cb3f87..090e973155334 100644
--- a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
+++ b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
@@ -146,9 +146,8 @@ bool IsGradOp(const std::string& op_name) {
 }
 
 bool IsSupportedHeterPlace(const phi::Place& place) {
-  return platform::is_gpu_place(place) || platform::is_npu_place(place) ||
-         platform::is_xpu_place(place) || platform::is_ipu_place(place) ||
-         platform::is_custom_place(place);
+  return platform::is_gpu_place(place) || platform::is_xpu_place(place) ||
+         platform::is_ipu_place(place) || platform::is_custom_place(place);
 }
 
 bool IsMemcpyD2H(const Instruction& instr) {
diff --git a/paddle/fluid/framework/new_executor/interpreter/stream_analyzer.cc b/paddle/fluid/framework/new_executor/interpreter/stream_analyzer.cc
index 6401248a2ff54..37d0c41953f96 100644
--- a/paddle/fluid/framework/new_executor/interpreter/stream_analyzer.cc
+++ b/paddle/fluid/framework/new_executor/interpreter/stream_analyzer.cc
@@ -152,8 +152,7 @@ DeviceContext* StreamAnalyzer::ParseDeviceContext(
 
   // only gpu/npu need update. xpu not need, because xpu memcpy op kernel is
   // synchronous.
-  if (platform::is_gpu_place(place_) || platform::is_npu_place(place_) ||
-      platform::is_custom_place(place_)) {
+  if (platform::is_gpu_place(place_) || platform::is_custom_place(place_)) {
     VLOG(6) << "Parse DeviceContext for " << op_type
             << ", execution stream = " << execution_stream;
     if (execution_stream != kDefaultStream) {
@@ -447,8 +446,6 @@ platform::DeviceType StreamAnalyzer::GetWaiterType(
   } else {
     if (platform::is_xpu_place(place_)) {
       return platform::kXPU;
-    } else if (platform::is_npu_place(place_)) {
-      return platform::kNPU;
     } else if (platform::is_custom_place(place_)) {
       return platform::kCUSTOM_DEVICE;
     }
@@ -464,7 +461,7 @@ DownstreamRunType StreamAnalyzer::AnalyseRunTypeForTwoInstructions(
   }
 
   // npu d2h kernel is asynchronous.
-  if (platform::is_npu_place(place_) || platform::is_custom_place(place_)) {
+  if (platform::is_custom_place(place_)) {
     if (interpreter::IsCpuOp(cur_instr) ||
         interpreter::IsMemcpyH2D(next_instr)) {
       return DownstreamRunType::kDirectRun;
diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index ce0c2891c7dd6..675ec593d9366 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -672,7 +672,7 @@ ParallelExecutor::ParallelExecutor(const std::vector<platform::Place> &places,
                                    const BuildStrategy &build_strategy,
                                    ir::Graph *graph)
     : member_(new ParallelExecutorPrivate(places, scope)) {
-  PADDLE_ENFORCE_EQ(places.size() > 0 && !platform::is_npu_place(places[0]),
+  PADDLE_ENFORCE_EQ(places.size() > 0,
                     true,
                     platform::errors::Unavailable(
                         "NPU is not supported in ParallelExecutor."));
diff --git a/paddle/fluid/framework/tensor_util.cc b/paddle/fluid/framework/tensor_util.cc
index 38e33520f53ce..28a8d9564ec0b 100644
--- a/paddle/fluid/framework/tensor_util.cc
+++ b/paddle/fluid/framework/tensor_util.cc
@@ -275,7 +275,7 @@ void TensorCopyImpl(const TENSOR& src,
                     TENSOR* dst) {
   platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
   const platform::DeviceContext* dev_ctx;
-  if (platform::is_gpu_place(dst_place) || platform::is_npu_place(dst_place) ||
+  if (platform::is_gpu_place(dst_place) ||
       platform::is_custom_place(dst_place)) {
     dev_ctx = pool.Get(dst_place);
   } else {
@@ -614,7 +614,6 @@ void TensorFromStream(std::istream& is,
     size_t size = tensor->numel() * framework::SizeOfType(desc.data_type());
     if (platform::is_gpu_place(dev_ctx.GetPlace()) ||
         platform::is_xpu_place(dev_ctx.GetPlace()) ||
-        platform::is_npu_place(dev_ctx.GetPlace()) ||
         platform::is_custom_place(dev_ctx.GetPlace())) {
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) || \
     defined(PADDLE_WITH_XPU) || defined(PADDLE_WITH_CUSTOM_DEVICE)
@@ -626,8 +625,7 @@ void TensorFromStream(std::istream& is,
       is.read(static_cast<char*>(buf), size);
       auto dst_place = dev_ctx.GetPlace();
       framework::TensorCopy(cpu_tensor, dst_place, dev_ctx, tensor);
-      if (platform::is_npu_place(dev_ctx.GetPlace()) ||
-          platform::is_custom_place(dev_ctx.GetPlace())) {
+      if (platform::is_custom_place(dev_ctx.GetPlace())) {
         dev_ctx.Wait();
       }
 #else
@@ -689,7 +687,6 @@ void TensorFromStream(std::istream& is,
     size_t size = tensor->numel() * framework::SizeOfType(desc.data_type());
     if (platform::is_gpu_place(dev_ctx.GetPlace()) ||
         platform::is_xpu_place(dev_ctx.GetPlace()) ||
-        platform::is_npu_place(dev_ctx.GetPlace()) ||
         platform::is_custom_place(dev_ctx.GetPlace())) {
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) || \
     defined(PADDLE_WITH_XPU) || defined(PADDLE_WITH_CUSTOM_DEVICE)
@@ -701,8 +698,7 @@ void TensorFromStream(std::istream& is,
       is.read(static_cast<char*>(buf), size);
       auto dst_place = dev_ctx.GetPlace();
       framework::TensorCopy(cpu_tensor, dst_place, dev_ctx, tensor);
-      if (platform::is_npu_place(dev_ctx.GetPlace()) ||
-          platform::is_custom_place(dev_ctx.GetPlace())) {
+      if (platform::is_custom_place(dev_ctx.GetPlace())) {
         dev_ctx.Wait();
       }
 #else
@@ -712,9 +708,6 @@ void TensorFromStream(std::istream& is,
       } else if (platform::is_xpu_place(dev_ctx.GetPlace())) {
         PADDLE_THROW(platform::errors::Unimplemented(
             "XPUPlace is not supported when not compiled with XPU"));
-      } else if (platform::is_npu_place(dev_ctx.GetPlace())) {
-        PADDLE_THROW(platform::errors::Unimplemented(
-            "NPUPlace is not supported when not compiled with NPU"));
       } else {
         PADDLE_THROW(platform::errors::Unimplemented(
             "CutomPlace is not supported when not compiled with CustomDevice"));
diff --git a/paddle/fluid/imperative/amp_auto_cast.cc b/paddle/fluid/imperative/amp_auto_cast.cc
index bf6bdf382ce44..335bd4b132533 100644
--- a/paddle/fluid/imperative/amp_auto_cast.cc
+++ b/paddle/fluid/imperative/amp_auto_cast.cc
@@ -51,7 +51,6 @@ OpSupportedInfos(const std::string& place,
       {"GPU", &platform::is_gpu_place},
       {"CPU", &platform::is_cpu_place},
       {"XPU", &platform::is_xpu_place},
-      {"NPU", &platform::is_npu_place},
   };
   PADDLE_ENFORCE_NE(is_target_place.count(query_place),
                     0,
@@ -245,7 +244,6 @@ inline bool NeedCast(const std::shared_ptr<VarType>& var) {
       paddle::platform::is_cuda_pinned_place(place) ||
       paddle::platform::is_xpu_place(place) ||
       paddle::platform::is_custom_place(place) ||
-      paddle::platform::is_npu_place(place) ||
       paddle::platform::is_npu_pinned_place(place)) {
     // CudaPinndePlace is added for varbase created by dataloader
     if (data_type == paddle::framework::proto::VarType::FP32 ||
diff --git a/paddle/fluid/imperative/reducer.cc b/paddle/fluid/imperative/reducer.cc
index 8c8ff75b2de96..f90e1243d0f8d 100644
--- a/paddle/fluid/imperative/reducer.cc
+++ b/paddle/fluid/imperative/reducer.cc
@@ -42,9 +42,6 @@ void Group::DivNRanks(const platform::DeviceContext &context, int64_t nranks) {
 #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
     DivNRanks(tensor, nranks, context);
 #endif
-  } else if (platform::is_npu_place(tensor->place())) {
-    // TODO(kuizhiqing)
-    VLOG(4) << "divnrank for npu not support yet";
   } else if (platform::is_cpu_place(tensor->place())) {
     VLOG(4) << "before div 2" << *tensor;
     VLOG(4) << "NDiv for cpu devices : rank = " << nranks;
diff --git a/paddle/fluid/imperative/tracer.cc b/paddle/fluid/imperative/tracer.cc
index af39832b4f5e4..6207fc54f4d16 100644
--- a/paddle/fluid/imperative/tracer.cc
+++ b/paddle/fluid/imperative/tracer.cc
@@ -135,11 +135,6 @@ paddle::framework::GarbageCollector* Tracer::MutableGarbageCollectorIfNotExists(
     } else if (platform::is_cpu_place(place)) {
       gc.reset(new framework::CPUGarbageCollector(place, 0));
       VLOG(10) << "Created GarbageCollector at " << place;
-    } else if (platform::is_npu_place(place)) {
-      PADDLE_THROW(platform::errors::PermissionDenied(
-          "Paddle can't use NPU device since it's not compiled with NPU,"
-          "Please recompile or reinstall Paddle with NPU support."));
-
     } else if (platform::is_ipu_place(place)) {
 #if defined(PADDLE_WITH_IPU)
       gc.reset(new framework::IPUGarbageCollector(place, 0));
@@ -289,9 +284,6 @@ void Tracer::TraceOpImpl(const std::string& type,
       PADDLE_THROW(platform::errors::PreconditionNotMet(
           "PaddlePaddle should compile with XPU if use XPUPlace."));
 #endif
-    } else if (platform::is_npu_place(place)) {
-      PADDLE_THROW(platform::errors::PreconditionNotMet(
-          "PaddlePaddle should compile with NPU if use NPUPlace."));
     } else if (platform::is_custom_place(place)) {
 #ifdef PADDLE_WITH_CUSTOM_DEVICE
       phi::DeviceManager::SetDevice(place);
diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc
index b0ef79a0c7bdf..219c3c2754c68 100644
--- a/paddle/fluid/inference/api/analysis_predictor.cc
+++ b/paddle/fluid/inference/api/analysis_predictor.cc
@@ -1874,9 +1874,6 @@ std::unique_ptr<ZeroCopyTensor> AnalysisPredictor::GetInputTensor(
       auto xpu_place = place_;
       res->SetPlace(PaddlePlace::kXPU, xpu_place.GetDeviceId());
     }
-  } else if (platform::is_npu_place(place_)) {
-    auto npu_place = place_;
-    res->SetPlace(PaddlePlace::kNPU, npu_place.GetDeviceId());
   } else if (platform::is_custom_place(place_)) {
     auto custom_place = place_;
     auto paddleplace = static_cast<PaddlePlace>(
@@ -1931,9 +1928,6 @@ std::unique_ptr<ZeroCopyTensor> AnalysisPredictor::GetOutputTensor(
       auto xpu_place = place_;
       res->SetPlace(PaddlePlace::kXPU, xpu_place.GetDeviceId());
     }
-  } else if (platform::is_npu_place(place_)) {
-    auto npu_place = place_;
-    res->SetPlace(PaddlePlace::kNPU, npu_place.GetDeviceId());
   } else if (platform::is_custom_place(place_)) {
     auto custom_place = place_;
     auto paddleplace = static_cast<PaddlePlace>(
diff --git a/paddle/fluid/inference/api/details/zero_copy_tensor.cc b/paddle/fluid/inference/api/details/zero_copy_tensor.cc
index c10e6b4a43fdd..9fd94fa5d57e2 100644
--- a/paddle/fluid/inference/api/details/zero_copy_tensor.cc
+++ b/paddle/fluid/inference/api/details/zero_copy_tensor.cc
@@ -152,8 +152,6 @@ T *Tensor::data(PlaceType *place, int *size) const {
     *place = PlaceType::kGPU;
   } else if (paddle::platform::is_xpu_place(tensor->place())) {
     *place = PlaceType::kXPU;
-  } else if (paddle::platform::is_npu_place(tensor->place())) {
-    *place = PlaceType::kNPU;
   } else if (paddle::platform::is_custom_place(tensor->place())) {
     *place = PlaceType::kCUSTOM;
   } else {
diff --git a/paddle/fluid/operators/array_operator.h b/paddle/fluid/operators/array_operator.h
index fb3e96db4dbd1..c7b8ce3f381d1 100644
--- a/paddle/fluid/operators/array_operator.h
+++ b/paddle/fluid/operators/array_operator.h
@@ -52,7 +52,6 @@ class ArrayOp : public framework::OperatorBase {
     size_t offset;
     if (platform::is_gpu_place(i_tensor.place()) ||
         platform::is_xpu_place(i_tensor.place()) ||
-        platform::is_npu_place(i_tensor.place()) ||
         platform::is_custom_place(i_tensor.place())) {
       // FIXME: Avoid copy from GPU to CPU
       phi::DenseTensor t;
diff --git a/paddle/fluid/operators/beam_search_decode_op.h b/paddle/fluid/operators/beam_search_decode_op.h
index c4f7b3b5785f4..a6d807b028c1b 100644
--- a/paddle/fluid/operators/beam_search_decode_op.h
+++ b/paddle/fluid/operators/beam_search_decode_op.h
@@ -36,8 +36,7 @@ struct BeamSearchDecodeFunctor {
     tensor_on_gpu_ = false;
     tensor_on_npu_ = false;
     // First make a copy of GPU data on CPU
-    if (platform::is_gpu_place(step_ids_origin_[0].place()) ||
-        platform::is_npu_place(step_ids_origin_[0].place())) {
+    if (platform::is_gpu_place(step_ids_origin_[0].place())) {
       if (platform::is_gpu_place(step_ids_origin_[0].place())) {
         tensor_on_gpu_ = true;
       } else {
@@ -61,8 +60,7 @@ struct BeamSearchDecodeFunctor {
         step_ids_.push_back(out);
       }
     }
-    if (platform::is_gpu_place(step_scores_origin_[0].place()) ||
-        platform::is_npu_place(step_scores_origin_[0].place())) {
+    if (platform::is_gpu_place(step_scores_origin_[0].place())) {
       if (platform::is_gpu_place(step_scores_origin_[0].place())) {
         tensor_on_gpu_ = true;
       } else {
diff --git a/paddle/fluid/operators/controlflow/conditional_block_op.h b/paddle/fluid/operators/controlflow/conditional_block_op.h
index 7b89a211ce4ad..0f04a295ed263 100644
--- a/paddle/fluid/operators/controlflow/conditional_block_op.h
+++ b/paddle/fluid/operators/controlflow/conditional_block_op.h
@@ -83,7 +83,6 @@ class ConditionalOp : public framework::OperatorBase {
       platform::DeviceContextPool::Instance().Get(ips[0]->place())->Wait();
       res = cpu_tensor.data<bool>()[0];
 #endif
-    } else if (platform::is_npu_place(ips[0]->place())) {
     } else if (platform::is_xpu_place(ips[0]->place())) {
 #ifdef PADDLE_WITH_XPU
       phi::DenseTensor cpu_tensor;
diff --git a/paddle/fluid/operators/controlflow/while_op_helper.cc b/paddle/fluid/operators/controlflow/while_op_helper.cc
index 38865a1c53e0b..6ae32f33e957a 100644
--- a/paddle/fluid/operators/controlflow/while_op_helper.cc
+++ b/paddle/fluid/operators/controlflow/while_op_helper.cc
@@ -225,7 +225,6 @@ bool GetCondData(const phi::DenseTensor &cond) {
     return cond.data<bool>()[0];
   }
   // when platform::is_gpu_place(cond.place()) or
-  // platform::is_npu_place(cond.place()) or
   // platform::is_xpu_place(cond.place()) is true
   std::unique_ptr<phi::DenseTensor> cpu_cond{new phi::DenseTensor()};
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) || \
diff --git a/paddle/fluid/operators/generator/get_expected_kernel_func.cc b/paddle/fluid/operators/generator/get_expected_kernel_func.cc
index 558b0c400a4ca..6085dabaed6d8 100644
--- a/paddle/fluid/operators/generator/get_expected_kernel_func.cc
+++ b/paddle/fluid/operators/generator/get_expected_kernel_func.cc
@@ -75,7 +75,6 @@ phi::KernelKey GetReduceExpectedKernelType(
   if (input_data_type == framework::proto::VarType::FP16) {
     PADDLE_ENFORCE_EQ(
         platform::is_gpu_place(ctx.GetPlace()) ||
-            platform::is_npu_place(ctx.GetPlace()) ||
             platform::is_xpu_place(ctx.GetPlace()) ||
             platform::is_custom_place(ctx.GetPlace()),
         true,
diff --git a/paddle/fluid/operators/reader/buffered_reader.cc b/paddle/fluid/operators/reader/buffered_reader.cc
index 3551b829b0487..f0f54eafaa36b 100644
--- a/paddle/fluid/operators/reader/buffered_reader.cc
+++ b/paddle/fluid/operators/reader/buffered_reader.cc
@@ -384,8 +384,6 @@ void BufferedReader::ReadNextImpl(paddle::framework::LoDTensorArray *out) {
 
   if (platform::is_gpu_place(place_)) {
     *out = std::move(cuda_buffer_[i]);
-  } else if (platform::is_npu_place(place_)) {
-    *out = std::move(npu_buffer_[i]);
   } else if (platform::is_xpu_place(place_)) {
     *out = std::move(xpu_buffer_[i]);
   } else if (platform::is_custom_place(place_)) {
diff --git a/paddle/fluid/operators/reduce_ops/reduce_op.h b/paddle/fluid/operators/reduce_ops/reduce_op.h
index 3c2b0b948bb22..3349400a2f93b 100644
--- a/paddle/fluid/operators/reduce_ops/reduce_op.h
+++ b/paddle/fluid/operators/reduce_ops/reduce_op.h
@@ -624,7 +624,6 @@ class ReduceBaseOp : public framework::OperatorWithKernel {
     if (input_data_type == framework::proto::VarType::FP16) {
       PADDLE_ENFORCE_EQ(
           platform::is_gpu_place(ctx.GetPlace()) ||
-              platform::is_npu_place(ctx.GetPlace()) ||
               platform::is_xpu_place(ctx.GetPlace()) ||
               platform::is_custom_place(ctx.GetPlace()),
           true,
diff --git a/paddle/fluid/operators/softmax_op.cc b/paddle/fluid/operators/softmax_op.cc
index ab5816965f05c..633ef748be698 100644
--- a/paddle/fluid/operators/softmax_op.cc
+++ b/paddle/fluid/operators/softmax_op.cc
@@ -43,7 +43,6 @@ class SoftmaxOp : public framework::OperatorWithKernel {
     if (input_data_type == framework::proto::VarType::FP16) {
       PADDLE_ENFORCE_EQ(
           platform::is_gpu_place(ctx.GetPlace()) ||
-              platform::is_npu_place(ctx.GetPlace()) ||
               platform::is_xpu_place(ctx.GetPlace()) ||
               platform::is_custom_place(ctx.GetPlace()),
           true,
@@ -128,7 +127,6 @@ class SoftmaxOpGrad : public framework::OperatorWithKernel {
         ctx, framework::GradVarName("Out"));
     if (input_data_type == framework::proto::VarType::FP16) {
       if (!(platform::is_gpu_place(ctx.GetPlace()) ||
-            platform::is_npu_place(ctx.GetPlace()) ||
             platform::is_xpu_place(ctx.GetPlace()) ||
             platform::is_custom_place(ctx.GetPlace())))
         PADDLE_THROW(platform::errors::InvalidArgument(
diff --git a/paddle/fluid/operators/tile_op_functor.h b/paddle/fluid/operators/tile_op_functor.h
index 4d49510d71d7b..5b2dc31beb044 100644
--- a/paddle/fluid/operators/tile_op_functor.h
+++ b/paddle/fluid/operators/tile_op_functor.h
@@ -29,8 +29,7 @@ inline std::vector<int> get_repeat_times(
     auto* repeat_data = repeat_tensor->data<int>();
     phi::DenseTensor cpu_repeat_tensor;
     if (platform::is_gpu_place(repeat_tensor->place()) ||
-        platform::is_xpu_place(repeat_tensor->place()) ||
-        platform::is_npu_place(repeat_tensor->place())) {
+        platform::is_xpu_place(repeat_tensor->place())) {
       paddle::framework::TensorCopySync(
           *repeat_tensor, platform::CPUPlace(), &cpu_repeat_tensor);
       repeat_data = cpu_repeat_tensor.data<int>();
@@ -48,8 +47,7 @@ inline std::vector<int> get_repeat_times(
     for (size_t i = 0; i < list_repeat_times_tensor.size(); ++i) {
       auto tensor = list_repeat_times_tensor[i];
       if (platform::is_gpu_place(tensor->place()) ||
-          platform::is_xpu_place(tensor->place()) ||
-          platform::is_npu_place(tensor->place())) {
+          platform::is_xpu_place(tensor->place())) {
         phi::DenseTensor temp;
         paddle::framework::TensorCopySync(*tensor, platform::CPUPlace(), &temp);
         vec_repeat_times.push_back(*temp.data<int32_t>());
diff --git a/paddle/fluid/platform/device_context.cc b/paddle/fluid/platform/device_context.cc
index ee12b42c80530..e14ba8b1710cf 100644
--- a/paddle/fluid/platform/device_context.cc
+++ b/paddle/fluid/platform/device_context.cc
@@ -45,8 +45,6 @@ DeviceType Place2DeviceType(const platform::Place& place) {
     return platform::DeviceType::XPU;
   } else if (platform::is_ipu_place(place)) {
     return platform::DeviceType::IPU;
-  } else if (platform::is_npu_place(place)) {
-    return platform::DeviceType::NPU;
   } else if (platform::is_custom_place(place)) {
     return platform::DeviceType::CUSTOM_DEVICE;
   } else {
diff --git a/paddle/fluid/platform/place.cc b/paddle/fluid/platform/place.cc
index e87531e2b8819..354259ca91b74 100644
--- a/paddle/fluid/platform/place.cc
+++ b/paddle/fluid/platform/place.cc
@@ -33,10 +33,6 @@ bool is_xpu_place(const Place &p) {
   return p.GetType() == phi::AllocationType::XPU;
 }
 
-bool is_npu_place(const Place &p) {
-  return p.GetType() == phi::AllocationType::NPU;
-}
-
 bool is_ipu_place(const Place &p) {
   return p.GetType() == phi::AllocationType::IPU;
 }
@@ -73,8 +69,6 @@ bool is_same_place(const Place &p1, const Place &p2) {
       return true;
     } else if (is_xpu_place(p1)) {
       return p1 == p2;
-    } else if (is_npu_place(p1)) {
-      return p1 == p2;
     } else if (is_ipu_place(p1)) {
       return p1 == p2;
     } else if (is_custom_place(p1)) {
@@ -93,8 +87,6 @@ std::string PlaceHelper::GetDeviceType(const Place &place) {
     return "cpu";
   } else if (is_gpu_place(place)) {
     return "gpu";
-  } else if (is_npu_place(place)) {
-    return "npu";
   } else if (is_xpu_place(place)) {
     return "xpu";
   } else if (is_custom_place(place)) {
diff --git a/paddle/fluid/platform/place.h b/paddle/fluid/platform/place.h
index c5f96f5db80c2..f787a1b28783d 100644
--- a/paddle/fluid/platform/place.h
+++ b/paddle/fluid/platform/place.h
@@ -47,7 +47,6 @@ class PlaceHelper {
 
 bool is_gpu_place(const Place &);
 bool is_xpu_place(const Place &);
-bool is_npu_place(const Place &);
 bool is_ipu_place(const Place &);
 bool is_cpu_place(const Place &);
 bool is_cuda_pinned_place(const Place &);
diff --git a/paddle/fluid/pybind/eager.cc b/paddle/fluid/pybind/eager.cc
index cd980340c47a4..d69417a6c0a73 100644
--- a/paddle/fluid/pybind/eager.cc
+++ b/paddle/fluid/pybind/eager.cc
@@ -153,8 +153,6 @@ void InitTensorWithNumpyValue(TensorObject* self,
   } else if (platform::is_cuda_pinned_place(place)) {
     SetTensorFromPyArray<platform::CUDAPinnedPlace>(
         impl_ptr, array, place, zero_copy);
-  } else if (platform::is_npu_place(place)) {
-    SetTensorFromPyArray<platform::NPUPlace>(impl_ptr, array, place, zero_copy);
   } else if (platform::is_custom_place(place)) {
     SetTensorFromPyArray<platform::CustomPlace>(
         impl_ptr, array, place, zero_copy);
diff --git a/paddle/fluid/pybind/eager_math_op_patch.cc b/paddle/fluid/pybind/eager_math_op_patch.cc
index 8756abb5f3673..d764aedd8a1d6 100644
--- a/paddle/fluid/pybind/eager_math_op_patch.cc
+++ b/paddle/fluid/pybind/eager_math_op_patch.cc
@@ -102,8 +102,6 @@ void InitTensorWithNumpyValue(const py::object& array,
   } else if (platform::is_cuda_pinned_place(place)) {
     SetTensorFromPyArray<platform::CUDAPinnedPlace>(
         impl_ptr, array, place, zero_copy);
-  } else if (platform::is_npu_place(place)) {
-    SetTensorFromPyArray<platform::NPUPlace>(impl_ptr, array, place, zero_copy);
   } else if (platform::is_custom_place(place)) {
     SetTensorFromPyArray<platform::CustomPlace>(
         impl_ptr, array, place, zero_copy);
diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc
index d6a5a8b8dfc87..39a28c7487c30 100644
--- a/paddle/fluid/pybind/imperative.cc
+++ b/paddle/fluid/pybind/imperative.cc
@@ -199,8 +199,6 @@ static void InitVarBaseAndTensor(imperative::VarBase *self,
   } else if (platform::is_cuda_pinned_place(place)) {
     SetTensorFromPyArray<platform::CUDAPinnedPlace>(
         tensor, array, place, zero_copy);
-  } else if (platform::is_npu_place(place)) {
-    SetTensorFromPyArray<platform::NPUPlace>(tensor, array, place, zero_copy);
   } else if (platform::is_ipu_place(place)) {
     SetTensorFromPyArray<platform::IPUPlace>(tensor, array, place, zero_copy);
   } else if (platform::is_custom_place(place)) {
diff --git a/paddle/fluid/pybind/place.cc b/paddle/fluid/pybind/place.cc
index 3a464cdc545ed..1c27d70d1bea7 100644
--- a/paddle/fluid/pybind/place.cc
+++ b/paddle/fluid/pybind/place.cc
@@ -648,8 +648,6 @@ void BindPlace(pybind11::module &m) {  // NOLINT
            [](platform::Place &self) { return platform::is_cpu_place(self); })
       .def("is_xpu_place",
            [](platform::Place &self) { return platform::is_xpu_place(self); })
-      .def("is_npu_place",
-           [](platform::Place &self) { return platform::is_npu_place(self); })
       .def("is_ipu_place",
            [](platform::Place &self) { return platform::is_ipu_place(self); })
       .def("is_cuda_pinned_place",
diff --git a/paddle/fluid/pybind/tensor_py.h b/paddle/fluid/pybind/tensor_py.h
index d2bd64bc0fb15..05bacbbf54144 100644
--- a/paddle/fluid/pybind/tensor_py.h
+++ b/paddle/fluid/pybind/tensor_py.h
@@ -960,7 +960,6 @@ inline py::array TensorToPyArray(const phi::DenseTensor &tensor,
   }
   bool is_gpu_tensor = platform::is_gpu_place(tensor.place());
   bool is_xpu_tensor = platform::is_xpu_place(tensor.place());
-  bool is_npu_tensor = platform::is_npu_place(tensor.place());
   bool is_custom_device_tensor = platform::is_custom_place(tensor.place());
   const auto &tensor_dims = tensor.dims();
   auto tensor_dtype = framework::TransToProtoVarType(tensor.dtype());
@@ -981,8 +980,7 @@ inline py::array TensorToPyArray(const phi::DenseTensor &tensor,
   std::string py_dtype_str = details::TensorDTypeToPyDTypeStr(
       framework::TransToProtoVarType(tensor.dtype()));
 
-  if (!is_gpu_tensor && !is_xpu_tensor && !is_npu_tensor &&
-      !is_custom_device_tensor) {
+  if (!is_gpu_tensor && !is_xpu_tensor && !is_custom_device_tensor) {
     if (!need_deep_copy) {
       auto base = py::cast(std::move(tensor));
       return py::array(py::dtype(py_dtype_str.c_str()),

From 9d90738cdfa2b1547645bee62a65fe1109abcc84 Mon Sep 17 00:00:00 2001
From: GGBond8488 <33050871+GGBond8488@users.noreply.github.com>
Date: Mon, 24 Apr 2023 16:21:48 +0800
Subject: [PATCH 021/405] add 0D support for trace (#53208)

* add 0D support for trace, test=allcase

* fix trace gpu kernel 0d error, test=allcase

* fix windows error, test=allcase
---
 paddle/phi/infermeta/unary.cc                 |  1 -
 paddle/phi/kernels/gpu/trace_kernel.cu        |  5 +++-
 .../phi/kernels/impl/trace_grad_kernel_impl.h |  3 ++-
 .../tests/unittests/test_zero_dim_tensor.py   | 24 +++++++++++++++++++
 4 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc
index bfe744446a97b..ea27eba513051 100644
--- a/paddle/phi/infermeta/unary.cc
+++ b/paddle/phi/infermeta/unary.cc
@@ -4402,7 +4402,6 @@ void TraceInferMeta(
   auto sizes = vectorize(x_dims);
   if (x_dims.size() == 2) {
     sizes.clear();
-    sizes.push_back(1);
   } else {
     sizes.erase(sizes.begin() + std::max(dim1_, dim2_));
     sizes.erase(sizes.begin() + std::min(dim1_, dim2_));
diff --git a/paddle/phi/kernels/gpu/trace_kernel.cu b/paddle/phi/kernels/gpu/trace_kernel.cu
index 671ca490e136a..304bf778094d3 100644
--- a/paddle/phi/kernels/gpu/trace_kernel.cu
+++ b/paddle/phi/kernels/gpu/trace_kernel.cu
@@ -32,7 +32,10 @@ void TraceKernel(const Context& ctx,
   auto diag = funcs::Diagonal<T, Context>(ctx, &x, offset, axis1, axis2);
   if (diag.numel() > 0) {
     std::vector<int> reduce_dims;
-    reduce_dims.push_back(out->dims().size());
+    // Adapt to 0D output
+    auto out_dim_size = out->dims().size();
+    if (out_dim_size == 0) out_dim_size = 1;
+    reduce_dims.push_back(out_dim_size);
     funcs::ReduceKernel<T, T, kps::AddFunctor, kps::IdentityFunctor<T>>(
         ctx, diag, out, kps::IdentityFunctor<T>(), reduce_dims);
   } else {
diff --git a/paddle/phi/kernels/impl/trace_grad_kernel_impl.h b/paddle/phi/kernels/impl/trace_grad_kernel_impl.h
index 90a2327ef3e20..1099f27f3622e 100644
--- a/paddle/phi/kernels/impl/trace_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/trace_grad_kernel_impl.h
@@ -91,7 +91,8 @@ void TraceGradKernel(const Context& ctx,
   auto input_dims = in_grad->dims();
   auto input_stride = phi::stride(input_dims);
   auto output_dims = out_grad.dims();
-  auto output_stride = phi::stride(output_dims);
+  auto output_stride = output_dims.size() == 0 ? phi::DDim(output_dims)
+                                               : phi::stride(output_dims);
 
   auto* out_data = out_grad.data<T>();
   T* x_data = ctx.template Alloc<T>(in_grad);
diff --git a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
index 7ea98f7c889a3..965bcae57d9db 100644
--- a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
+++ b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
@@ -2437,6 +2437,16 @@ def test_multi_dot(self):
         self.assertEqual(b.grad.shape, [4, 5])
         self.assertEqual(c.grad.shape, [5])
 
+    def test_trace(self):
+        x = paddle.to_tensor([[3, 2], [1, 9]], dtype="float32")
+        x.stop_gradient = False
+        out = paddle.trace(x)
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        np.testing.assert_allclose(out, np.array(12))
+        self.assertEqual(x.grad.shape, [2, 2])
+
 
 class TestSundryAPIStatic(unittest.TestCase):
     def setUp(self):
@@ -4426,6 +4436,20 @@ def test_multi_dot(self):
         self.assertEqual(res[2].shape, (4, 5))
         self.assertEqual(res[3].shape, (5,))
 
+    @prog_scope()
+    def test_trace(self):
+        x = paddle.to_tensor([[3, 2], [1, 9]], dtype="float32")
+        x.stop_gradient = False
+        out = paddle.trace(x)
+        paddle.static.append_backward(out)
+
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out, x.grad_name])
+
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (2, 2))
+        np.testing.assert_allclose(res[0], np.array(12))
+
 
 # Use to test API whose zero-dim input tensors don't have grad and not need to test backward in OpTest.
 class TestNoBackwardAPI(unittest.TestCase):

From adc2b7459f2bcbe9757f6522af41dddc9f0d5df5 Mon Sep 17 00:00:00 2001
From: tianshuo78520a <707759223@qq.com>
Date: Mon, 24 Apr 2023 16:28:16 +0800
Subject: [PATCH 022/405] Mv eager ut (#53167)

* Mv eager tests

* fix

* fix build error

* fix build error

* fix codestyle
---
 paddle/fluid/eager/CMakeLists.txt             |  1 -
 paddle/fluid/eager/tests/CMakeLists.txt       |  3 --
 test/cpp/CMakeLists.txt                       |  1 +
 test/cpp/eager/CMakeLists.txt                 | 35 +++++++++++++++++++
 .../data_structure_tests/CMakeLists.txt       |  0
 .../accumulation_node_test.cc                 |  0
 .../autograd_meta_test.cc                     |  2 +-
 .../data_structure_tests/eager_tensor_test.cc |  0
 .../grad_node_info_test.cc                    |  2 +-
 .../data_structure_tests/grad_node_test.h     |  0
 .../grad_tensor_holder_test.cc                |  0
 .../tensor_wrapper_test.cc                    |  2 +-
 .../eager}/performance_tests/CMakeLists.txt   |  0
 .../performance_tests/benchmark_eager_cpu.cc  |  4 +--
 .../performance_tests/benchmark_eager_cuda.cc |  4 +--
 .../performance_tests/benchmark_fluid_cpu.cc  |  4 +--
 .../performance_tests/benchmark_fluid_cuda.cc |  4 +--
 .../performance_tests/benchmark_utils.cc      |  4 +--
 .../performance_tests/benchmark_utils.h       |  0
 .../cpp/eager}/task_tests/CMakeLists.txt      |  0
 .../cpp/eager}/task_tests/backward_test.cc    |  2 +-
 .../cross_batch_accumulation_test.cc          |  2 +-
 .../cpp/eager}/task_tests/eager_utils_test.cc |  4 +--
 .../task_tests/forward_autograd_test.cc       |  2 +-
 .../eager}/task_tests/fwd_bwd_joint_test.cc   |  2 +-
 .../cpp/eager}/task_tests/generated_test.cc   |  2 +-
 .../cpp/eager}/task_tests/grad_test.cc        |  2 +-
 .../cpp/eager}/task_tests/hook_test.cc        |  2 +-
 .../task_tests/hook_test_intermidiate.cc      |  2 +-
 .../eager}/task_tests/nan_inf_utils_test.cc   |  0
 .../eager}/task_tests/tensor_utils_test.cc    |  2 +-
 .../tests => test/cpp/eager}/test_utils.h     |  0
 test/cpp/prim/test_eager_prim.cc              |  2 +-
 33 files changed, 61 insertions(+), 29 deletions(-)
 delete mode 100644 paddle/fluid/eager/tests/CMakeLists.txt
 create mode 100644 test/cpp/eager/CMakeLists.txt
 rename {paddle/fluid/eager/tests => test/cpp/eager}/data_structure_tests/CMakeLists.txt (100%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/data_structure_tests/accumulation_node_test.cc (100%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/data_structure_tests/autograd_meta_test.cc (97%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/data_structure_tests/eager_tensor_test.cc (100%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/data_structure_tests/grad_node_info_test.cc (98%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/data_structure_tests/grad_node_test.h (100%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/data_structure_tests/grad_tensor_holder_test.cc (100%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/data_structure_tests/tensor_wrapper_test.cc (97%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/performance_tests/CMakeLists.txt (100%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/performance_tests/benchmark_eager_cpu.cc (98%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/performance_tests/benchmark_eager_cuda.cc (98%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/performance_tests/benchmark_fluid_cpu.cc (98%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/performance_tests/benchmark_fluid_cuda.cc (98%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/performance_tests/benchmark_utils.cc (99%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/performance_tests/benchmark_utils.h (100%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/task_tests/CMakeLists.txt (100%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/task_tests/backward_test.cc (99%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/task_tests/cross_batch_accumulation_test.cc (98%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/task_tests/eager_utils_test.cc (98%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/task_tests/forward_autograd_test.cc (99%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/task_tests/fwd_bwd_joint_test.cc (99%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/task_tests/generated_test.cc (99%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/task_tests/grad_test.cc (99%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/task_tests/hook_test.cc (99%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/task_tests/hook_test_intermidiate.cc (99%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/task_tests/nan_inf_utils_test.cc (100%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/task_tests/tensor_utils_test.cc (97%)
 rename {paddle/fluid/eager/tests => test/cpp/eager}/test_utils.h (100%)

diff --git a/paddle/fluid/eager/CMakeLists.txt b/paddle/fluid/eager/CMakeLists.txt
index 46d1ff43e0dfe..a0ff3300ffa29 100755
--- a/paddle/fluid/eager/CMakeLists.txt
+++ b/paddle/fluid/eager/CMakeLists.txt
@@ -39,7 +39,6 @@ add_subdirectory(api)
 add_subdirectory(custom_operator)
 if(NOT ((NOT WITH_PYTHON) AND ON_INFER))
   add_subdirectory(accumulation)
-  add_subdirectory(tests)
   add_subdirectory(pylayer)
   cc_library(
     grad_tensor_holder
diff --git a/paddle/fluid/eager/tests/CMakeLists.txt b/paddle/fluid/eager/tests/CMakeLists.txt
deleted file mode 100644
index 6bcd34262c8ab..0000000000000
--- a/paddle/fluid/eager/tests/CMakeLists.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-add_subdirectory(data_structure_tests)
-add_subdirectory(task_tests)
-add_subdirectory(performance_tests)
diff --git a/test/cpp/CMakeLists.txt b/test/cpp/CMakeLists.txt
index 603b7eb78ae7e..2f395cb62ed65 100644
--- a/test/cpp/CMakeLists.txt
+++ b/test/cpp/CMakeLists.txt
@@ -5,3 +5,4 @@ add_subdirectory(prim)
 add_subdirectory(imperative)
 add_subdirectory(ir)
 add_subdirectory(inference)
+add_subdirectory(eager)
diff --git a/test/cpp/eager/CMakeLists.txt b/test/cpp/eager/CMakeLists.txt
new file mode 100644
index 0000000000000..d8d3a6304cffc
--- /dev/null
+++ b/test/cpp/eager/CMakeLists.txt
@@ -0,0 +1,35 @@
+set(eager_deps
+    phi_api
+    phi_dygraph_api
+    hook_utils
+    tensor_utils
+    utils
+    global_utils
+    backward
+    phi_tensor
+    tracer
+    layer
+    autograd_meta
+    eager_nan_inf_utils
+    grad_node_info
+    grad_tensor_holder
+    custom_operator_node)
+
+if(NOT (NOT WITH_PYTHON AND ON_INFER))
+  set(eager_deps ${eager_deps} accumulation_node prim_utils)
+endif()
+
+set(fluid_deps
+    tracer
+    layer
+    proto_desc
+    operator
+    op_registry
+    variable_helper
+    memcpy)
+set(generated_deps final_dygraph_function final_dygraph_node dygraph_function
+                   dygraph_node)
+
+add_subdirectory(data_structure_tests)
+add_subdirectory(task_tests)
+add_subdirectory(performance_tests)
diff --git a/paddle/fluid/eager/tests/data_structure_tests/CMakeLists.txt b/test/cpp/eager/data_structure_tests/CMakeLists.txt
similarity index 100%
rename from paddle/fluid/eager/tests/data_structure_tests/CMakeLists.txt
rename to test/cpp/eager/data_structure_tests/CMakeLists.txt
diff --git a/paddle/fluid/eager/tests/data_structure_tests/accumulation_node_test.cc b/test/cpp/eager/data_structure_tests/accumulation_node_test.cc
similarity index 100%
rename from paddle/fluid/eager/tests/data_structure_tests/accumulation_node_test.cc
rename to test/cpp/eager/data_structure_tests/accumulation_node_test.cc
diff --git a/paddle/fluid/eager/tests/data_structure_tests/autograd_meta_test.cc b/test/cpp/eager/data_structure_tests/autograd_meta_test.cc
similarity index 97%
rename from paddle/fluid/eager/tests/data_structure_tests/autograd_meta_test.cc
rename to test/cpp/eager/data_structure_tests/autograd_meta_test.cc
index f588f8c8c759a..651e3b63f07ac 100644
--- a/paddle/fluid/eager/tests/data_structure_tests/autograd_meta_test.cc
+++ b/test/cpp/eager/data_structure_tests/autograd_meta_test.cc
@@ -18,8 +18,8 @@
 #include "gtest/gtest.h"
 #include "paddle/fluid/eager/eager_tensor.h"
 #include "paddle/fluid/eager/grad_node_info.h"
-#include "paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h"
 #include "paddle/phi/api/lib/utils/allocator.h"
+#include "test/cpp/eager/data_structure_tests/grad_node_test.h"
 
 TEST(AutogradMeta, Constructor) {
   paddle::Tensor et1;
diff --git a/paddle/fluid/eager/tests/data_structure_tests/eager_tensor_test.cc b/test/cpp/eager/data_structure_tests/eager_tensor_test.cc
similarity index 100%
rename from paddle/fluid/eager/tests/data_structure_tests/eager_tensor_test.cc
rename to test/cpp/eager/data_structure_tests/eager_tensor_test.cc
diff --git a/paddle/fluid/eager/tests/data_structure_tests/grad_node_info_test.cc b/test/cpp/eager/data_structure_tests/grad_node_info_test.cc
similarity index 98%
rename from paddle/fluid/eager/tests/data_structure_tests/grad_node_info_test.cc
rename to test/cpp/eager/data_structure_tests/grad_node_info_test.cc
index c56db9e611713..0948e6f72aa0b 100644
--- a/paddle/fluid/eager/tests/data_structure_tests/grad_node_info_test.cc
+++ b/test/cpp/eager/data_structure_tests/grad_node_info_test.cc
@@ -19,8 +19,8 @@
 #include "paddle/fluid/eager/autograd_meta.h"
 #include "paddle/fluid/eager/eager_tensor.h"
 #include "paddle/fluid/eager/hooks.h"
-#include "paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h"
 #include "paddle/phi/api/lib/utils/allocator.h"
+#include "test/cpp/eager/data_structure_tests/grad_node_test.h"
 
 TEST(GradNodeInfo, GradSlotMeta) {
   auto grad_slot = egr::GradSlotMeta();
diff --git a/paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h b/test/cpp/eager/data_structure_tests/grad_node_test.h
similarity index 100%
rename from paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h
rename to test/cpp/eager/data_structure_tests/grad_node_test.h
diff --git a/paddle/fluid/eager/tests/data_structure_tests/grad_tensor_holder_test.cc b/test/cpp/eager/data_structure_tests/grad_tensor_holder_test.cc
similarity index 100%
rename from paddle/fluid/eager/tests/data_structure_tests/grad_tensor_holder_test.cc
rename to test/cpp/eager/data_structure_tests/grad_tensor_holder_test.cc
diff --git a/paddle/fluid/eager/tests/data_structure_tests/tensor_wrapper_test.cc b/test/cpp/eager/data_structure_tests/tensor_wrapper_test.cc
similarity index 97%
rename from paddle/fluid/eager/tests/data_structure_tests/tensor_wrapper_test.cc
rename to test/cpp/eager/data_structure_tests/tensor_wrapper_test.cc
index 86973debfe323..a3a82b0c3b201 100644
--- a/paddle/fluid/eager/tests/data_structure_tests/tensor_wrapper_test.cc
+++ b/test/cpp/eager/data_structure_tests/tensor_wrapper_test.cc
@@ -16,8 +16,8 @@
 
 #include "glog/logging.h"
 #include "gtest/gtest.h"
-#include "paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h"
 #include "paddle/fluid/eager/utils.h"
+#include "test/cpp/eager/data_structure_tests/grad_node_test.h"
 
 TEST(TensorWrapper, Basic) {
   VLOG(6) << "Test Full reserved";
diff --git a/paddle/fluid/eager/tests/performance_tests/CMakeLists.txt b/test/cpp/eager/performance_tests/CMakeLists.txt
similarity index 100%
rename from paddle/fluid/eager/tests/performance_tests/CMakeLists.txt
rename to test/cpp/eager/performance_tests/CMakeLists.txt
diff --git a/paddle/fluid/eager/tests/performance_tests/benchmark_eager_cpu.cc b/test/cpp/eager/performance_tests/benchmark_eager_cpu.cc
similarity index 98%
rename from paddle/fluid/eager/tests/performance_tests/benchmark_eager_cpu.cc
rename to test/cpp/eager/performance_tests/benchmark_eager_cpu.cc
index 54deba9d51652..c3c7d847f9794 100644
--- a/paddle/fluid/eager/tests/performance_tests/benchmark_eager_cpu.cc
+++ b/test/cpp/eager/performance_tests/benchmark_eager_cpu.cc
@@ -22,10 +22,10 @@
 #include "paddle/fluid/eager/api/all.h"
 #include "paddle/fluid/eager/autograd_meta.h"
 #include "paddle/fluid/eager/backward.h"
-#include "paddle/fluid/eager/tests/performance_tests/benchmark_utils.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/fluid/imperative/tracer.h"
 #include "paddle/phi/core/flags.h"
+#include "test/cpp/eager/performance_tests/benchmark_utils.h"
+#include "test/cpp/eager/test_utils.h"
 
 #ifdef WITH_GPERFTOOLS
 #include "gperftools/profiler.h"
diff --git a/paddle/fluid/eager/tests/performance_tests/benchmark_eager_cuda.cc b/test/cpp/eager/performance_tests/benchmark_eager_cuda.cc
similarity index 98%
rename from paddle/fluid/eager/tests/performance_tests/benchmark_eager_cuda.cc
rename to test/cpp/eager/performance_tests/benchmark_eager_cuda.cc
index d7beb958df852..48c2054e7076e 100644
--- a/paddle/fluid/eager/tests/performance_tests/benchmark_eager_cuda.cc
+++ b/test/cpp/eager/performance_tests/benchmark_eager_cuda.cc
@@ -21,10 +21,10 @@
 #include "paddle/fluid/eager/api/all.h"
 #include "paddle/fluid/eager/autograd_meta.h"
 #include "paddle/fluid/eager/backward.h"
-#include "paddle/fluid/eager/tests/performance_tests/benchmark_utils.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/fluid/imperative/tracer.h"
 #include "paddle/phi/core/flags.h"
+#include "test/cpp/eager/performance_tests/benchmark_utils.h"
+#include "test/cpp/eager/test_utils.h"
 
 #ifdef WITH_GPERFTOOLS
 #include "gperftools/profiler.h"
diff --git a/paddle/fluid/eager/tests/performance_tests/benchmark_fluid_cpu.cc b/test/cpp/eager/performance_tests/benchmark_fluid_cpu.cc
similarity index 98%
rename from paddle/fluid/eager/tests/performance_tests/benchmark_fluid_cpu.cc
rename to test/cpp/eager/performance_tests/benchmark_fluid_cpu.cc
index 2289a9e8680b4..18bdb8d2241cf 100644
--- a/paddle/fluid/eager/tests/performance_tests/benchmark_fluid_cpu.cc
+++ b/test/cpp/eager/performance_tests/benchmark_fluid_cpu.cc
@@ -23,11 +23,11 @@
 
 #include "glog/logging.h"
 #include "gtest/gtest.h"
-#include "paddle/fluid/eager/tests/performance_tests/benchmark_utils.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/fluid/imperative/basic_engine.h"
 #include "paddle/fluid/imperative/tracer.h"
 #include "paddle/fluid/memory/memcpy.h"
+#include "test/cpp/eager/performance_tests/benchmark_utils.h"
+#include "test/cpp/eager/test_utils.h"
 
 #ifdef WITH_GPERFTOOLS
 #include "gperftools/profiler.h"
diff --git a/paddle/fluid/eager/tests/performance_tests/benchmark_fluid_cuda.cc b/test/cpp/eager/performance_tests/benchmark_fluid_cuda.cc
similarity index 98%
rename from paddle/fluid/eager/tests/performance_tests/benchmark_fluid_cuda.cc
rename to test/cpp/eager/performance_tests/benchmark_fluid_cuda.cc
index c0f2250fbae18..d24c79f390c91 100644
--- a/paddle/fluid/eager/tests/performance_tests/benchmark_fluid_cuda.cc
+++ b/test/cpp/eager/performance_tests/benchmark_fluid_cuda.cc
@@ -23,11 +23,11 @@
 
 #include "glog/logging.h"
 #include "gtest/gtest.h"
-#include "paddle/fluid/eager/tests/performance_tests/benchmark_utils.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/fluid/imperative/basic_engine.h"
 #include "paddle/fluid/imperative/tracer.h"
 #include "paddle/fluid/memory/memcpy.h"
+#include "test/cpp/eager/performance_tests/benchmark_utils.h"
+#include "test/cpp/eager/test_utils.h"
 
 #ifdef WITH_GPERFTOOLS
 #include "gperftools/profiler.h"
diff --git a/paddle/fluid/eager/tests/performance_tests/benchmark_utils.cc b/test/cpp/eager/performance_tests/benchmark_utils.cc
similarity index 99%
rename from paddle/fluid/eager/tests/performance_tests/benchmark_utils.cc
rename to test/cpp/eager/performance_tests/benchmark_utils.cc
index c84dd7661b669..83d14a6b45b89 100644
--- a/paddle/fluid/eager/tests/performance_tests/benchmark_utils.cc
+++ b/test/cpp/eager/performance_tests/benchmark_utils.cc
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/fluid/eager/tests/performance_tests/benchmark_utils.h"
+#include "test/cpp/eager/performance_tests/benchmark_utils.h"
 
 #include <iostream>
 #include <memory>
@@ -24,8 +24,8 @@
 #include "paddle/fluid/eager/api/all.h"
 #include "paddle/fluid/eager/autograd_meta.h"
 #include "paddle/fluid/eager/backward.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/fluid/eager/utils.h"
+#include "test/cpp/eager/test_utils.h"
 
 // Eager Generated
 #include "paddle/fluid/eager/api/generated/eager_generated/forwards/dygraph_functions.h"
diff --git a/paddle/fluid/eager/tests/performance_tests/benchmark_utils.h b/test/cpp/eager/performance_tests/benchmark_utils.h
similarity index 100%
rename from paddle/fluid/eager/tests/performance_tests/benchmark_utils.h
rename to test/cpp/eager/performance_tests/benchmark_utils.h
diff --git a/paddle/fluid/eager/tests/task_tests/CMakeLists.txt b/test/cpp/eager/task_tests/CMakeLists.txt
similarity index 100%
rename from paddle/fluid/eager/tests/task_tests/CMakeLists.txt
rename to test/cpp/eager/task_tests/CMakeLists.txt
diff --git a/paddle/fluid/eager/tests/task_tests/backward_test.cc b/test/cpp/eager/task_tests/backward_test.cc
similarity index 99%
rename from paddle/fluid/eager/tests/task_tests/backward_test.cc
rename to test/cpp/eager/task_tests/backward_test.cc
index 43a994068ca82..b26945d00ae14 100644
--- a/paddle/fluid/eager/tests/task_tests/backward_test.cc
+++ b/test/cpp/eager/task_tests/backward_test.cc
@@ -24,10 +24,10 @@
 #include "paddle/fluid/eager/api/utils/tensor_utils.h"
 #include "paddle/fluid/eager/autograd_meta.h"
 #include "paddle/fluid/eager/grad_node_info.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/phi/core/dense_tensor.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/core/tensor_meta.h"
+#include "test/cpp/eager/test_utils.h"
 
 PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
diff --git a/paddle/fluid/eager/tests/task_tests/cross_batch_accumulation_test.cc b/test/cpp/eager/task_tests/cross_batch_accumulation_test.cc
similarity index 98%
rename from paddle/fluid/eager/tests/task_tests/cross_batch_accumulation_test.cc
rename to test/cpp/eager/task_tests/cross_batch_accumulation_test.cc
index 28cc3b2b30b34..6d48970f2e95f 100644
--- a/paddle/fluid/eager/tests/task_tests/cross_batch_accumulation_test.cc
+++ b/test/cpp/eager/task_tests/cross_batch_accumulation_test.cc
@@ -23,10 +23,10 @@
 #include "paddle/fluid/eager/autograd_meta.h"
 #include "paddle/fluid/eager/backward.h"
 #include "paddle/fluid/eager/grad_node_info.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/phi/core/dense_tensor.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/core/tensor_meta.h"
+#include "test/cpp/eager/test_utils.h"
 
 PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
 
diff --git a/paddle/fluid/eager/tests/task_tests/eager_utils_test.cc b/test/cpp/eager/task_tests/eager_utils_test.cc
similarity index 98%
rename from paddle/fluid/eager/tests/task_tests/eager_utils_test.cc
rename to test/cpp/eager/task_tests/eager_utils_test.cc
index d9d277ad59b19..77902fa5eed50 100644
--- a/paddle/fluid/eager/tests/task_tests/eager_utils_test.cc
+++ b/test/cpp/eager/task_tests/eager_utils_test.cc
@@ -18,11 +18,11 @@
 #include "paddle/fluid/eager/accumulation/accumulation_node.h"
 #include "paddle/fluid/eager/eager_tensor.h"
 #include "paddle/fluid/eager/grad_node_info.h"
-#include "paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/fluid/eager/utils.h"
 #include "paddle/phi/api/lib/utils/allocator.h"
 #include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/eager/data_structure_tests/grad_node_test.h"
+#include "test/cpp/eager/test_utils.h"
 
 PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
 
diff --git a/paddle/fluid/eager/tests/task_tests/forward_autograd_test.cc b/test/cpp/eager/task_tests/forward_autograd_test.cc
similarity index 99%
rename from paddle/fluid/eager/tests/task_tests/forward_autograd_test.cc
rename to test/cpp/eager/task_tests/forward_autograd_test.cc
index b25b40571fb71..41ca170998203 100644
--- a/paddle/fluid/eager/tests/task_tests/forward_autograd_test.cc
+++ b/test/cpp/eager/task_tests/forward_autograd_test.cc
@@ -21,10 +21,10 @@
 #include "paddle/fluid/eager/api/utils/tensor_utils.h"
 #include "paddle/fluid/eager/autograd_meta.h"
 #include "paddle/fluid/eager/grad_node_info.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/phi/core/dense_tensor.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/core/tensor_meta.h"
+#include "test/cpp/eager/test_utils.h"
 
 PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
 
diff --git a/paddle/fluid/eager/tests/task_tests/fwd_bwd_joint_test.cc b/test/cpp/eager/task_tests/fwd_bwd_joint_test.cc
similarity index 99%
rename from paddle/fluid/eager/tests/task_tests/fwd_bwd_joint_test.cc
rename to test/cpp/eager/task_tests/fwd_bwd_joint_test.cc
index 5098be17bca47..4e9dffa28e415 100644
--- a/paddle/fluid/eager/tests/task_tests/fwd_bwd_joint_test.cc
+++ b/test/cpp/eager/task_tests/fwd_bwd_joint_test.cc
@@ -23,10 +23,10 @@
 #include "paddle/fluid/eager/backward.h"
 #include "paddle/fluid/eager/grad_node_info.h"
 #include "paddle/fluid/eager/hooks.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/phi/core/dense_tensor.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/core/tensor_meta.h"
+#include "test/cpp/eager/test_utils.h"
 
 PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
diff --git a/paddle/fluid/eager/tests/task_tests/generated_test.cc b/test/cpp/eager/task_tests/generated_test.cc
similarity index 99%
rename from paddle/fluid/eager/tests/task_tests/generated_test.cc
rename to test/cpp/eager/task_tests/generated_test.cc
index 7b7c47ecb71f7..dcb04ba66ca66 100644
--- a/paddle/fluid/eager/tests/task_tests/generated_test.cc
+++ b/test/cpp/eager/task_tests/generated_test.cc
@@ -22,10 +22,10 @@
 #include "paddle/fluid/eager/api/utils/tensor_utils.h"
 #include "paddle/fluid/eager/autograd_meta.h"
 #include "paddle/fluid/eager/backward.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/fluid/eager/utils.h"
 #include "paddle/fluid/imperative/tracer.h"
 #include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/eager/test_utils.h"
 
 PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(matmul, CPU, ALL_LAYOUT);
diff --git a/paddle/fluid/eager/tests/task_tests/grad_test.cc b/test/cpp/eager/task_tests/grad_test.cc
similarity index 99%
rename from paddle/fluid/eager/tests/task_tests/grad_test.cc
rename to test/cpp/eager/task_tests/grad_test.cc
index 8f76b6d05ee87..a1e2dece37595 100644
--- a/paddle/fluid/eager/tests/task_tests/grad_test.cc
+++ b/test/cpp/eager/task_tests/grad_test.cc
@@ -23,10 +23,10 @@
 #include "paddle/fluid/eager/autograd_meta.h"
 #include "paddle/fluid/eager/backward.h"
 #include "paddle/fluid/eager/grad_node_info.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/phi/core/dense_tensor.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/core/tensor_meta.h"
+#include "test/cpp/eager/test_utils.h"
 
 PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
diff --git a/paddle/fluid/eager/tests/task_tests/hook_test.cc b/test/cpp/eager/task_tests/hook_test.cc
similarity index 99%
rename from paddle/fluid/eager/tests/task_tests/hook_test.cc
rename to test/cpp/eager/task_tests/hook_test.cc
index 2c67427c55bbe..0105d52846ed0 100644
--- a/paddle/fluid/eager/tests/task_tests/hook_test.cc
+++ b/test/cpp/eager/task_tests/hook_test.cc
@@ -23,10 +23,10 @@
 #include "paddle/fluid/eager/backward.h"
 #include "paddle/fluid/eager/grad_node_info.h"
 #include "paddle/fluid/eager/hooks.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/phi/core/dense_tensor.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/core/tensor_meta.h"
+#include "test/cpp/eager/test_utils.h"
 
 PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
 
diff --git a/paddle/fluid/eager/tests/task_tests/hook_test_intermidiate.cc b/test/cpp/eager/task_tests/hook_test_intermidiate.cc
similarity index 99%
rename from paddle/fluid/eager/tests/task_tests/hook_test_intermidiate.cc
rename to test/cpp/eager/task_tests/hook_test_intermidiate.cc
index fe63ff7da48d1..5c91b1be253a9 100644
--- a/paddle/fluid/eager/tests/task_tests/hook_test_intermidiate.cc
+++ b/test/cpp/eager/task_tests/hook_test_intermidiate.cc
@@ -20,10 +20,10 @@
 #include "paddle/fluid/eager/backward.h"
 #include "paddle/fluid/eager/grad_node_info.h"
 #include "paddle/fluid/eager/hooks.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/fluid/imperative/tracer.h"
 #include "paddle/phi/core/dense_tensor.h"
 #include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/eager/test_utils.h"
 
 PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(matmul, CPU, ALL_LAYOUT);
diff --git a/paddle/fluid/eager/tests/task_tests/nan_inf_utils_test.cc b/test/cpp/eager/task_tests/nan_inf_utils_test.cc
similarity index 100%
rename from paddle/fluid/eager/tests/task_tests/nan_inf_utils_test.cc
rename to test/cpp/eager/task_tests/nan_inf_utils_test.cc
diff --git a/paddle/fluid/eager/tests/task_tests/tensor_utils_test.cc b/test/cpp/eager/task_tests/tensor_utils_test.cc
similarity index 97%
rename from paddle/fluid/eager/tests/task_tests/tensor_utils_test.cc
rename to test/cpp/eager/task_tests/tensor_utils_test.cc
index f055839e516ad..21e6304fe03aa 100644
--- a/paddle/fluid/eager/tests/task_tests/tensor_utils_test.cc
+++ b/test/cpp/eager/task_tests/tensor_utils_test.cc
@@ -20,9 +20,9 @@
 #include "paddle/fluid/eager/eager_tensor.h"
 #include "paddle/fluid/eager/grad_node_info.h"
 #include "paddle/fluid/eager/grad_tensor_holder.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/phi/api/lib/utils/allocator.h"
 #include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/eager/test_utils.h"
 
 PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
 
diff --git a/paddle/fluid/eager/tests/test_utils.h b/test/cpp/eager/test_utils.h
similarity index 100%
rename from paddle/fluid/eager/tests/test_utils.h
rename to test/cpp/eager/test_utils.h
diff --git a/test/cpp/prim/test_eager_prim.cc b/test/cpp/prim/test_eager_prim.cc
index 5055e439c40fd..d2cdeb80495c1 100644
--- a/test/cpp/prim/test_eager_prim.cc
+++ b/test/cpp/prim/test_eager_prim.cc
@@ -20,11 +20,11 @@
 #include "paddle/fluid/eager/api/generated/eager_generated/forwards/dygraph_functions.h"
 #include "paddle/fluid/eager/api/utils/hook_utils.h"
 #include "paddle/fluid/eager/backward.h"
-#include "paddle/fluid/eager/tests/test_utils.h"
 #include "paddle/fluid/prim/utils/utils.h"
 #include "paddle/phi/core/dense_tensor.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/core/tensor_meta.h"
+#include "test/cpp/eager/test_utils.h"
 #include "test/cpp/prim/init_env_utils.h"
 
 DECLARE_string(tensor_operants_mode);

From 71474b10c2f0d3eb6f4f296a1a053d1d6c7c5ca0 Mon Sep 17 00:00:00 2001
From: Yuanle Liu <yuanlehome@163.com>
Date: Mon, 24 Apr 2023 16:34:59 +0800
Subject: [PATCH 023/405] fix static_assert with no message (#53222)

---
 paddle/phi/kernels/funcs/fused_gemm_epilogue.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/paddle/phi/kernels/funcs/fused_gemm_epilogue.h b/paddle/phi/kernels/funcs/fused_gemm_epilogue.h
index 4f1a1c6f0bde1..8a2309ba26000 100644
--- a/paddle/phi/kernels/funcs/fused_gemm_epilogue.h
+++ b/paddle/phi/kernels/funcs/fused_gemm_epilogue.h
@@ -535,8 +535,10 @@ void ComputeFusedGemmEpilogueBackwardImpl(const phi::GPUContext& dev_ctx,
                                           bool use_addto_dx,
                                           bool use_addto_dy) {
   using MT = typename phi::dtype::MPTypeTrait<T>::Type;
-  static_assert(std::is_same<DXT, T>::value || std::is_same<DXT, MT>::value);
-  static_assert(std::is_same<DYT, T>::value || std::is_same<DYT, MT>::value);
+  static_assert(std::is_same<DXT, T>::value || std::is_same<DXT, MT>::value,
+                "");
+  static_assert(std::is_same<DYT, T>::value || std::is_same<DYT, MT>::value,
+                "");
   using Trait = FusedGEMMGradTrait<TransX, TransY>;
 
   if (dx) {

From ae426b784af7af148d3804c39748a46eb3ef39fe Mon Sep 17 00:00:00 2001
From: zyfncg <zhangyunfei07@baidu.com>
Date: Mon, 24 Apr 2023 16:37:16 +0800
Subject: [PATCH 024/405] fix compile bug of kps (#53251)

---
 .../kps/{elementwise_kernel.cu => elementwise_raw_kernel.cu}      | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename paddle/phi/kernels/legacy/kps/{elementwise_kernel.cu => elementwise_raw_kernel.cu} (100%)

diff --git a/paddle/phi/kernels/legacy/kps/elementwise_kernel.cu b/paddle/phi/kernels/legacy/kps/elementwise_raw_kernel.cu
similarity index 100%
rename from paddle/phi/kernels/legacy/kps/elementwise_kernel.cu
rename to paddle/phi/kernels/legacy/kps/elementwise_raw_kernel.cu

From bfa5d6b8f3198da42bdcfe52f2ee2e07866be0a8 Mon Sep 17 00:00:00 2001
From: zhupengyang <zhu_py@qq.com>
Date: Mon, 24 Apr 2023 16:53:32 +0800
Subject: [PATCH 025/405] transform cachekv datalayout of
 fused_multi_transformer_xpu (#53144)

---
 paddle/fluid/framework/ir/CMakeLists.txt      |   6 +
 paddle/fluid/framework/ir/pass.cc             |   1 +
 ...i_transformer_cachekv_layout_trans_pass.cc | 202 ++++++++++++++++++
 ...ti_transformer_cachekv_layout_trans_pass.h |  79 +++++++
 ...nsformer_cachekv_layout_trans_pass_test.cc | 125 +++++++++++
 .../inference/api/paddle_pass_builder.cc      |   1 +
 paddle/phi/infermeta/fusion.cc                |  73 +++----
 7 files changed, 448 insertions(+), 39 deletions(-)
 create mode 100644 paddle/fluid/framework/ir/xpu/fused_multi_transformer_cachekv_layout_trans_pass.cc
 create mode 100644 paddle/fluid/framework/ir/xpu/fused_multi_transformer_cachekv_layout_trans_pass.h
 create mode 100644 paddle/fluid/framework/ir/xpu/fused_multi_transformer_cachekv_layout_trans_pass_test.cc

diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt
index 9fa7527827ae4..944dbd07bab5c 100755
--- a/paddle/fluid/framework/ir/CMakeLists.txt
+++ b/paddle/fluid/framework/ir/CMakeLists.txt
@@ -245,6 +245,8 @@ if(WITH_XPU)
   pass_library(fused_multi_transformer_xpu_quant_pass inference DIR xpu DEPS
                ${XPU_PASS_DEPS})
   pass_library(stack_fuse_pass inference DIR xpu DEPS ${XPU_PASS_DEPS})
+  pass_library(fused_multi_transformer_cachekv_layout_trans_pass inference DIR
+               xpu DEPS ${XPU_PASS_DEPS})
 endif()
 
 cc_library(
@@ -528,4 +530,8 @@ if(WITH_XPU)
     test_stack_fuse_pass
     SRCS xpu/stack_fuse_pass_test.cc
     DEPS stack_fuse_pass)
+  cc_test(
+    test_fused_multi_transformer_cachekv_layout_trans_pass
+    SRCS xpu/fused_multi_transformer_cachekv_layout_trans_pass_test.cc
+    DEPS fused_multi_transformer_cachekv_layout_trans_pass)
 endif()
diff --git a/paddle/fluid/framework/ir/pass.cc b/paddle/fluid/framework/ir/pass.cc
index cc4033f7f5a54..6611be59fcc80 100644
--- a/paddle/fluid/framework/ir/pass.cc
+++ b/paddle/fluid/framework/ir/pass.cc
@@ -62,6 +62,7 @@ static const std::vector<std::string> xpu_support_subgraph_passes = {
     "embedding_with_eltwise_add_xpu_fuse_pass",
     "multi_encoder_xpu_fuse_pass",
     "multi_encoder_xpu_slice_fuse_pass",
+    "fused_multi_transformer_cachekv_layout_trans_pass",
     "one_beam_size_fuse_pass",
     "stack_fuse_pass",
     "fused_multi_transformer_xpu_quant_pass",
diff --git a/paddle/fluid/framework/ir/xpu/fused_multi_transformer_cachekv_layout_trans_pass.cc b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_cachekv_layout_trans_pass.cc
new file mode 100644
index 0000000000000..993b5a055280d
--- /dev/null
+++ b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_cachekv_layout_trans_pass.cc
@@ -0,0 +1,202 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/framework/ir/xpu/fused_multi_transformer_cachekv_layout_trans_pass.h"
+#include <string>
+#include "paddle/fluid/framework/ir/graph_pattern_detector.h"
+#include "paddle/fluid/framework/ir/xpu/pass_utils.h"
+#include "paddle/fluid/framework/op_version_registry.h"
+#include "paddle/fluid/platform/enforce.h"
+
+namespace paddle {
+namespace framework {
+namespace ir {
+namespace patterns {
+
+struct FusedMultiTransformerFillConstantPattern : public PatternBase {
+  FusedMultiTransformerFillConstantPattern(PDPattern* pattern,
+                                           const std::string& name_scope);
+
+  // declare operator node's name
+  PATTERN_DECL_NODE(fill_constant);
+  PATTERN_DECL_NODE(fused_multi_transformer);
+  // declare variable node's name
+  PATTERN_DECL_NODE(fill_constant_out);
+};  // struct FusedMultiTransformerFillConstantPattern
+
+FusedMultiTransformerFillConstantPattern::
+    FusedMultiTransformerFillConstantPattern(PDPattern* pattern,
+                                             const std::string& name_scope)
+    : PatternBase(pattern, name_scope, name_scope) {
+  auto* fill_constant = pattern->NewNode(fill_constant_repr())
+                            ->assert_is_op("fill_constant")
+                            ->assert_has_n_inputs(5)
+                            ->assert_more([](Node* node) {
+                              return node->Op()->GetAttrIfExists<std::string>(
+                                         "friendly_device_type") != "xpu";
+                            });
+  auto* fill_constant_out = pattern->NewNode(fill_constant_out_repr())
+                                ->assert_is_op_output("fill_constant", "Out");
+  auto* fused_multi_transformer =
+      pattern->NewNode(fused_multi_transformer_repr())
+          ->assert_is_op("fused_multi_transformer");
+
+  fill_constant->LinksTo({fill_constant_out});
+  fused_multi_transformer->LinksFrom({fill_constant_out});
+}
+
+struct FusedMultiTransformerGatherPattern : public PatternBase {
+  FusedMultiTransformerGatherPattern(PDPattern* pattern,
+                                     const std::string& name_scope);
+
+  // declare operator node's name
+  PATTERN_DECL_NODE(fused_multi_transformer);
+  PATTERN_DECL_NODE(gather);
+  // declare variable node's name
+  PATTERN_DECL_NODE(gather_in);
+  PATTERN_DECL_NODE(gather_out);
+};  // struct FusedMultiTransformerGatherPattern
+
+FusedMultiTransformerGatherPattern::FusedMultiTransformerGatherPattern(
+    PDPattern* pattern, const std::string& name_scope)
+    : PatternBase(pattern, name_scope, name_scope) {
+  auto* gather_in =
+      pattern->NewNode(gather_in_repr())->assert_is_op_input("gather", "X");
+  auto* gather = pattern->NewNode(gather_repr())
+                     ->assert_is_op("gather")
+                     ->assert_more([](Node* node) {
+                       return node->Op()->GetAttrIfExists<int>("axis") == 1;
+                     });
+  auto* gather_out =
+      pattern->NewNode(gather_out_repr())->assert_is_op_output("gather", "Out");
+  auto* fused_multi_transformer =
+      pattern->NewNode(fused_multi_transformer_repr())
+          ->assert_is_op("fused_multi_transformer");
+
+  gather->LinksFrom({gather_in}).LinksTo({gather_out});
+  fused_multi_transformer->LinksFrom({gather_out});
+}
+}  // namespace patterns
+
+void FusedMultiTransformerCacheKVLayoutTransPass::FillConstantReshapePass(
+    ir::Graph* graph) const {
+  PADDLE_ENFORCE_NOT_NULL(
+      graph, platform::errors::PreconditionNotMet("graph should not be null."));
+  GraphPatternDetector gpd;
+  patterns::FusedMultiTransformerFillConstantPattern pattern(
+      gpd.mutable_pattern(), name_scope_);
+
+  int found_subgraph_count = 0;
+  auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph,
+                     Graph* graph) {
+    VLOG(4) << "handle FillConstantReshapePass";
+    GET_IR_NODE(fused_multi_transformer);
+    GET_IR_NODE(fill_constant);
+    GET_IR_NODE(fill_constant_out);
+    auto cachekv_names = fused_multi_transformer->Op()->Input("CacheKV");
+    if (std::count(cachekv_names.begin(),
+                   cachekv_names.end(),
+                   fill_constant_out->Name()) == 0)
+      return;
+
+    auto fill_constant_input_names =
+        fill_constant->Op()->Input("ShapeTensorList");
+    auto fill_constant_trans_input_names =
+        std::vector<std::string>{fill_constant_input_names[0],
+                                 fill_constant_input_names[3],
+                                 fill_constant_input_names[1],
+                                 fill_constant_input_names[2],
+                                 fill_constant_input_names[4]};
+    fill_constant->Op()->SetInput("ShapeTensorList",
+                                  fill_constant_trans_input_names);
+
+    auto fill_constant_output_shape = fill_constant_out->Var()->GetShape();
+    fill_constant_out->Var()->SetShape({fill_constant_output_shape[0],
+                                        fill_constant_output_shape[3],
+                                        fill_constant_output_shape[1],
+                                        fill_constant_output_shape[2],
+                                        fill_constant_output_shape[4]});
+
+    fused_multi_transformer->Op()->SetAttr("friendly_device_type",
+                                           std::string("xpu"));
+    fill_constant->Op()->SetAttr("friendly_device_type", std::string("xpu"));
+    found_subgraph_count++;
+  };
+
+  gpd(graph, handler);
+  AddStatis(found_subgraph_count);
+}
+
+void FusedMultiTransformerCacheKVLayoutTransPass::GatherReshapePass(
+    ir::Graph* graph) const {
+  PADDLE_ENFORCE_NOT_NULL(
+      graph, platform::errors::PreconditionNotMet("graph should not be null."));
+  GraphPatternDetector gpd;
+  patterns::FusedMultiTransformerGatherPattern pattern(gpd.mutable_pattern(),
+                                                       name_scope_);
+
+  int found_subgraph_count = 0;
+  auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph,
+                     Graph* graph) {
+    VLOG(4) << "handle GatherReshapePass";
+    GET_IR_NODE(gather);
+    GET_IR_NODE(fused_multi_transformer);
+    GET_IR_NODE(gather_in);
+    GET_IR_NODE(gather_out);
+    auto cachekv_names = fused_multi_transformer->Op()->Input("CacheKV");
+    if (std::count(cachekv_names.begin(),
+                   cachekv_names.end(),
+                   gather_out->Name()) == 0)
+      return;
+
+    auto gather_in_shape = gather_in->Var()->GetShape();
+    auto gather_out_shape = gather_out->Var()->GetShape();
+    gather_in->Var()->SetShape({gather_in_shape[0],
+                                gather_in_shape[3],
+                                gather_in_shape[1],
+                                gather_in_shape[2],
+                                gather_in_shape[4]});
+    gather_out->Var()->SetShape({gather_out_shape[0],
+                                 gather_out_shape[3],
+                                 gather_out_shape[1],
+                                 gather_out_shape[2],
+                                 gather_out_shape[4]});
+    gather->Op()->SetAttr("axis", 2);
+    fused_multi_transformer->Op()->SetAttr("friendly_device_type",
+                                           std::string("xpu"));
+
+    found_subgraph_count++;
+  };
+
+  gpd(graph, handler);
+  AddStatis(found_subgraph_count);
+}
+
+void FusedMultiTransformerCacheKVLayoutTransPass::ApplyImpl(
+    ir::Graph* graph) const {
+  PADDLE_ENFORCE_NOT_NULL(
+      graph, platform::errors::PreconditionNotMet("graph should not be null."));
+  Init(name_scope_, graph);
+
+  FillConstantReshapePass(graph);
+  GatherReshapePass(graph);
+}
+
+}  // namespace ir
+}  // namespace framework
+}  // namespace paddle
+
+REGISTER_PASS(
+    fused_multi_transformer_cachekv_layout_trans_pass,
+    paddle::framework::ir::FusedMultiTransformerCacheKVLayoutTransPass);
diff --git a/paddle/fluid/framework/ir/xpu/fused_multi_transformer_cachekv_layout_trans_pass.h b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_cachekv_layout_trans_pass.h
new file mode 100644
index 0000000000000..cb87317a76e6a
--- /dev/null
+++ b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_cachekv_layout_trans_pass.h
@@ -0,0 +1,79 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <string>
+#include "paddle/fluid/framework/ir/fuse_pass_base.h"
+#include "paddle/fluid/framework/ir/pass.h"
+
+namespace phi {
+class DenseTensor;
+}  // namespace phi
+
+namespace paddle {
+namespace framework {
+class Scope;
+}  // namespace framework
+}  // namespace paddle
+
+namespace paddle {
+namespace framework {
+namespace ir {
+
+class FusedMultiTransformerCacheKVLayoutTransPass : public FusePassBase {
+ protected:
+  void ApplyImpl(ir::Graph* graph) const override;
+
+ private:
+  /*
+   Origin subgraph:
+        (ShapeTensorList: [d0,d1,d2,d3,d4])
+                    |
+              fill_constant
+                    |
+            fused_multi_transformer
+
+   Fused subgraph:
+        (ShapeTensorList: [d0,d3,d1,d2,d4])
+                    |
+              fill_constant
+                    |
+            fused_multi_transformer
+  */
+  void FillConstantReshapePass(ir::Graph* graph) const;
+
+  /*
+   Origin subgraph:
+        (gather_x: [d0,d1,d2,d3,d4])
+                    |
+              gather(axis=1)
+                    |
+            fused_multi_transformer
+
+   Fused subgraph:
+        (gather_x: [d0,d3,d1,d2,d4])
+                    |
+              gather(axis=2)
+                    |
+            fused_multi_transformer
+  */
+  void GatherReshapePass(ir::Graph* graph) const;
+
+  const std::string name_scope_{
+      "fused_multi_transformer_cachekv_layout_trans_pass"};
+};
+
+}  // namespace ir
+}  // namespace framework
+}  // namespace paddle
diff --git a/paddle/fluid/framework/ir/xpu/fused_multi_transformer_cachekv_layout_trans_pass_test.cc b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_cachekv_layout_trans_pass_test.cc
new file mode 100644
index 0000000000000..ec5dba201163f
--- /dev/null
+++ b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_cachekv_layout_trans_pass_test.cc
@@ -0,0 +1,125 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+#include "paddle/fluid/framework/ir/pass.h"
+#include "paddle/fluid/framework/ir/pass_tester_helper.h"
+
+namespace paddle {
+namespace framework {
+namespace ir {
+
+VarDesc* Data(paddle::framework::BlockDesc* block,
+              std::string name,
+              std::vector<int64_t> shape = {},
+              bool is_persistable = false,
+              proto::VarType::Type data_type = proto::VarType::FP32) {
+  auto* var = block->Var(name);
+  var->SetType(proto::VarType::LOD_TENSOR);
+  var->SetDataType(data_type);
+  var->SetShape(shape);
+  var->SetPersistable(is_persistable);
+  return var;
+}
+
+VarDesc* fill_constant(BlockDesc* block, std::vector<VarDesc*> shapes) {
+  VarDesc* out = Data(block, shapes[0]->Name() + "_out");
+  OpDesc* op = block->AppendOp();
+  op->SetType("fill_constant");
+  std::vector<std::string> shape_names;
+  for (auto shape : shapes) {
+    shape_names.push_back(shape->Name());
+  }
+  op->SetInput("ShapeTensorList", {shape_names});
+  op->SetOutput("Out", {out->Name()});
+  return out;
+}
+
+TEST(FillConstantReshapePass, basic) {
+  paddle::framework::ProgramDesc program;
+  auto* block = program.MutableBlock(0);
+  auto* shape0 = Data(block, "shape0");
+  auto* shape1 = Data(block, "shape1");
+  auto* shape2 = Data(block, "shape2");
+  auto* shape3 = Data(block, "shape3");
+  auto* shape4 = Data(block, "shape4");
+  auto* shape5 = Data(block, "shape5");
+  auto* shape6 = Data(block, "shape6");
+  auto* shape7 = Data(block, "shape7");
+  auto* shape8 = Data(block, "shape8");
+  auto* shape9 = Data(block, "shape9");
+  auto* fill0 = fill_constant(block, {shape0, shape1, shape2, shape3, shape4});
+  fill0->SetShape({1, 2, 3, 4, 5});
+  auto* fill1 = fill_constant(block, {shape5, shape6, shape7, shape8, shape9});
+  fill1->SetShape({1, 2, 3, 4, 5});
+  OpDesc* fused_multi_transformer = block->AppendOp();
+  fused_multi_transformer->SetType("fused_multi_transformer");
+  fused_multi_transformer->SetInput("CacheKV", {fill0->Name(), fill1->Name()});
+
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(program));
+  auto pass = PassRegistry::Instance().Get(
+      "fused_multi_transformer_cachekv_layout_trans_pass");
+  pass->Apply(graph.get());
+  auto fills = GetOpNodes(graph, "fill_constant");
+  auto fill0_in_names = fills[0]->Op()->Input("ShapeTensorList");
+  std::vector<std::string> expect_fill0_in_names{
+      "shape0", "shape3", "shape1", "shape2", "shape4"};
+  PADDLE_ENFORCE_EQ(fill0_in_names,
+                    expect_fill0_in_names,
+                    platform::errors::PreconditionNotMet(
+                        "fill_constant name should be updated."));
+  auto fill1_in_names = fills[1]->Op()->Input("ShapeTensorList");
+  std::vector<std::string> expect_fill1_in_names{
+      "shape5", "shape8", "shape6", "shape7", "shape9"};
+  PADDLE_ENFORCE_EQ(fill1_in_names,
+                    expect_fill1_in_names,
+                    platform::errors::PreconditionNotMet(
+                        "fill_constant name should be updated."));
+}
+
+TEST(GatherReshapePass, basic) {
+  Layers layers;
+  auto* gather0_x = layers.data("gather0_x", {2, 1, 24, 512, 64});
+  auto* gather0_index = layers.data("gather0_index", {1});
+  auto* gather0_out = layers.gather(gather0_x, gather0_index, 1);
+  gather0_out->SetShape({2, 1, 24, 512, 64});
+  auto* gather1_x = layers.data("gather1_x", {2, 1, 24, 512, 64});
+  auto* gather1_index = layers.data("gather1_index", {1});
+  auto* gather1_out = layers.gather(gather1_x, gather1_index, 1);
+  gather1_out->SetShape({2, 1, 24, 512, 64});
+  auto* block = layers.Block();
+  OpDesc* fused_multi_transformer = block->AppendOp();
+  fused_multi_transformer->SetType("fused_multi_transformer");
+  fused_multi_transformer->SetInput("CacheKV",
+                                    {gather0_out->Name(), gather1_out->Name()});
+
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(layers.main_program()));
+  auto pass = PassRegistry::Instance().Get(
+      "fused_multi_transformer_cachekv_layout_trans_pass");
+  pass->Apply(graph.get());
+  auto gathers = GetOpNodes(graph, "gather");
+  for (auto* gather : gathers) {
+    PADDLE_ENFORCE_EQ(
+        gather->Op()->GetAttrIfExists<int>("axis"),
+        2,
+        platform::errors::PreconditionNotMet(
+            "gather's axis attr should be updated to 2 by pass."));
+  }
+}
+
+}  // namespace ir
+}  // namespace framework
+}  // namespace paddle
+
+USE_PASS(fused_multi_transformer_cachekv_layout_trans_pass);
diff --git a/paddle/fluid/inference/api/paddle_pass_builder.cc b/paddle/fluid/inference/api/paddle_pass_builder.cc
index 1be90972b924b..e7c24272b81c5 100644
--- a/paddle/fluid/inference/api/paddle_pass_builder.cc
+++ b/paddle/fluid/inference/api/paddle_pass_builder.cc
@@ -519,6 +519,7 @@ XpuPassStrategy::XpuPassStrategy() : PassStrategy({}) {
       "embedding_with_eltwise_add_xpu_fuse_pass",
       "multi_encoder_xpu_fuse_pass",
       "multi_encoder_xpu_slice_fuse_pass",
+      "fused_multi_transformer_cachekv_layout_trans_pass",
       "one_beam_size_fuse_pass",
       "delete_cast_op_pass",
       "stack_fuse_pass",
diff --git a/paddle/phi/infermeta/fusion.cc b/paddle/phi/infermeta/fusion.cc
index 5c0aa3b8e89fd..f775cedce8c11 100644
--- a/paddle/phi/infermeta/fusion.cc
+++ b/paddle/phi/infermeta/fusion.cc
@@ -291,31 +291,26 @@ void FusedMultiTransformerXpuInferMeta(
     std::vector<MetaTensor*> cache_kv_out) {
   auto x_dim = x.dims();
   auto y_dim = qkvw[0]->dims();
-  PADDLE_ENFORCE_EQ(
-      x_dim.size(),
-      3,
-      phi::errors::InvalidArgument("The dimensions of x must be 3"
-                                   "(batch_size, seq_len, dim_embed),"
-                                   "but received dimensions of"
-                                   "Input is [%d]",
-                                   x_dim.size()));
+  PADDLE_ENFORCE_EQ(x_dim.size(),
+                    3,
+                    phi::errors::InvalidArgument(
+                        "The dimensions of x must be 3(batch_size, seq_len, "
+                        "dim_embed), but received dimensions of Input is [%d]",
+                        x_dim.size()));
   PADDLE_ENFORCE_EQ(
       y_dim.size(),
       4,
-      phi::errors::InvalidArgument("The dimensions of qkv_weight must be 4"
-                                   "(3, num_head, dim_head, dim_embed),"
-                                   "but received dimensions of"
-                                   "Input is [%d]",
-                                   y_dim.size()));
+      phi::errors::InvalidArgument(
+          "The dimensions of qkv_weight must be 4(3, num_head, dim_head, "
+          "dim_embed), but received dimensions of qkv_weight is [%d]",
+          y_dim.size()));
   PADDLE_ENFORCE_EQ(
       x_dim[2],
       trans_qkvw ? y_dim[3] : y_dim[0],
       phi::errors::InvalidArgument(
-          "ShapeError: the dimension of x_dim[2] and y_dim[3](trans_qkvw is "
-          "true) or y_dim[0](trans_qkvw is false)"
-          "must be equal. But received: the shape "
-          "of input x = [%s], and the shape of "
-          "input qkv_weight = [%s]",
+          "The dimension of x_dim[2] and y_dim[3](trans_qkvw is  true) or "
+          "y_dim[0](trans_qkvw is false) must be equal, but received: the "
+          "shape of input x = [%s], and the shape of input qkv_weight = [%s]",
           x_dim,
           y_dim));
   if (cache_kv.size() > 0) {
@@ -330,27 +325,27 @@ void FusedMultiTransformerXpuInferMeta(
                       phi::errors::InvalidArgument(
                           "The first dim of CacheKV must be 2, but got %d",
                           c_dim[0]));  // 2
-    PADDLE_ENFORCE_EQ(c_dim[1],
-                      x_dim[0],
-                      phi::errors::InvalidArgument(
-                          "The second dim of CacheKV must be equal with "
-                          "batch size %d, but got %d",
-                          x_dim[0],
-                          c_dim[1]));  // batch_size
-    PADDLE_ENFORCE_EQ(c_dim[2],
-                      trans_qkvw ? y_dim[1] : y_dim[2],
-                      phi::errors::InvalidArgument(
-                          "The third dim of CacheKV must be equal with num "
-                          "head %d, but got %d",
-                          trans_qkvw ? y_dim[1] : y_dim[2],
-                          c_dim[2]));  // num_head
-    PADDLE_ENFORCE_EQ(c_dim[4],
-                      trans_qkvw ? y_dim[2] : y_dim[3],
-                      phi::errors::InvalidArgument(
-                          "The fifth dim of CacheKV must be equal with head "
-                          "size %d, but got %d",
-                          trans_qkvw ? y_dim[2] : y_dim[3],
-                          c_dim[4]));  // head_size
+    PADDLE_ENFORCE_EQ(
+        c_dim[2],
+        x_dim[0],
+        phi::errors::InvalidArgument("The third dim of CacheKV must be equal "
+                                     "with batch size %d, but got %d",
+                                     x_dim[0],
+                                     c_dim[2]));  // batch_size
+    PADDLE_ENFORCE_EQ(
+        c_dim[3],
+        trans_qkvw ? y_dim[1] : y_dim[2],
+        phi::errors::InvalidArgument("The fourth dim of CacheKV must be equal "
+                                     "with num head %d, but got %d",
+                                     trans_qkvw ? y_dim[1] : y_dim[2],
+                                     c_dim[3]));  // num_head
+    PADDLE_ENFORCE_EQ(
+        c_dim[4],
+        trans_qkvw ? y_dim[2] : y_dim[3],
+        phi::errors::InvalidArgument("The fifth dim of CacheKV must be equal "
+                                     "with head size %d, but got %d",
+                                     trans_qkvw ? y_dim[2] : y_dim[3],
+                                     c_dim[4]));  // head_size
   }
 
   out->set_dims(x_dim);

From a0aff194a181de8d7b31932bbfcaee11ce7fcd8b Mon Sep 17 00:00:00 2001
From: Zhang Zheng <32410583+ZzSean@users.noreply.github.com>
Date: Mon, 24 Apr 2023 17:01:07 +0800
Subject: [PATCH 026/405] Fix the calculation of layer_norm_bwd (#53224)

* Fix the calculation of layer_norm_bwd

* fix
---
 paddle/phi/kernels/funcs/layer_norm_impl.cu.h | 29 +++++++++----------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/paddle/phi/kernels/funcs/layer_norm_impl.cu.h b/paddle/phi/kernels/funcs/layer_norm_impl.cu.h
index d8ade4612c85e..b240be28ec949 100644
--- a/paddle/phi/kernels/funcs/layer_norm_impl.cu.h
+++ b/paddle/phi/kernels/funcs/layer_norm_impl.cu.h
@@ -1603,13 +1603,13 @@ __global__ void LayerNormBackwardGradientAll(
 
   for (int64_t i = beg_idx; i < end_idx; i += stride) {
     int row_idx = i / feature_size;
-    auto var_val = real_sqrt(static_cast<U>(var[row_idx]) + epsilon);
+    auto var_val = rsqrt_(static_cast<U>(var[row_idx]) + epsilon);
     d_scale_partial += static_cast<U>(d_y[i]) *
-                       (static_cast<U>(x[i]) - mean[row_idx]) / var_val;
+                       (static_cast<U>(x[i]) - mean[row_idx]) * var_val;
     d_bias_partial += static_cast<U>(d_y[i]);
     if (HasDx) {
       d_x[i] = static_cast<T>(static_cast<U>(d_y[i]) *
-                              static_cast<U>(scale[blockIdx.x + col_offset]) /
+                              static_cast<U>(scale[blockIdx.x + col_offset]) *
                               var_val);
     }
   }
@@ -1659,10 +1659,10 @@ __global__ void LayerNormBackwardGradientScaleOrBias(
   for (int64_t i = beg_idx; i < end_idx; i += stride) {
     int row_idx = i / feature_size;
     auto var_val =
-        static_cast<U>(real_sqrt(static_cast<float>(var[row_idx]) + epsilon));
+        static_cast<U>(rsqrt_(static_cast<float>(var[row_idx]) + epsilon));
     if (HasDScale) {
       d_scale_or_d_bias_partial += static_cast<U>(d_y[i]) *
-                                   (static_cast<U>(x[i]) - mean[row_idx]) /
+                                   (static_cast<U>(x[i]) - mean[row_idx]) *
                                    var_val;
     } else {  // d_bias != nullptr
       d_scale_or_d_bias_partial += static_cast<U>(d_y[i]);
@@ -1671,10 +1671,10 @@ __global__ void LayerNormBackwardGradientScaleOrBias(
     if (HasDx) {
       if (scale != nullptr) {
         d_x[i] = static_cast<T>(static_cast<U>(d_y[i]) *
-                                static_cast<U>(scale[blockIdx.x + col_offset]) /
+                                static_cast<U>(scale[blockIdx.x + col_offset]) *
                                 var_val);
       } else {
-        d_x[i] = static_cast<T>(static_cast<U>(d_y[i]) / var_val);
+        d_x[i] = static_cast<T>(static_cast<U>(d_y[i]) * var_val);
       }
     }
   }
@@ -1762,13 +1762,13 @@ __global__ void LayerNormBackwardGradientOnlyDX(
   U d_x_mean_partial = static_cast<U>(0), d_x_var_partial = static_cast<U>(0);
   for (int64_t i = beg_idx; i < end_idx; i += BlockDim) {
     auto var_val =
-        static_cast<U>(real_sqrt(static_cast<float>(block_var) + epsilon));
+        static_cast<U>(rsqrt_(static_cast<float>(block_var) + epsilon));
     if (scale != nullptr) {
       int col_idx = i % feature_size;
       d_x[i] = static_cast<T>(static_cast<U>(d_y[i]) *
-                              static_cast<U>(scale[col_idx]) / var_val);
+                              static_cast<U>(scale[col_idx]) * var_val);
     } else {
-      d_x[i] = static_cast<T>(static_cast<U>(d_y[i]) / var_val);
+      d_x[i] = static_cast<T>(static_cast<U>(d_y[i]) * var_val);
     }
     d_x_mean_partial += static_cast<U>(d_x[i]);
     d_x_var_partial +=
@@ -1812,21 +1812,20 @@ __global__ void LayerNormBackwardWhenBatchSizeIsOne(
   int64_t idx = threadIdx.x + blockIdx.x * blockDim.x;
   using ScaleBiasT = LayerNormScaleBiasT<T, U, ScaleBiasWithSameTypeX>;
   if (idx < feature_size) {
-    auto var_val =
-        static_cast<U>(real_sqrt(static_cast<float>(var[0]) + epsilon));
+    auto var_val = static_cast<U>(rsqrt_(static_cast<float>(var[0]) + epsilon));
     if (d_x != nullptr) {
       if (d_scale == nullptr) {
-        d_x[idx] = static_cast<T>(static_cast<U>(d_y[idx]) / var_val);
+        d_x[idx] = static_cast<T>(static_cast<U>(d_y[idx]) * var_val);
       } else {
         d_x[idx] = static_cast<T>(static_cast<U>(d_y[idx]) *
-                                  static_cast<U>(scale[idx]) / var_val);
+                                  static_cast<U>(scale[idx]) * var_val);
       }
     }
 
     if (d_scale != nullptr) {
       d_scale[idx] =
           static_cast<ScaleBiasT>(static_cast<U>(d_y[idx]) *
-                                  (static_cast<U>(x[idx]) - mean[0]) / var_val);
+                                  (static_cast<U>(x[idx]) - mean[0]) * var_val);
     }
 
     if (d_bias != nullptr) {

From c0a604e79c83b57f1b75f539b11f17899ced8bd3 Mon Sep 17 00:00:00 2001
From: YangQun <qun.yang@intel.com>
Date: Mon, 24 Apr 2023 02:01:29 -0700
Subject: [PATCH 027/405] [Zero-Dim] support 0d tensor for shape and squeeze
 onednn kernel (#52832)

* support 0d tensor for shape and squeeze onednn kernel

* set python api for shape op ut
---
 paddle/fluid/operators/mkldnn/reshape_mkldnn_op.cc |  5 +++--
 paddle/phi/kernels/onednn/squeeze_grad_kernel.cc   |  4 +++-
 paddle/phi/kernels/onednn/squeeze_kernel.cc        |  6 ++++--
 .../unittests/white_list/check_shape_white_list.py |  2 ++
 test/mkldnn/test_shape_mkldnn_op.py                |  9 ++++++++-
 test/mkldnn/test_squeeze2_mkldnn_op.py             | 14 ++++++++++++++
 6 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/paddle/fluid/operators/mkldnn/reshape_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/reshape_mkldnn_op.cc
index 79e84e4755337..e2fc1c19f8e39 100644
--- a/paddle/fluid/operators/mkldnn/reshape_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/reshape_mkldnn_op.cc
@@ -89,8 +89,9 @@ class ReshapeMKLDNNKernel : public framework::OpKernel<T> {
     astream.wait();
 
     out->Resize(out_dims);
-    out->set_mem_desc(
-        reorder_dst_memory_p->get_desc().reshape(phi::vectorize(out_dims)));
+    auto reshape_dims = out_dims.size() != 0 ? phi::vectorize(out_dims)
+                                             : std::vector<int64_t>{1};
+    out->set_mem_desc(reorder_dst_memory_p->get_desc().reshape(reshape_dims));
   }
 
   void InferInOutShape(const framework::ExecutionContext& ctx,
diff --git a/paddle/phi/kernels/onednn/squeeze_grad_kernel.cc b/paddle/phi/kernels/onednn/squeeze_grad_kernel.cc
index 654acfe5700c3..93b8fc75a9a48 100644
--- a/paddle/phi/kernels/onednn/squeeze_grad_kernel.cc
+++ b/paddle/phi/kernels/onednn/squeeze_grad_kernel.cc
@@ -25,7 +25,9 @@ void SqueezeGradKernel(const Context& dev_ctx,
                        const DenseTensor& dout,
                        const IntArray& axes,
                        DenseTensor* dx) {
-  auto dout_vec_dims = vectorize(dout.dims());
+  auto dout_vec_dims = dout.dims().size() != 0 ? vectorize(dout.dims())
+                                               : std::vector<int64_t>{1};
+
   auto dout_type = funcs::ToOneDNNDataType(dout.dtype());
 
   funcs::ReorderOneDNNHandler reorder_handler(
diff --git a/paddle/phi/kernels/onednn/squeeze_kernel.cc b/paddle/phi/kernels/onednn/squeeze_kernel.cc
index 0ad82bfeddace..9b86f9e1a9c74 100644
--- a/paddle/phi/kernels/onednn/squeeze_kernel.cc
+++ b/paddle/phi/kernels/onednn/squeeze_kernel.cc
@@ -47,8 +47,10 @@ void ExecuteSqueeze(const Context& dev_ctx,
   astream.wait();
 
   out->Resize(out_dims);
-  out->set_mem_desc(
-      reorder_dst_memory_p->get_desc().reshape(vectorize(out_dims)));
+
+  auto reshape_dims =
+      out_dims.size() != 0 ? vectorize(out_dims) : std::vector<int64_t>{1};
+  out->set_mem_desc(reorder_dst_memory_p->get_desc().reshape(reshape_dims));
 }
 
 template <typename T, typename Context>
diff --git a/python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py b/python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py
index 86f84d00cea55..db5a710867277 100644
--- a/python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py
+++ b/python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py
@@ -31,4 +31,6 @@
     'multi_dot',
     'index_add',
     'reshape2',
+    'squeeze',
+    'squeeze2',
 ]
diff --git a/test/mkldnn/test_shape_mkldnn_op.py b/test/mkldnn/test_shape_mkldnn_op.py
index 2616ad65458c0..b954336de23e2 100644
--- a/test/mkldnn/test_shape_mkldnn_op.py
+++ b/test/mkldnn/test_shape_mkldnn_op.py
@@ -21,10 +21,10 @@
 from paddle.fluid.tests.unittests.eager_op_test import OpTest, OpTestTool
 
 
-@OpTestTool.skip_if_not_cpu_bf16()
 class TestShape3DFP32OneDNNOp(OpTest):
     def setUp(self):
         self.op_type = "shape"
+        self.python_api = paddle.tensor.shape
         self.config()
         self.attrs = {'use_mkldnn': True}
         self.inputs = {'Input': np.zeros(self.shape).astype(self.dtype)}
@@ -38,6 +38,13 @@ def test_check_output(self):
         self.check_output_with_place(core.CPUPlace())
 
 
+class TestShape0DFP32OneDNNOp(TestShape3DFP32OneDNNOp):
+    def config(self):
+        self.shape = []
+        self.dtype = np.float32
+
+
+@OpTestTool.skip_if_not_cpu_bf16()
 class TestShape6DBF16OneDNNOp(TestShape3DFP32OneDNNOp):
     def config(self):
         self.shape = [10, 2, 3, 4, 5, 2]
diff --git a/test/mkldnn/test_squeeze2_mkldnn_op.py b/test/mkldnn/test_squeeze2_mkldnn_op.py
index ea914f62438d8..0904be811c0bc 100644
--- a/test/mkldnn/test_squeeze2_mkldnn_op.py
+++ b/test/mkldnn/test_squeeze2_mkldnn_op.py
@@ -76,6 +76,20 @@ def test_check_output(self):
         self.check_output_with_place(core.CPUPlace())
 
 
+class TestSqueeze2OneDNNOp_ZeroDim(TestSqueeze2OneDNNOp):
+    def init_test_case(self):
+        self.ori_shape = [1]
+        self.axes = ()
+        self.new_shape = ()
+
+
+class TestSqueezeOneDNNOp_ZeroDim(TestSqueezeOneDNNOp):
+    def init_test_case(self):
+        self.ori_shape = [1]
+        self.axes = ()
+        self.new_shape = ()
+
+
 class TestSqueeze2OneDNNOp1(TestSqueeze2OneDNNOp):
     def init_test_case(self):
         self.ori_shape = (1, 20, 1, 5)

From 41138718fa8e14e0d95350837385d6d462a3ba72 Mon Sep 17 00:00:00 2001
From: niuliling123 <51102941+niuliling123@users.noreply.github.com>
Date: Mon, 24 Apr 2023 17:23:25 +0800
Subject: [PATCH 028/405] Add     "enable_tensor_checker" and  
 "disable_tensor_checker" to api list (#52936)

---
 paddle/fluid/eager/nan_inf_utils.cc           |  26 +-
 paddle/fluid/eager/nan_inf_utils.h            |   4 +
 .../framework/details/nan_inf_utils_detail.cc |   8 +
 .../framework/details/nan_inf_utils_detail.cu |   2 +-
 .../framework/details/nan_inf_utils_detail.h  |   4 +
 paddle/fluid/pybind/pybind.cc                 |  12 +
 python/paddle/amp/debugging.py                | 356 ++++++++++--------
 .../fluid/tests/unittests/test_nan_inf_dir.py | 145 +------
 .../tests/unittests/test_tensor_checker.py    | 111 ++++++
 9 files changed, 359 insertions(+), 309 deletions(-)
 create mode 100644 python/paddle/fluid/tests/unittests/test_tensor_checker.py

diff --git a/paddle/fluid/eager/nan_inf_utils.cc b/paddle/fluid/eager/nan_inf_utils.cc
index 6eae40fca36cf..e71ae7cf11939 100644
--- a/paddle/fluid/eager/nan_inf_utils.cc
+++ b/paddle/fluid/eager/nan_inf_utils.cc
@@ -24,8 +24,6 @@
 DECLARE_int32(check_nan_inf_level);
 namespace egr {
 
-static std::once_flag dump_list_init_flag;
-
 static std::unordered_set<std::string>& nan_inf_check_op_list() {
   static std::unordered_set<std::string> _check_op_list = {};
   return _check_op_list;
@@ -36,39 +34,32 @@ static std::unordered_set<std::string>& nan_inf_skip_op_list() {
   return _skip_op_list;
 }
 
-static void InitDumpListFormEnv() {
+void SetCheckOpList(const std::string& check_op_list = "") {
   nan_inf_check_op_list();
-  nan_inf_skip_op_list();
-  const char* check_op_list = std::getenv("Paddle_check_nan_inf_op_list");
-  const char* skip_op_list = std::getenv("Paddle_skip_nan_inf_op_list");
-
-  if (check_op_list) {
+  if (check_op_list.size() != 0) {
     std::stringstream ss(check_op_list);
     std::string op_type;
     LOG(INFO) << "Please set op's name according to the "
                  "paddle.amp.low_precision_op_list()";
     while (std::getline(ss, op_type, ',')) {
       nan_inf_check_op_list().emplace(op_type);
+      VLOG(4) << "Check nan inf op list: " << op_type;
     }
   }
+}
 
-  if (skip_op_list) {
+void SetSkipOpList(const std::string& skip_op_list = "") {
+  nan_inf_skip_op_list();
+  if (skip_op_list.size() != 0) {
     std::stringstream ss(skip_op_list);
     std::string op_type;
     LOG(INFO) << "Please set op's name according to the "
                  "paddle.amp.low_precision_op_list()";
     while (std::getline(ss, op_type, ',')) {
       nan_inf_skip_op_list().emplace(op_type);
+      VLOG(4) << "Skip nan inf op list: " << op_type;
     }
   }
-
-  for (auto const& key : nan_inf_check_op_list()) {
-    LOG(INFO) << "Check nan inf op list: " << key;
-  }
-
-  for (auto const& key : nan_inf_skip_op_list()) {
-    LOG(INFO) << "Skip nan inf op list: " << key;
-  }
 }
 
 bool CheckOp(const std::string& api_name) {
@@ -89,7 +80,6 @@ bool CheckOp(const std::string& api_name) {
 }
 
 void CheckTensorHasNanOrInf(const std::string& api_name, const Tensor& tensor) {
-  std::call_once(dump_list_init_flag, InitDumpListFormEnv);
   auto op_name = phi::TransToFluidOpName(api_name);
   if (tensor.initialized() && CheckOp(op_name)) {
     auto& tensor_name = tensor.name();
diff --git a/paddle/fluid/eager/nan_inf_utils.h b/paddle/fluid/eager/nan_inf_utils.h
index 8d7ed7ffb76b2..4f412cf6db88d 100644
--- a/paddle/fluid/eager/nan_inf_utils.h
+++ b/paddle/fluid/eager/nan_inf_utils.h
@@ -65,6 +65,10 @@ void CheckTensorHasNanOrInf(
 void CheckTensorHasNanOrInf(const std::string& api_name,
                             const TupleOfTensorAndVector& tensors);
 
+void SetCheckOpList(const std::string& check_op_list);
+
+void SetSkipOpList(const std::string& skip_op_list);
+
 void CheckTensorHasNanOrInf(
     const std::string& api_name,
     const paddle::small_vector<std::vector<paddle::Tensor>,
diff --git a/paddle/fluid/framework/details/nan_inf_utils_detail.cc b/paddle/fluid/framework/details/nan_inf_utils_detail.cc
index e3e08e8b7df28..7890e37e67246 100644
--- a/paddle/fluid/framework/details/nan_inf_utils_detail.cc
+++ b/paddle/fluid/framework/details/nan_inf_utils_detail.cc
@@ -30,6 +30,7 @@ namespace details {
 struct DebugTools {
   DebugTools() {}
   std::string path = "";
+  int stack_limit = 1;
 };
 static DebugTools debug_nan_inf;
 
@@ -45,6 +46,13 @@ std::string GetNanPath() {
   return debug_nan_inf.path + "/";
 }
 
+void SetNanInfStackLimit(const int& stack_limit) {
+  debug_nan_inf.stack_limit = stack_limit;
+  VLOG(4) << "Set the stack limit of debug tools : " << stack_limit;
+}
+
+int GetNanInfStackLimit() { return debug_nan_inf.stack_limit; }
+
 static std::once_flag white_list_init_flag;
 
 static int op_role_nan_inf_white_list = 0;
diff --git a/paddle/fluid/framework/details/nan_inf_utils_detail.cu b/paddle/fluid/framework/details/nan_inf_utils_detail.cu
index dd99adfecfcd9..3e001299e8e38 100644
--- a/paddle/fluid/framework/details/nan_inf_utils_detail.cu
+++ b/paddle/fluid/framework/details/nan_inf_utils_detail.cu
@@ -516,7 +516,7 @@ void TensorCheckerVisitor<phi::GPUContext>::apply(
                                        check_nan_inf_level,
                                        nan_inf_zero_tensor.data<int64_t>());
 
-  if (check_nan_inf_level == 0) {
+  if (check_nan_inf_level == 0 && GetNanInfStackLimit() > 0) {
     auto nan_cpu =
         phi::memory_utils::Alloc(phi::CPUPlace(), sizeof(int64_t) * 3);
     int64_t* nan_cpu_ptr = reinterpret_cast<int64_t*>(nan_cpu->ptr());
diff --git a/paddle/fluid/framework/details/nan_inf_utils_detail.h b/paddle/fluid/framework/details/nan_inf_utils_detail.h
index 8f5eb5352ac7b..59865162cc365 100644
--- a/paddle/fluid/framework/details/nan_inf_utils_detail.h
+++ b/paddle/fluid/framework/details/nan_inf_utils_detail.h
@@ -40,6 +40,10 @@ void SetNanInfDebugPath(const std::string& nan_inf_path);
 
 std::string GetNanPath();
 
+void SetNanInfStackLimit(const int& stack_limit);
+
+int GetNanInfStackLimit();
+
 template <typename T,
           typename MT,
           std::enable_if_t<std::is_same<T, float>::value, bool> = true>
diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc
index bde6357ccbe2f..359c2266f8ea3 100644
--- a/paddle/fluid/pybind/pybind.cc
+++ b/paddle/fluid/pybind/pybind.cc
@@ -2670,9 +2670,21 @@ All parameter, weight, gradient are variables in Paddle.
   m.def("use_layout_autotune",
         [] { return egr::Controller::Instance().UseLayoutAutoTune(); });
   // Add the api for nan op debug
+  m.def("set_nan_inf_stack_limit",
+        &paddle::framework::details::SetNanInfStackLimit);
+
+  // Add the api for nan op debug
   m.def("set_nan_inf_debug_path",
         &paddle::framework::details::SetNanInfDebugPath);
 
+  // Add check op lost
+  m.def("set_checked_op_list",
+        [](const std::string &op_list) { egr::SetCheckOpList(op_list); });
+
+  // Add skipped op list
+  m.def("set_skipped_op_list",
+        [](const std::string &op_list) { egr::SetSkipOpList(op_list); });
+
   m.def("check_numerics",
         [](const std::string &op_name, const paddle::Tensor &tensor) {
           VLOG(4) << "Check tensor whether has nan or inf.";
diff --git a/python/paddle/amp/debugging.py b/python/paddle/amp/debugging.py
index d103622198457..69ee21fe4c322 100644
--- a/python/paddle/amp/debugging.py
+++ b/python/paddle/amp/debugging.py
@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import contextlib
-import os
 import random
 from enum import Enum
 
@@ -24,82 +23,119 @@
 from paddle.fluid.framework import dygraph_only
 
 __all__ = [
+    "DebugMode",
+    "TensorCheckerConfig",
     "enable_operator_stats_collection",
     "disable_operator_stats_collection",
     "collect_operator_stats",
+    "enable_tensor_checker",
+    "disable_tensor_checker",
 ]
 
 
 class DebugMode(Enum):
+    """
+    The DebugMode is a feature that helps to present the state of the TensorCheckerConfig. Each DebugMode has a specific meaning, which is explained below:
+
+    - DebugMode.CHECK_NAN_INF_AND_ABORT: This mode prints or saves information about Tensors that contain NaN/Inf and interrupts the program.
+
+    - DebugMode.CHECK_NAN_INF: This mode prints or saves critical information about Tensors that contain NaN/Inf but allows the program to continue running.
+
+    - DebugMode.CHECK_ALL_FOR_OVERFLOW: This mode checks the output of the FP32 operator and prints or saves information about key Tensors that exceed the FP16 representation range, such as overflow or underflow.
+
+    - DebugMode.CHECK_ALL: This mode prints or saves output Tensor key information for all operators.
+
+    """
+
     CHECK_NAN_INF_AND_ABORT = 0
     CHECK_NAN_INF = 1
     CHECK_ALL_FOR_OVERFLOW = 2
     CHECK_ALL = 3
-    CHECK_ALL_AND_ABORT = 4
-    DUMP_ALL = 5
+    # CHECK_ALL_AND_ABORT = 4
+    # DUMP_ALL = 5
+
+
+def set_checked_op_list(checked_op_list):
+    # check checked_op_list
+    if checked_op_list is not None:
+        if isinstance(checked_op_list, (list, tuple)):
+            check_op_list = ",".join(value for value in checked_op_list)
+            paddle.fluid.core.set_checked_op_list(check_op_list)
+        else:
+            raise ValueError("checked_op_list must be list or tuple")
+
+
+def set_skipped_op_list(skipped_op_list):
+    # check skipped_op_list
+    if skipped_op_list is not None:
+        if isinstance(skipped_op_list, (list, tuple)):
+            skip_op_list = ",".join(value for value in skipped_op_list)
+            paddle.fluid.core.set_skipped_op_list(skip_op_list)
+        else:
+            raise ValueError("skipped_op_list must be list or tuple")
 
 
 class TensorCheckerConfig:
     """
-    Collect the config for checking nan and inf in module or op tensor.
+    The purpose of this class is to collect the configuration for checking NaN and Inf values in the tensors of a module or operator. It takes the following arguments:
 
     Args:
-    * enable: Whether to enable Tensor's value detection function. The default value is False, which means that these tools will never be used.
+        enable(bool): Indicating whether to enable the detection of NaN and Inf values in tensors. The default value is False, which means that these tools will not be used.
 
-    * debug_mode: Debug mode,There are 6 kinds of debug mode.
-        CHECK_NAN_INF_AND_ABORT(default): Print or save Tensor key information with NaN/Inf and interrupt the program
-        CHECK_NAN_INF: Print or save Tensor critical information with NaN/Inf, but continue to run
-        CHECK_ALL_AND_ABORT: Print or save the output Tensor key information of all operators, and interrupt the program if NaN/Inf occurs
-        CHECK_ALL_FOR_OVERFLOW: Check the output of the FP32 operator, print or save key Tensor information that exceeds the FP16 representation range (overflow, underflow)
-        CHECK_ALL: Print or save output Tensor key information for all operators
-        DUMP_ALL: Saves all Tensor data. This mode does not print on the terminal
+        debug_mode(DebugMode, optional): A parameter that determines the type of debugging to be used. Default is DebugMode.CHECK_NAN_INF_AND_ABORT.
 
-    * dump_dir: The collection data storage path. If it is None, it will be directly printed to the terminal
+        output_dir(string, optional): The path to store collected data. If this parameter is set to None, the data will be printed to the terminal. Default is None.
 
-    * checked_op_list: A list of operators you want to check
+        checked_op_list(list|tuple, optional): Specifies a list of operators that need to be checked during program execution, for example, checked_op_list=['elementwise_add', 'conv2d'], indicating that the output results of elementwise_add and conv2d should be checked for nan/inf during program execution. Default is None.
 
-    * skipped_op_list: A list of operators to skip checking
+        skipped_op_list(list|tuple, optional): Specifies a list of operators that do not need to be checked during program execution, for example, skipped_op_list=['elementwise_add', 'conv2d'], indicating that the output results of elementwise_add and conv2d should not be checked for nan/inf during program execution. None is None.
 
-    * debug_step: The iteration scope of debugging
+        debug_step(list|tuple, optional): A list or tuple used primarily for nan/inf checking during model training. For example, debug_step=[1,5] indicates that nan/inf checking should only be performed on model training iterations 1 to 5. Default is None.
 
-    * stack_height_limit: The maximum depth of the call stack, and supports printing the call stack at the error location. The specific scheme needs to be investigated
-
-    * enable_traceback_filtering: Whether to filter the traceback. The main purpose is to filter out the internal code call stack of the framework and only display the user code call stack
+        stack_height_limit(int, optional): An integer value specifying the maximum depth of the call stack. This feature supports printing the call stack at the error location. Currently, only enabling or disabling call stack printing is supported. If you want to print the corresponding C++ call stack when NaN is detected in GPU Kernel, set stack_height_limit to 1, otherwise set it to 0. Default is 1.
 
     Examples:
-       .. code-block:: python
-          import paddle
 
-          checker_config = paddle.amp.debugging.TensorCheckerConfig(enable=True, debug_mode=DebugMode.CHECK_NAN_INF_AND_ABORT)
-          paddle.amp.debugging.enable_tensor_checker(checker_config)
+        ..  code-block:: python
+
+            import paddle
+
+            checker_config = paddle.amp.debugging.TensorCheckerConfig(enable=True, debug_mode=paddle.amp.debugging.DebugMode.CHECK_NAN_INF)
+            paddle.amp.debugging.enable_tensor_checker(checker_config)
 
-          x = paddle.to_tensor([1, 0, 3], place=paddle.CPUPlace(), dtype='float32', stop_gradient=False)
-          y = paddle.to_tensor([0.2, 0, 0.5], place=paddle.CPUPlace(), dtype='float32')
-          res = paddle.pow(x, y)
+            x = paddle.to_tensor([1, 0, 3], place=paddle.CPUPlace(), dtype='float32', stop_gradient=False)
+            y = paddle.to_tensor([0.2, 0, 0.5], place=paddle.CPUPlace(), dtype='float32')
+            res = paddle.pow(x, y)
+            paddle.autograd.backward(res, retain_graph=True)
+            paddle.amp.debugging.disable_tensor_checker()
 
-          paddle.autograd.backward(res, retain_graph=True)
-          paddle.amp.debugging.disable_tensor_checker()
+            #[PRECISION] [ERROR] in [device=cpu, op=elementwise_pow_grad, tensor=, dtype=fp32], numel=3, num_nan=1, num_inf=0, num_zero=0, max=2.886751e-01, min=2.000000e-01, mean=-nan
+
+            # when DebugMode.CHECK_NAN_INF_AND_ABORT and stack_height_limit = 1
+            #Traceback (most recent call last):
+            #    res = paddle.pow(x, y)
+            #  File "/usr/local/lib/python3.8/dist-packages/paddle/tensor/math.py", line 447, in pow
+            #    return _C_ops.elementwise_pow(x, y)
 
     """
 
     # For module debugging
-    Current_step_id = 0
+    current_step_id = 0
 
     def __init__(
         self,
         enable,
         debug_mode=DebugMode.CHECK_NAN_INF_AND_ABORT,
-        dump_dir=None,
+        output_dir=None,
         checked_op_list=None,
         skipped_op_list=None,
         debug_step=None,
-        stack_height_limit=3,
-        enable_traceback_filtering=False,
+        stack_height_limit=1,
     ):
 
         self.enable = enable
         self.debug_mode = debug_mode
-        self.dump_dir = dump_dir
+        self.output_dir = output_dir
 
         self.checked_op_list = checked_op_list
         self.skipped_op_list = skipped_op_list
@@ -107,8 +143,6 @@ def __init__(
         self.debug_step = debug_step
         self.stack_height_limit = stack_height_limit
 
-        self.enable_traceback_filtering = enable_traceback_filtering
-
         self.start_step = None
         self.end_step = None
 
@@ -146,60 +180,43 @@ def __init__(
                 DebugMode.__members__,
             )
 
-        # check checked_op_list
-        if self.checked_op_list is not None:
-            if isinstance(self.checked_op_list, (list, tuple)):
-                check_op_list = ",".join(
-                    value for value in self.checked_op_list
-                )
-                os.environ["Paddle_check_nan_inf_op_list"] = str(check_op_list)
-            else:
-                raise ValueError("checked_op_list must be list or tuple")
+        set_checked_op_list(self.checked_op_list)
 
-        # check skipped_op_list
-        if self.skipped_op_list is not None:
-            if isinstance(self.skipped_op_list, (list, tuple)):
-                skipped_op_list = ",".join(
-                    value for value in self.skipped_op_list
-                )
-                os.environ["Paddle_skip_nan_inf_op_list"] = str(skipped_op_list)
-            else:
-                raise ValueError("skipped_op_list must be list or tuple")
+        set_skipped_op_list(self.skipped_op_list)
 
         if self.enable:
             self._set_seed(self.enable)
 
-    def keep_random(self, seed, flag):
+    def _set_seed(self, flag):
+        if self.initial_seed != self.seed:
+            self.seed = self.initial_seed
+
+        if self.seed > np.iinfo(np.uint32).max or self.seed < 0:
+            print("[Warnning: Seed must be between 0 and 2**32 - 1")
+            self.seed = 123
+
         # get random seed
-        self.seed = seed
         paddle.seed(self.seed)
         np.random.seed(self.seed)
         random.seed(self.seed)
 
+        # info
+        print("AMP Debugging TensorCheckerConfig: seed ", self.seed)
+
         # set cudnn and cpu
         if core.is_compiled_with_cuda():
             paddle.set_flags({"FLAGS_cudnn_deterministic": flag})
-        paddle.set_flags({"FLAGS_cpu_deterministic": flag})
+            print(
+                "AMP Debugging TensorCheckerConfig: FLAGS_cudnn_deterministic is ",
+                flag,
+            )
 
-        # info
-        print("AMP Debugging TensorCheckerConfig: seed ", self.seed)
-        print(
-            "AMP Debugging TensorCheckerConfig: FLAGS_cudnn_deterministic is ",
-            flag,
-        )
+        paddle.set_flags({"FLAGS_cpu_deterministic": flag})
         print(
             "AMP Debugging TensorCheckerConfig: FLAGS_cpu_deterministic is ",
             flag,
         )
 
-    def _set_seed(self, enable):
-        if self.initial_seed != self.seed:
-            self.seed = self.initial_seed
-        if self.seed > 4294967295 or self.seed < 0:
-            print("[Warnning: Seed must be between 0 and 2**32 - 1")
-            self.seed = 123
-            self.keep_random(self.seed, True)
-
     def _set_env(self, check_flag):
         paddle.set_flags({"FLAGS_check_nan_inf": check_flag})
         if check_flag:
@@ -209,35 +226,35 @@ def _set_env(self, check_flag):
             )
 
             # set output_dir
-            if self.dump_dir is not None:
-                paddle.fluid.core.set_nan_inf_debug_path(self.dump_dir)
+            if self.output_dir is not None:
+                paddle.fluid.core.set_nan_inf_debug_path(self.output_dir)
 
             # set stack_height_limit
             if isinstance(self.stack_height_limit, (int)):
-                paddle.set_flags(
-                    {"FLAGS_call_stack_level": self.stack_height_limit}
+                paddle.fluid.core.set_nan_inf_stack_limit(
+                    self.stack_height_limit
                 )
             else:
                 raise ValueError("stack_height_limit must be int")
 
-    def check(self):
+    def update_and_check_step_id(self):
         if self.enable:
             if self.start_step is not None and self.end_step is not None:
                 if (
-                    self.start_step > TensorCheckerConfig.Current_step_id
-                    or TensorCheckerConfig.Current_step_id >= self.end_step
+                    self.start_step > TensorCheckerConfig.current_step_id
+                    or TensorCheckerConfig.current_step_id >= self.end_step
                 ):
                     return False
                 else:
-                    TensorCheckerConfig.Current_step_id += 1
+                    TensorCheckerConfig.current_step_id += 1
             return True
         return False
 
-    def run(self):
+    def start_check_nan_inf(self):
         if self.enable:
             self._set_env(self.enable)
 
-    def end(self):
+    def stop_check_nan_inf(self):
         self._set_env(False)
 
 
@@ -302,26 +319,26 @@ def enable_operator_stats_collection():
 
     Examples:
 
-     .. code-block:: python
+        ..  code-block:: python
 
-        import paddle
+            import paddle
 
-        conv = paddle.nn.Conv2D(3, 2, 3)
-        x = paddle.rand([10, 3, 32, 32])
+            conv = paddle.nn.Conv2D(3, 2, 3)
+            x = paddle.rand([10, 3, 32, 32])
 
-        paddle.amp.debugging.enable_operator_stats_collection()
-        # AMP list including conv2d, elementwise_add, reshape2, cast (transfer_dtype)
-        with paddle.amp.auto_cast(enable=True, level='O2'):
-            out = conv(x)
-        # Print to the standard output.
-        paddle.amp.debugging.disable_operator_stats_collection()
-        # <------------------------------------------------------- op list -------------------------------------------------------->
-        # <--------------- Op Name ---------------- | -- FP16 Calls --- | -- BF16 Calls --- | --- FP32 Calls--- | -- Other Calls -->
-        #   conv2d                                  |  1                |  0                |  0                |  0
-        #   elementwise_add                         |  1                |  0                |  0                |  0
-        #   reshape2                                |  1                |  0                |  0                |  0
-        #   transfer_dtype                          |  0                |  0                |  3                |  0
-        # <----------------------------------------------------- op count: 4 ------------------------------------------------------>
+            paddle.amp.debugging.enable_operator_stats_collection()
+            # AMP list including conv2d, elementwise_add, reshape2, cast (transfer_dtype)
+            with paddle.amp.auto_cast(enable=True, level='O2'):
+                out = conv(x)
+            # Print to the standard output.
+            paddle.amp.debugging.disable_operator_stats_collection()
+            # <------------------------------------------------------- op list -------------------------------------------------------->
+            # <--------------- Op Name ---------------- | -- FP16 Calls --- | -- BF16 Calls --- | --- FP32 Calls--- | -- Other Calls -->
+            #   conv2d                                  |  1                |  0                |  0                |  0
+            #   elementwise_add                         |  1                |  0                |  0                |  0
+            #   reshape2                                |  1                |  0                |  0                |  0
+            #   transfer_dtype                          |  0                |  0                |  3                |  0
+            # <----------------------------------------------------- op count: 4 ------------------------------------------------------>
 
     """
     # Clear the previous stats.
@@ -340,26 +357,26 @@ def disable_operator_stats_collection():
 
     Examples:
 
-     .. code-block:: python
+        ..  code-block:: python
 
-        import paddle
+            import paddle
 
-        conv = paddle.nn.Conv2D(3, 2, 3)
-        x = paddle.rand([10, 3, 32, 32])
+            conv = paddle.nn.Conv2D(3, 2, 3)
+            x = paddle.rand([10, 3, 32, 32])
 
-        paddle.amp.debugging.enable_operator_stats_collection()
-        # AMP list including conv2d, elementwise_add, reshape2, cast (transfer_dtype)
-        with paddle.amp.auto_cast(enable=True, level='O2'):
-            out = conv(x)
-        # Print to the standard output.
-        paddle.amp.debugging.disable_operator_stats_collection()
-        # <------------------------------------------------------- op list -------------------------------------------------------->
-        # <--------------- Op Name ---------------- | -- FP16 Calls --- | -- BF16 Calls --- | --- FP32 Calls--- | -- Other Calls -->
-        #   conv2d                                  |  1                |  0                |  0                |  0
-        #   elementwise_add                         |  1                |  0                |  0                |  0
-        #   reshape2                                |  1                |  0                |  0                |  0
-        #   transfer_dtype                          |  0                |  0                |  3                |  0
-        # <----------------------------------------------------- op count: 4 ------------------------------------------------------>
+            paddle.amp.debugging.enable_operator_stats_collection()
+            # AMP list including conv2d, elementwise_add, reshape2, cast (transfer_dtype)
+            with paddle.amp.auto_cast(enable=True, level='O2'):
+                out = conv(x)
+            # Print to the standard output.
+            paddle.amp.debugging.disable_operator_stats_collection()
+            # <------------------------------------------------------- op list -------------------------------------------------------->
+            # <--------------- Op Name ---------------- | -- FP16 Calls --- | -- BF16 Calls --- | --- FP32 Calls--- | -- Other Calls -->
+            #   conv2d                                  |  1                |  0                |  0                |  0
+            #   elementwise_add                         |  1                |  0                |  0                |  0
+            #   reshape2                                |  1                |  0                |  0                |  0
+            #   transfer_dtype                          |  0                |  0                |  3                |  0
+            # <----------------------------------------------------- op count: 4 ------------------------------------------------------>
 
     """
     if not _get_operator_stats_flag():
@@ -381,25 +398,25 @@ def collect_operator_stats():
 
     Examples:
 
-     .. code-block:: python
+        ..  code-block:: python
 
-        import paddle
+            import paddle
 
-        conv = paddle.nn.Conv2D(3, 2, 3)
-        x = paddle.rand([10, 3, 32, 32])
+            conv = paddle.nn.Conv2D(3, 2, 3)
+            x = paddle.rand([10, 3, 32, 32])
 
-        with paddle.amp.debugging.collect_operator_stats():
-            # AMP list including conv2d, elementwise_add, reshape2, cast (transfer_dtype)
-            with paddle.amp.auto_cast(enable=True, level='O2'):
-                out = conv(x)
-        # Print to the standard output.
-        # <------------------------------------------------------- op list -------------------------------------------------------->
-        # <--------------- Op Name ---------------- | -- FP16 Calls --- | -- BF16 Calls --- | --- FP32 Calls--- | -- Other Calls -->
-        #   conv2d                                  |  1                |  0                |  0                |  0
-        #   elementwise_add                         |  1                |  0                |  0                |  0
-        #   reshape2                                |  1                |  0                |  0                |  0
-        #   transfer_dtype                          |  0                |  0                |  3                |  0
-        # <----------------------------------------------------- op count: 4 ------------------------------------------------------>
+            with paddle.amp.debugging.collect_operator_stats():
+                # AMP list including conv2d, elementwise_add, reshape2, cast (transfer_dtype)
+                with paddle.amp.auto_cast(enable=True, level='O2'):
+                    out = conv(x)
+            # Print to the standard output.
+            # <------------------------------------------------------- op list -------------------------------------------------------->
+            # <--------------- Op Name ---------------- | -- FP16 Calls --- | -- BF16 Calls --- | --- FP32 Calls--- | -- Other Calls -->
+            #   conv2d                                  |  1                |  0                |  0                |  0
+            #   elementwise_add                         |  1                |  0                |  0                |  0
+            #   reshape2                                |  1                |  0                |  0                |  0
+            #   transfer_dtype                          |  0                |  0                |  3                |  0
+            # <----------------------------------------------------- op count: 4 ------------------------------------------------------>
 
     """
     enable_operator_stats_collection()
@@ -409,57 +426,74 @@ def collect_operator_stats():
 
 def enable_tensor_checker(checker_config):
     """
-    enable_tensor_checker(checker_config) is enables model level accuracy checking, which is used together with disables_tensor_checker() to achieve model level precision checking through the combination of these two APIs, checking the output Tensors of all operators within the specified range.
+    The enable_tensor_checker(checker_config) function enables model-level accuracy checking and is used in combination with disables_tensor_checker() to achieve model-level precision checking by checking the output Tensors of all operators within the specified range.
 
-    Attention:
-
-    * If disable is called before loss. backward()_tensor_checker(), the gradient operator is not checked;
+    Args:
+        checker_config(TensorCheckerConfig): Checker_config is to collect the configuration for checking NaN and Inf values in the tensors of a module or operator.
 
-    * If disable is called before optimizer.step() tensor_checker(), the optimizer and other weight update related operators will not be checked
+    Note:
+        If disable_tensor_checker() is called before backward(), the gradient operator will not be checked.
+        If disable_tensor_checker() is called before optimizer.step(), the optimizer and other weight update related operators will not be checked.
 
     Examples:
-       .. code-block:: python
-           import paddle
 
-           checker_config = paddle.amp.debugging.TensorCheckerConfig(enable=True, debug_mode=DebugMode.CHECK_NAN_INF_AND_ABORT)
-           paddle.amp.debugging.enable_tensor_checker(checker_config)
+        ..  code-block:: python
+
+            import paddle
+
+            checker_config = paddle.amp.debugging.TensorCheckerConfig(enable=True, debug_mode=paddle.amp.debugging.DebugMode.CHECK_NAN_INF)
+            paddle.amp.debugging.enable_tensor_checker(checker_config)
 
-           x = paddle.to_tensor([1, 0, 3], place=paddle.CPUPlace(), dtype='float32', stop_gradient=False)
-           y = paddle.to_tensor([0.2, 0, 0.5], place=paddle.CPUPlace(), dtype='float32')
-           res = paddle.pow(x, y)
-           paddle.autograd.backward(res, retain_graph=True)
+            x = paddle.to_tensor([1, 0, 3], place=paddle.CPUPlace(), dtype='float32', stop_gradient=False)
+            y = paddle.to_tensor([0.2, 0, 0.5], place=paddle.CPUPlace(), dtype='float32')
+            res = paddle.pow(x, y)
+            paddle.autograd.backward(res, retain_graph=True)
+            paddle.amp.debugging.disable_tensor_checker()
+            #[PRECISION] [ERROR] in [device=cpu, op=elementwise_pow_grad, tensor=, dtype=fp32], numel=3, num_nan=1, num_inf=0, num_zero=0, max=2.886751e-01, min=2.000000e-01, mean=-nan
+
+            # when DebugMode.CHECK_NAN_INF_AND_ABORT and stack_height_limit = 1
+            # Traceback (most recent call last):
+            #   File "tp.py", line 8, in <module>
+            #     res = paddle.pow(x, y)
+            #   File "/usr/local/lib/python3.8/dist-packages/paddle/tensor/math.py", line 447, in pow
+            #     return _C_ops.elementwise_pow(x, y)
 
-           paddle.amp.debugging.disable_tensor_checker()
     """
-    if checker_config.check():
-        checker_config.run()
+    if checker_config.update_and_check_step_id():
+        checker_config.start_check_nan_inf()
     else:
-        checker_config.end()
+        checker_config.stop_check_nan_inf()
 
 
 def disable_tensor_checker():
     """
-    disable_tensor_checker() to disables the accuracy checking, which is used together with enables_tensor_checker(config) to achieve model level precision checking through the combination of these two APIs, checking the output Tensors of all operators within the specified range.
+    disable_tensor_checker() is used to disable accuracy checking, and is used together with enable_tensor_checker(config) to achieve model-level precision checking by checking the output Tensors of all operators within the specified range.
 
-    Attention:
+    Note:
+        If disable_tensor_checker() is called before backward(), the gradient operator will not be checked;
+        If disable_tensor_checker() is called before optimizer.step(), the optimizer and other weight update related operators will not be checked.
 
-    * If disable_tensor_checker() is called before loss.backward(), the gradient operator is not checked;
+    Examples:
 
-    * If disable_tensor_checker() is called before optimizer.step(), the optimizer and other weight update related operators will not be checked
+        ..  code-block:: python
 
-    Examples:
-       .. code-block:: python
-           import paddle
+            import paddle
 
-           checker_config = paddle.amp.debugging.TensorCheckerConfig(enable=True, debug_mode=DebugMode.CHECK_NAN_INF_AND_ABORT)
-           paddle.amp.debugging.enable_tensor_checker(checker_config)
+            checker_config = paddle.amp.debugging.TensorCheckerConfig(enable=True, debug_mode=paddle.amp.debugging.DebugMode.CHECK_NAN_INF)
+            paddle.amp.debugging.enable_tensor_checker(checker_config)
 
-           x = paddle.to_tensor([1, 0, 3], place=paddle.CPUPlace(), dtype='float32', stop_gradient=False)
-           y = paddle.to_tensor([0.2, 0, 0.5], place=paddle.CPUPlace(), dtype='float32')
-           res = paddle.pow(x, y)
-           paddle.autograd.backward(res, retain_graph=True)
+            x = paddle.to_tensor([1, 0, 3], place=paddle.CPUPlace(), dtype='float32', stop_gradient=False)
+            y = paddle.to_tensor([0.2, 0, 0.5], place=paddle.CPUPlace(), dtype='float32')
+            res = paddle.pow(x, y)
+            paddle.autograd.backward(res, retain_graph=True)
+            paddle.amp.debugging.disable_tensor_checker()
+            #[PRECISION] [ERROR] in [device=cpu, op=elementwise_pow_grad, tensor=, dtype=fp32], numel=3, num_nan=1, num_inf=0, num_zero=0, max=2.886751e-01, min=2.000000e-01, mean=-nan
 
-           paddle.amp.debugging.disable_tensor_checker()
+            # when DebugMode.CHECK_NAN_INF_AND_ABORT and stack_height_limit = 1
+            # Traceback (most recent call last):
+            #     res = paddle.pow(x, y)
+            #   File "/usr/local/lib/python3.8/dist-packages/paddle/tensor/math.py", line 447, in pow
+            #     return _C_ops.elementwise_pow(x, y)
 
     """
     paddle.set_flags({"FLAGS_check_nan_inf": 0})
diff --git a/python/paddle/fluid/tests/unittests/test_nan_inf_dir.py b/python/paddle/fluid/tests/unittests/test_nan_inf_dir.py
index 425dc9a7e997e..06695c56f243d 100644
--- a/python/paddle/fluid/tests/unittests/test_nan_inf_dir.py
+++ b/python/paddle/fluid/tests/unittests/test_nan_inf_dir.py
@@ -78,7 +78,14 @@ def get_num_nan_inf(
 
     def test_num_nan_inf(self):
         path = "nan_inf_log_dir"
-        paddle.fluid.core.set_nan_inf_debug_path(path)
+
+        checker_config = paddle.amp.debugging.TensorCheckerConfig(
+            enable=True,
+            debug_mode=paddle.amp.debugging.DebugMode.CHECK_ALL,
+            output_dir=path,
+        )
+
+        paddle.amp.debugging.enable_tensor_checker(checker_config)
 
         def _check_num_nan_inf(use_cuda):
             shape = [32, 32]
@@ -86,145 +93,25 @@ def _check_num_nan_inf(use_cuda):
             num_nan_np, num_inf_np = self.get_reference_num_nan_inf(x_np)
             add_assert = (num_nan_np + num_inf_np) > 0
             num_nan, num_inf = self.get_num_nan_inf(
-                x_np, use_cuda, add_assert, path
+                x_np,
+                use_cuda,
+                add_assert,
+                path,
             )
             if not use_cuda:
                 assert num_nan == num_nan_np and num_inf == num_inf_np
 
-        paddle.set_flags(
-            {"FLAGS_check_nan_inf": 1, "FLAGS_check_nan_inf_level": 3}
-        )
-        _check_num_nan_inf(use_cuda=False)
         if paddle.fluid.core.is_compiled_with_cuda():
             _check_num_nan_inf(use_cuda=True)
+        else:
+            _check_num_nan_inf(use_cuda=False)
+
         x = paddle.to_tensor([2, 3, 4], 'float32')
         y = paddle.to_tensor([1, 5, 2], 'float32')
         z = paddle.add(x, y)
         path = ""
         paddle.fluid.core.set_nan_inf_debug_path(path)
-
-    def test_nan_inf_op(self):
-        import paddle
-
-        num_nan = 0
-        num_inf = 0
-        # check op list
-        x = paddle.to_tensor(
-            [1, 0, 1],
-            place=paddle.CPUPlace(),
-            dtype='float32',
-            stop_gradient=False,
-        )
-        y = paddle.to_tensor(
-            [0.2, -1, 0.5], place=paddle.CPUPlace(), dtype='float32'
-        )
-        try:
-            res = paddle.pow(x, y)
-        except Exception as e:
-            # Cannot catch the log in CUDA kernel.
-            err_str_list = (
-                str(e)
-                .replace("(", " ")
-                .replace(")", " ")
-                .replace(",", " ")
-                .split(" ")
-            )
-            for err_str in err_str_list:
-                if "num_nan" in err_str:
-                    num_nan = int(err_str.split("=")[1])
-                elif "num_inf" in err_str:
-                    num_inf = int(err_str.split("=")[1])
-            print(
-                "[CHECK_NAN_INF_AND_ABORT] num_nan={}, num_inf={}".format(
-                    num_nan, num_inf
-                )
-            )
-        return num_inf
-
-    def test_check_op_list(self):
-        import paddle
-
-        num_nan = 0
-        num_inf = 0
-
-        checker_config = paddle.amp.debugging.TensorCheckerConfig(
-            enable=True,
-            debug_mode=paddle.amp.debugging.DebugMode.CHECK_NAN_INF_AND_ABORT,
-            skipped_op_list=["elementwise_div"],
-        )
-
-        x = paddle.to_tensor(
-            [0, 0, 0],
-            place=paddle.CPUPlace(),
-            dtype='float32',
-            stop_gradient=False,
-        )
-        y = paddle.to_tensor(
-            [0.2, -1, 0.5], place=paddle.CPUPlace(), dtype='float32'
-        )
-        paddle.amp.debugging.enable_tensor_checker(checker_config)
-        try:
-            res = paddle.divide(y, x)
-        except Exception as e:
-            # Cannot catch the log in CUDA kernel.
-            err_str_list = (
-                str(e)
-                .replace("(", " ")
-                .replace(")", " ")
-                .replace(",", " ")
-                .split(" ")
-            )
-            for err_str in err_str_list:
-                if "num_nan" in err_str:
-                    num_nan = int(err_str.split("=")[1])
-                elif "num_inf" in err_str:
-                    num_inf = int(err_str.split("=")[1])
-            print(
-                "[CHECK_NAN_INF_AND_ABORT] num_nan={}, num_inf={}".format(
-                    num_nan, num_inf
-                )
-            )
-        paddle.amp.debugging.enable_tensor_checker(checker_config)
-
-    def test_tensor_checker(self):
-        import paddle
-
-        def _assert_flag(value):
-            flags = ['FLAGS_check_nan_inf', 'FLAGS_check_nan_inf_level']
-            res = paddle.get_flags(flags)
-            assert res["FLAGS_check_nan_inf"] == value
-
-        paddle.set_flags({"FLAGS_check_nan_inf": 0})
-        paddle.seed(102)
-        checker_config = paddle.amp.debugging.TensorCheckerConfig(
-            enable=True,
-            debug_mode=paddle.amp.debugging.DebugMode.CHECK_NAN_INF_AND_ABORT,
-            checked_op_list=["elementwise_pow"],
-            skipped_op_list=["elementwise_add"],
-            debug_step=[0, 3],
-        )
-        # check seed
-        assert checker_config.initial_seed == 102
-        assert checker_config.seed == 102
-        _assert_flag(False)
-        for index in range(5):
-            paddle.amp.debugging.enable_tensor_checker(checker_config)
-            if index <= 2:
-                _assert_flag(True)
-                assert (
-                    index + 1
-                    == paddle.amp.debugging.TensorCheckerConfig.Current_step_id
-                )
-                assert 1 == self.test_nan_inf_op()
-            else:
-                assert (
-                    3
-                    == paddle.amp.debugging.TensorCheckerConfig.Current_step_id
-                )
-                _assert_flag(False)
-                assert 0 == self.test_nan_inf_op()
-            paddle.amp.debugging.disable_tensor_checker()
-            _assert_flag(False)
+        paddle.amp.debugging.disable_tensor_checker()
 
 
 if __name__ == '__main__':
diff --git a/python/paddle/fluid/tests/unittests/test_tensor_checker.py b/python/paddle/fluid/tests/unittests/test_tensor_checker.py
new file mode 100644
index 0000000000000..a5b5e82034fd0
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_tensor_checker.py
@@ -0,0 +1,111 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import paddle
+
+
+class TestTensorChecker(unittest.TestCase):
+    def get_num_inf(self, e):
+        num_nan = 0
+        num_inf = 0
+        # Cannot catch the log in CUDA kernel.
+        err_str_list = (
+            str(e)
+            .replace("(", " ")
+            .replace(")", " ")
+            .replace(",", " ")
+            .split(" ")
+        )
+        for err_str in err_str_list:
+            if "num_nan" in err_str:
+                num_nan = int(err_str.split("=")[1])
+            elif "num_inf" in err_str:
+                num_inf = int(err_str.split("=")[1])
+        print(
+            "[CHECK_NAN_INF_AND_ABORT] num_nan={}, num_inf={}".format(
+                num_nan, num_inf
+            )
+        )
+        return num_nan
+
+    def generate_num_inf(self, place):
+        num_inf = 0
+        num_nan = 0
+        paddle.set_device(place)
+        # check op list
+        x = paddle.to_tensor(
+            [1, 0, 0],
+            dtype='float32',
+            stop_gradient=False,
+        )
+        y = paddle.to_tensor([0, 0, 1], dtype='float32')
+        try:
+            res = paddle.pow(x, y)
+            # test backward
+            paddle.autograd.backward([res])
+            res = paddle.divide(y, x)
+        except Exception as e:
+            num_inf = self.get_num_inf(e)
+        return num_inf
+
+    def test_tensor_checker(self):
+        def _assert_flag(value):
+            flags = ['FLAGS_check_nan_inf', 'FLAGS_check_nan_inf_level']
+            res = paddle.get_flags(flags)
+            assert res["FLAGS_check_nan_inf"] == value
+
+        paddle.set_flags({"FLAGS_check_nan_inf": 0})
+        paddle.seed(102)
+        checker_config = paddle.amp.debugging.TensorCheckerConfig(
+            enable=True,
+            debug_mode=paddle.amp.debugging.DebugMode.CHECK_NAN_INF_AND_ABORT,
+            checked_op_list=["elementwise_pow_grad"],
+            skipped_op_list=["elementwise_div"],
+            debug_step=[0, 3],
+        )
+        places = ['cpu']
+        if paddle.is_compiled_with_cuda():
+            places.append('gpu')
+        # check seed
+        self.assertEqual(checker_config.initial_seed, 102)
+        self.assertEqual(checker_config.seed, 102)
+        _assert_flag(False)
+
+        for place in places:
+            paddle.amp.debugging.TensorCheckerConfig.current_step_id = 0
+            for index in range(5):
+                paddle.amp.debugging.enable_tensor_checker(checker_config)
+                if index <= 2:
+                    _assert_flag(True)
+                    self.assertEqual(
+                        index + 1,
+                        paddle.amp.debugging.TensorCheckerConfig.current_step_id,
+                    )
+                    self.assertEqual(1, self.generate_num_inf(place))
+                else:
+                    self.assertEqual(
+                        3,
+                        paddle.amp.debugging.TensorCheckerConfig.current_step_id,
+                    )
+                    _assert_flag(False)
+                    self.assertEqual(0, self.generate_num_inf(place))
+
+                paddle.amp.debugging.disable_tensor_checker()
+                _assert_flag(False)
+
+
+if __name__ == '__main__':
+    unittest.main()

From f0f5866524a765376770535523b88ce37e47ddf5 Mon Sep 17 00:00:00 2001
From: kangguangli <kangguangli@hotmail.com>
Date: Mon, 24 Apr 2023 17:27:16 +0800
Subject: [PATCH 029/405] [BugFix] wrong match between depend and
 c_allreduce_sum (#53089)

* fix bug: wrong match between depend and c_allreduce_sum

* fix codestyle

* fix bug

* add c_sync_calc_stream back

* fix

* revert

* use flag to control

* fix for code coverage
---
 .../meta_optimizers/raw_program_optimizer.py  | 37 +++++++++++++++++--
 .../test_dist_fleet_raw_program_optimizer.py  | 17 +++++++++
 2 files changed, 51 insertions(+), 3 deletions(-)

diff --git a/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py
index bd1faf1401402..8919ded2e245c 100755
--- a/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py
+++ b/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py
@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 
+import os
+
 from paddle import static
 from paddle.fluid import core
 from paddle.framework import _global_flags
@@ -62,6 +64,9 @@ def _set_basic_info(
             self.calc_comm_same_stream = (
                 user_defined_strategy._calc_comm_same_stream
             )
+            self.sync_before_allreduce = os.environ.get(
+                'FLAGS_sync_before_allreduce', None
+            )
 
     def _can_apply(self):
         if not self.role_maker._is_collective:
@@ -433,17 +438,28 @@ def get_after_idx_of_fuse_group(grad_param_segments):
                     OP_ROLE_KEY: OpRole.Backward,
                 },
             )
+            if not self.calc_comm_same_stream and self.sync_before_allreduce:
+                block._insert_op_without_sync(
+                    after_idx + 1,
+                    type='c_sync_calc_stream',
+                    inputs={'X': fused_var},
+                    outputs={'Out': fused_var},
+                    attrs={OP_ROLE_KEY: OpRole.Backward},
+                )
         idx = 0
-        if not self.calc_comm_same_stream:
+        if not self.calc_comm_same_stream and not self.sync_before_allreduce:
             for i in range(len(grad_param_segments)):
-                while block.ops[idx].type != 'c_allreduce_sum':
+                while (
+                    block.ops[idx].type != 'c_allreduce_sum'
+                    or fused_vars[i].name not in block.ops[idx].input_arg_names
+                ):
                     idx += 1
                 grad_segment, param_segment = grad_param_segments[i]
                 for grad in grad_segment:
                     block._insert_op_without_sync(
                         idx + 1,
                         type='depend',
-                        inputs={'X': grad, 'Dep': fused_var},
+                        inputs={'X': grad, 'Dep': fused_vars[i]},
                         outputs={'Out': grad},
                     )
                     idx += 1
@@ -486,6 +502,21 @@ def get_after_idx_of_fuse_group(grad_param_segments):
                 },
             )
 
+        if self.calc_comm_same_stream or not self.sync_before_allreduce:
+            block._sync_with_cpp()
+            return
+
+        # insert the sync comm op
+        for idx, op in enumerate(block.ops):
+            if is_optimizer_op(op):
+                block._insert_op_without_sync(
+                    idx,
+                    type='c_sync_comm_stream',
+                    inputs={'X': fused_vars},
+                    outputs={'Out': fused_vars},
+                    attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Backward},
+                )
+                break
         block._sync_with_cpp()
 
     def __get_ouputs_name_to_idx(self, first_backward_idx, block):
diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py
index ba826bcf4ff17..c19791a3c33a8 100644
--- a/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py
+++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py
@@ -45,5 +45,22 @@ def test_dist_train(self):
             )
 
 
+class TestFleetMetaOptimizerPrecisionWithSync(TestFleetMetaOptimizerPrecision):
+    def need_envs(self):
+        return {'FLAGS_sync_before_allreduce': '1'}
+
+    def test_dist_train(self):
+        from paddle import fluid
+
+        if fluid.core.is_compiled_with_cuda():
+            self.check_with_place(
+                "dist_fleet_raw_program_optimizer.py",
+                delta=1e-5,
+                check_error_log=True,
+                log_name=flag_name + 'with_sync',
+                need_envs=self.need_envs(),
+            )
+
+
 if __name__ == '__main__':
     unittest.main()

From 21508090058baceb46d96b70418f1308d95eb993 Mon Sep 17 00:00:00 2001
From: Wang Xin <xinwang614@gmail.com>
Date: Mon, 24 Apr 2023 18:09:40 +0800
Subject: [PATCH 030/405] fix 'Werror-maybe-uninitialized' compiler error in
 GCC 11.3 (#53246)

---
 paddle/phi/api/lib/tensor.cc               | 2 +-
 paddle/phi/kernels/funcs/sequence2batch.cc | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/paddle/phi/api/lib/tensor.cc b/paddle/phi/api/lib/tensor.cc
index a63c2cc9cfea4..634c37933cbe9 100644
--- a/paddle/phi/api/lib/tensor.cc
+++ b/paddle/phi/api/lib/tensor.cc
@@ -97,7 +97,7 @@ int64_t Tensor::size() const { return impl_->numel(); }
 const phi::DDim &Tensor::dims() const { return impl_->dims(); }
 
 std::vector<int64_t> Tensor::shape() const {
-  auto dims = impl_->dims();
+  const auto &dims = impl_->dims();
   return phi::vectorize<int64_t>(dims);
 }
 
diff --git a/paddle/phi/kernels/funcs/sequence2batch.cc b/paddle/phi/kernels/funcs/sequence2batch.cc
index 11a687cdeafac..a172f09cf36f6 100644
--- a/paddle/phi/kernels/funcs/sequence2batch.cc
+++ b/paddle/phi/kernels/funcs/sequence2batch.cc
@@ -26,8 +26,8 @@ class CopyMatrixRowsFunctor<phi::CPUContext, T> {
                   phi::DenseTensor* dst,
                   bool is_src_index) {
     size_t* index = index_lod.data();
-    auto src_dims = src.dims();
-    auto dst_dims = dst->dims();
+    const auto& src_dims = vectorize<int>(src.dims());
+    const auto& dst_dims = vectorize<int>(dst->dims());
     PADDLE_ENFORCE_EQ(src_dims.size(),
                       2UL,
                       phi::errors::InvalidArgument(

From 81c89dd6c6b0cf48a529fa8391babd045958bd31 Mon Sep 17 00:00:00 2001
From: JYChen <zoooo0820@qq.com>
Date: Mon, 24 Apr 2023 18:52:27 +0800
Subject: [PATCH 031/405] fix right value is 0d and index is List/Tensor
 (#53225)

---
 python/paddle/fluid/variable_index.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/python/paddle/fluid/variable_index.py b/python/paddle/fluid/variable_index.py
index fbfcf78991271..fe7833f048f04 100644
--- a/python/paddle/fluid/variable_index.py
+++ b/python/paddle/fluid/variable_index.py
@@ -185,7 +185,8 @@ def set_item(self, tensor_origin, value):
 
         for i in range(len(gather_tensor_shape)):
             if not (
-                value_dims_bd[i] == gather_tensor_shape[i]
+                len(value_dims_bd) == 0
+                or value_dims_bd[i] == gather_tensor_shape[i]
                 or value_dims_bd[i] == 1
             ):
                 raise ValueError(

From 9d0befe90d1c16676cad87c84d68af2580357655 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Mon, 24 Apr 2023 18:40:05 +0800
Subject: [PATCH 032/405] Reorganize the forward codes of flash-attention.

---
 .../manual/fluid_manual/dygraph_forward_api.h |   1 +
 .../forwards/fused_gate_attention_fwd_func.cc |  14 +-
 .../operators/fused/fused_gate_attention.h    | 750 +++++++++---------
 .../fused/fused_gate_attention_op.cc          |   6 +-
 .../fused/fused_gate_attention_op.cu          |  38 +-
 5 files changed, 434 insertions(+), 375 deletions(-)

diff --git a/paddle/fluid/eager/api/manual/fluid_manual/dygraph_forward_api.h b/paddle/fluid/eager/api/manual/fluid_manual/dygraph_forward_api.h
index 7fda5aa69b70f..d8a4fee0caf6e 100644
--- a/paddle/fluid/eager/api/manual/fluid_manual/dygraph_forward_api.h
+++ b/paddle/fluid/eager/api/manual/fluid_manual/dygraph_forward_api.h
@@ -27,6 +27,7 @@ std::tuple<paddle::Tensor,
            paddle::Tensor,
            paddle::Tensor,
            paddle::Tensor,
+           paddle::Tensor,
            paddle::Tensor>
 fused_gate_attention_dygraph_function(
     const paddle::Tensor& Query,
diff --git a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gate_attention_fwd_func.cc b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gate_attention_fwd_func.cc
index fd3d32401d922..b0585bc7acd54 100644
--- a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gate_attention_fwd_func.cc
+++ b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gate_attention_fwd_func.cc
@@ -26,6 +26,7 @@ std::tuple<paddle::Tensor,
            paddle::Tensor,
            paddle::Tensor,
            paddle::Tensor,
+           paddle::Tensor,
            paddle::Tensor>
 fused_gate_attention_dygraph_function(
     const paddle::Tensor& Query,
@@ -181,6 +182,9 @@ fused_gate_attention_dygraph_function(
        {"SoftmaxOut",
         {std::make_shared<egr::EagerVariable>(
             egr::Controller::Instance().GenerateUniqueName())}},
+       {"SoftmaxLse",
+        {std::make_shared<egr::EagerVariable>(
+            egr::Controller::Instance().GenerateUniqueName())}},
        {"FMHAOut",
         {std::make_shared<egr::EagerVariable>(
             egr::Controller::Instance().GenerateUniqueName())}},
@@ -256,6 +260,8 @@ fused_gate_attention_dygraph_function(
   egr::EagerUtils::GetOutput(outs["QKVTransposeOut"][0], &QKVTransposeOut);
   paddle::Tensor SoftmaxOut;
   egr::EagerUtils::GetOutput(outs["SoftmaxOut"][0], &SoftmaxOut);
+  paddle::Tensor SoftmaxLse;
+  egr::EagerUtils::GetOutput(outs["SoftmaxLse"][0], &SoftmaxLse);
   paddle::Tensor FMHAOut;
   egr::EagerUtils::GetOutput(outs["FMHAOut"][0], &FMHAOut);
   paddle::Tensor GateOut;
@@ -296,7 +302,7 @@ fused_gate_attention_dygraph_function(
                                         p_autograd_Out);
       // Create GradOpNode
       auto grad_node = std::shared_ptr<fused_gate_attentionGradNodeCompat>(
-          new fused_gate_attentionGradNodeCompat(8, 12));
+          new fused_gate_attentionGradNodeCompat(9, 12));
 
       bool merge_qkv = true;
       if (attrs.count("merge_qkv")) {
@@ -308,6 +314,11 @@ fused_gate_attention_dygraph_function(
         has_gating = PADDLE_GET_CONST(bool, attrs.at("has_gating"));
       }
 
+      // bool use_flash_attn = false;
+      // if (attrs.count("use_flash_attn")) {
+      //   use_flash_attn = PADDLE_GET_CONST(bool, attrs.at("use_flash_attn"));
+      // }
+
       // Set Attributes
       grad_node->SetAttrMap(std::move(attrs));
       grad_node->SetDefaultAttrMap(std::move(default_attrs));
@@ -379,6 +390,7 @@ fused_gate_attention_dygraph_function(
                          ValueTransposeOut,
                          QKVTransposeOut,
                          SoftmaxOut,
+                         SoftmaxLse,
                          FMHAOut,
                          GateOut,
                          Out);
diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 7cacf8f1b1f31..d0310383abe8b 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -99,7 +99,7 @@ struct GateAttentionConfig {
                       const phi::DenseTensor* qkv_weight,
                       bool merge_qkv,
                       bool has_gating,
-                      bool use_flash_attn = false)
+                      bool use_flash_attn)
       : dev_ctx(dev_ctx),
         merge_qkv(merge_qkv),
         has_gating(has_gating),
@@ -158,8 +158,8 @@ struct GateAttentionConfig {
     gate_out_dims = {batch_size, seq_len_m, seq_len_r, num_heads, head_dim};
   }
 
-  bool UseFlashAttn(const bool merge_qkv, const bool run_flash_attn) {
-    if (!run_flash_attn) {
+  bool CanUseFlashAttn() const {
+    if (!std::is_same<T, phi::dtype::bfloat16>::value) {
       return false;
     }
 
@@ -169,7 +169,7 @@ struct GateAttentionConfig {
         case 32:
         case 64:
         case 128:
-          return true;
+          return use_flash_attn;
         default:
           return false;
       }
@@ -436,369 +436,145 @@ class FMHAGateRef {
     T* q_ptr = nullptr;
     T* k_ptr = nullptr;
     T* v_ptr = nullptr;
-    bool is_bf16 =
-        qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
-
-    if (std::is_same<T, phi::dtype::float16>::value) {
-      std::cout << "T is phi::dtype::float16. \n";
-    } else if (std::is_same<T, phi::dtype::bfloat16>::value) {
-      std::cout << "T is phi::dtype::bfloat16. \n";
-    } else if (std::is_same<T, float>::value) {
-      std::cout << "T is float. \n";
-    }
 
-    if (config->UseFlashAttn(merge_qkv_, config->use_flash_attn && is_bf16)) {
+    if (merge_qkv_) {
+      // qkv_transpose_out = transpose(qkv_out)
       PADDLE_ENFORCE_NOT_NULL(
           qkv_transpose_out,
           platform::errors::NotFound("The input qkv_transpose_out can not be "
                                      "nullptr when merge_qkv is true."));
 
-      // 1. Dealing with qkv_out for flash_attn.
       phi::DenseTensor* qkv_out = config->GetQKVOut();
-      ComputeQKVTransposeForwardForFlashAttn(*qkv_out, qkv_transpose_out);
+      ComputeQKVTransposeForward(*qkv_out, qkv_transpose_out);
       config->ClearQKVOut();
 
-      int seq_batch_size = static_cast<int>(config->batch_size) *
-                           static_cast<int>(config->seq_len_m);
-      qkv_transpose_out->Resize(
-          {3,
-           seq_batch_size * static_cast<int>(config->seq_len_r),
-           static_cast<int>(config->num_heads),
-           static_cast<int>(config->head_dim)});
-      DBG_WAIT;
-
       // q_size == k_size
       int64_t q_size = config->GetQuerySize();
       q_ptr = qkv_transpose_out->data<T>();
       k_ptr = q_ptr + q_size;
       v_ptr = k_ptr + q_size;
-
-      // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
-      phi::DenseTensor cu_seq_q, cu_seq_k;
-      int64_t end_size = (seq_batch_size + 1);
-      int64_t seq_size = 0;
-      int64_t start = 0, end = end_size,
-              step = static_cast<int>(config->seq_len_r);
-      phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
-      cu_seq_q.Resize({end_size});
-      cu_seq_k.Resize({end_size});
-      AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_q", &cu_seq_q);
-      AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_k", &cu_seq_k);
-      int64_t block = std::min(seq_size, static_cast<int64_t>(256));
-      int64_t grid = (seq_size + block - 1) / block;
-      FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
-          start, step, end, cu_seq_q.data<int32_t>(), cu_seq_k.data<int32_t>());
-      VLOG(4) << "[Flash_attn] cu_seq_len : start = " << start
-              << ", step = " << step << ", end = " << end;
-      DBG_WAIT;
-
-      // 3. Dealing with mask and bias for flash_attn.
-      phi::DenseTensor temp_mask, temp_bias;
-      auto dims_merge_func = [&](const phi::DenseTensor* src_tensor,
-                                 phi::DenseTensor* dst_tensor,
-                                 const std::string& prefix) {
-        if (src_tensor) {
-          int64_t first_dim = 1;
-          dst_tensor->ShareDataWith(*src_tensor);
-          auto dims_ = src_tensor->dims();
-          for (int i = 0; i < dims_.size() - 3; ++i) {
-            first_dim *= dims_[i];
-          }
-          auto dims_rank = dims_.size();
-          dst_tensor->Resize({first_dim,
-                              dims_[dims_rank - 3],
-                              dims_[dims_rank - 2],
-                              dims_[dims_rank - 1]});
-          GetFlashAttnDimsString(prefix, temp_mask.dims());
-        }
-      };
-      auto& qkv_dims = qkv_transpose_out->dims();
-      dims_merge_func(src_mask, &temp_mask, "mask_dim");
-      dims_merge_func(nonbatched_bias, &temp_bias, "bias_dim");
-      GetFlashAttnDimsString("qkv_transpose_out", qkv_dims);
-      DBG_WAIT;
-      // 4. flash_attn parameter setting.
-
-      int batch_size_ = seq_batch_size;
-      int total_q_ = qkv_dims[1];    // q.dims()[0]
-      int total_k_ = qkv_dims[1];    // q.dims()[0]
-      int num_heads_ = qkv_dims[2];  // q.dims()[1]
-      int head_size_ = qkv_dims[3];  // q.dims()[2]
-      int max_seqlen_q_ = batch_size_;
-      int max_seqlen_k_ = batch_size_;
-      int num_splits = 0;  // 0 for an internal heuristic, which is optimal
-      VLOG(6) << "[Flash_attn Fwd] batch_size : " << batch_size_;
-      VLOG(6) << "[Flash_attn Fwd] total_q   : " << total_q_;
-      VLOG(6) << "[Flash_attn Fwd] total_k   : " << total_k_;
-      VLOG(6) << "[Flash_attn Fwd] num_heads : " << num_heads_;
-      VLOG(6) << "[Flash_attn Fwd] head_size : " << head_size_;
-      VLOG(6) << "[Flash_attn Fwd] max_seqlen_q : " << max_seqlen_q_;
-      VLOG(6) << "[Flash_attn Fwd] max_seqlen_k : " << max_seqlen_k_;
-
-      // 5. construct softmax_lse
-      phi::DenseTensor softmax_lse;
-      int softmax_lse_last_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
-      softmax_lse.Resize({batch_size_, num_heads_, softmax_lse_last_dim});
-      AllocWithDebugInfo<float>(
-          dev_ctx_, "flash_attn: softmax_lse", &softmax_lse);
-
-      DBG_WAIT;
-      // 6. construct random seed
-      auto gen = dev_ctx_.GetGenerator();
-      uint64_t inc = batch_size_ * num_heads_ * 32;
-      auto seed_offset_pair = gen->IncrementOffset(inc);
-      uint64_t seed = seed_offset_pair.first;
-      uint64_t offset = seed_offset_pair.second;
-
-      GetFlashAttnDimsString("softmax_out", softmax_out->dims());
-      GetFlashAttnDimsString("softmax_lse", softmax_lse.dims());
-      GetFlashAttnDimsString("cu_seq_q", cu_seq_q.dims());
-      GetFlashAttnDimsString("cu_seq_k", cu_seq_k.dims());
-      DBG_WAIT;
-
-      // 7. flas_attn part one, get temp worksapce size.
-      float p_dropout = 0.f;
-      float softmax_scale = static_cast<float>(1);
-      cudaStream_t stream = dev_ctx_.stream();
-      uint64_t workspace_size;
-      bool succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
-          static_cast<const void*>(q_ptr),
-          static_cast<const void*>(k_ptr),
-          static_cast<const void*>(v_ptr),
-          nullptr,  // for calculation workspace size
-          cu_seq_q.data<int32_t>(),
-          cu_seq_k.data<int32_t>(),
-          total_q_,
-          total_k_,
-          batch_size_,
-          num_heads_,
-          head_size_,
-          max_seqlen_q_,
-          max_seqlen_k_,
-          p_dropout,
-          softmax_scale,
-          /*zero_tensors=*/false,
-          /*is_causal=*/false,
-          is_bf16,
-          num_splits,
-          softmax_lse.data(),
-          softmax_out->data(),
-          nullptr,
-          &workspace_size,
-          stream,
-          seed,
-          offset,
-          src_mask ? temp_mask.data() : nullptr,
-          nonbatched_bias ? temp_bias.data() : nullptr,
-          temp_mask.dims().Get(),
-          temp_bias.dims().Get());
-      if (!succ) {
-        PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
-      }
-      DBG_WAIT;
-
-      phi::DenseTensor workspace;
-      printf("workspace_size = %d\n", workspace_size);
-      if (workspace_size > 0) {
-        workspace = phi::Empty<float, phi::GPUContext>(
-            dev_ctx_, {int64_t(workspace_size / sizeof(float))});
-        DBGPTR(workspace.data(), "workspace");
-      }
-      DBG_WAIT;
-
-#define DBG_INIT(prefix, x)                                       \
-  do {                                                            \
-    printf("[%s, %d] ", __func__, __LINE__);                      \
-    if (x->initialized()) {                                       \
-      std::cout << prefix << " is initialized." << std::endl;     \
-    } else {                                                      \
-      std::cout << prefix << " is not initialized." << std::endl; \
-    }                                                             \
-  } while (0);
-      DBG_INIT("qkv_transpose_out", qkv_transpose_out);
-      DBG_INIT("softmax_out", softmax_out);
-      DBG_INIT("src_mask", src_mask);
-      DBG_INIT("fmha_out", fmha_out);
-      DBG_INIT("gate_out", gate_out);
-
-      // 8. flas_attn part two, run impl.
-      succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
-          static_cast<const void*>(q_ptr),
-          static_cast<const void*>(k_ptr),
-          static_cast<const void*>(v_ptr),
-          static_cast<void*>(
-              fmha_out->data()),  // for calculation workspace size
-          cu_seq_q.data<int32_t>(),
-          cu_seq_k.data<int32_t>(),
-          total_q_,
-          total_k_,
-          batch_size_,
-          num_heads_,
-          head_size_,
-          max_seqlen_q_,
-          max_seqlen_k_,
-          p_dropout,
-          softmax_scale,
-          /*zero_tensors=*/false,
-          /*is_causal=*/false,
-          is_bf16,
-          num_splits,
-          softmax_lse.data(),
-          softmax_out->data(),
-          (workspace_size > 0) ? static_cast<void*>(workspace.data()) : nullptr,
-          &workspace_size,
-          stream,
-          seed,
-          offset,
-          src_mask ? temp_mask.data() : nullptr,
-          nonbatched_bias ? temp_bias.data() : nullptr,
-          temp_mask.dims().Get(),
-          temp_bias.dims().Get());
-      DBG_WAIT;
-      if (!succ) {
-        PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
-      }
-      DBG_WAIT;
     } else {
-      if (merge_qkv_) {
-        // qkv_transpose_out = transpose(qkv_out)
-        PADDLE_ENFORCE_NOT_NULL(
-            qkv_transpose_out,
-            platform::errors::NotFound("The input qkv_transpose_out can not be "
-                                       "nullptr when merge_qkv is true."));
-
-        phi::DenseTensor* qkv_out = config->GetQKVOut();
-        ComputeQKVTransposeForward(*qkv_out, qkv_transpose_out);
-        config->ClearQKVOut();
-
-        // q_size == k_size
-        int64_t q_size = config->GetQuerySize();
-        q_ptr = qkv_transpose_out->data<T>();
-        k_ptr = q_ptr + q_size;
-        v_ptr = k_ptr + q_size;
-      } else {
-        PADDLE_ENFORCE_NOT_NULL(
-            q_transpose_out,
-            platform::errors::NotFound("The input q_transpose_out can not be "
-                                       "nullptr when merge_qkv is false."));
-        PADDLE_ENFORCE_NOT_NULL(
-            k_transpose_out,
-            platform::errors::NotFound("The input k_transpose_out can not be "
-                                       "nullptr when merge_qkv is false."));
-        PADDLE_ENFORCE_NOT_NULL(
-            v_transpose_out,
-            platform::errors::NotFound("The input v_transpose_out can not be "
-                                       "nullptr when merge_qkv is false."));
-
-        phi::DenseTensor* query_out = config->GetQueryOut();
-        phi::DenseTensor* key_out = config->GetKeyOut();
-        phi::DenseTensor* value_out = config->GetValueOut();
-        ComputeQKVTransposeForward(*query_out,
-                                   *key_out,
-                                   *value_out,
-                                   q_transpose_out,
-                                   k_transpose_out,
-                                   v_transpose_out);
-
-        // q_size != k_size
-        q_ptr = q_transpose_out->data<T>();
-        k_ptr = k_transpose_out->data<T>();
-        v_ptr = v_transpose_out->data<T>();
-      }
-      // qk_out = BatchedGEMM(Q, K^T)
-      // [batch_size, seq_len_m, num_heads, seq_len_r, head_dim] *
-      //                [batch_size, seq_len_m, num_heads, m_size, head_dim]
-      // -> [batch_size, seq_len_m, num_heads, seq_len_r, m_size]
-      phi::DenseTensor* qk_out = config->GetQKOut(softmax_out);
-      T* qk_out_ptr = qk_out->data<T>();
-
-      int64_t gemm_batch_size =
-          config->batch_size * config->seq_len_m * config->num_heads;
-      int64_t gemm_m = config->seq_len_r;
-      int64_t gemm_n = config->m_size;
-      int64_t gemm_k = config->head_dim;
-      // attn = torch.matmul(q, k.transpose(-1, -2))
-      T alpha = static_cast<T>(1.0 / sqrt(config->head_dim));
-      // ComputeBatchedGEMM(merge_qkv_ ?
-      //                         phi::slice_ddim(qkv_transpose_out->dims(),
-      //                                         1,
-      //                                         qkv_transpose_out->dims().size()
-      //                                         - 1) : q_transpose_out->dims(),
-      //                    merge_qkv_ ?
-      //                         phi::slice_ddim(qkv_transpose_out->dims(),
-      //                                         1,
-      //                                         qkv_transpose_out->dims().size()
-      //                                         - 1) : k_transpose_out->dims(),
-      //                    q_ptr,
-      //                    k_ptr,
-      //                    qk_out_ptr,
-      //                    false,
-      //                    true,
-      //                    gemm_m,
-      //                    gemm_n,
-      //                    gemm_k,
-      //                    gemm_batch_size,
-      //                    alpha);
-
-      ComputeBatchedGEMM(q_ptr,
-                         k_ptr,
-                         qk_out_ptr,
-                         false,
-                         true,
-                         gemm_m,
-                         gemm_n,
-                         gemm_k,
-                         gemm_batch_size,
-                         alpha);
-      // attn = softmax_dropout(attn, 0, self.training, mask=mask, bias=bias)
-      // softmax_out = softmax(qk_out + nonbatched_bias + src_mask)
-      ComputeBiasMaskSoftmaxForward(
-          nonbatched_bias, src_mask, qk_out, softmax_out);
-      config->ClearQKOut();
-
-      // qktv_out = BatchedGEMM(softmax_out, V)
-      // [batch_size, seq_len_m, num_heads, seq_len_r, m_size] *
-      // [batch_size, seq_len_m, num_heads, m_size,    head_dim]
-      // -> [batch_size, seq_len_m, num_heads, seq_len_r, head_dim]
-      phi::DenseTensor* qktv_out = config->GetQKTVOut(gate_out);
-      T* qktv_out_ptr = qktv_out->data<T>();
-
-      gemm_m = config->seq_len_r;
-      gemm_n = config->head_dim;
-      gemm_k = config->m_size;
-
-      // o = torch.matmul(attn, v)
-      T* softmax_out_ptr = softmax_out->data<T>();
-      // ComputeBatchedGEMM(softmax_out->dims(),
-      //                    merge_qkv_ ?
-      //                         phi::slice_ddim(qkv_transpose_out->dims(),
-      //                                         1,
-      //                                         qkv_transpose_out->dims().size()
-      //                                         - 1) : v_transpose_out->dims(),
-      //                    softmax_out_ptr,
-      //                    v_ptr,
-      //                    qktv_out_ptr,
-      //                    false,
-      //                    false,
-      //                    gemm_m,
-      //                    gemm_n,
-      //                    gemm_k,
-      //                    gemm_batch_size);
-
-      ComputeBatchedGEMM(softmax_out_ptr,
-                         v_ptr,
-                         qktv_out_ptr,
-                         false,
-                         false,
-                         gemm_m,
-                         gemm_n,
-                         gemm_k,
-                         gemm_batch_size);
+      PADDLE_ENFORCE_NOT_NULL(
+          q_transpose_out,
+          platform::errors::NotFound("The input q_transpose_out can not be "
+                                     "nullptr when merge_qkv is false."));
+      PADDLE_ENFORCE_NOT_NULL(
+          k_transpose_out,
+          platform::errors::NotFound("The input k_transpose_out can not be "
+                                     "nullptr when merge_qkv is false."));
+      PADDLE_ENFORCE_NOT_NULL(
+          v_transpose_out,
+          platform::errors::NotFound("The input v_transpose_out can not be "
+                                     "nullptr when merge_qkv is false."));
 
-      // fmha_out = transpose(qktv_out)
-      // o = o.transpose(-2, -3).contiguous()
-      ComputeQKTVTransposeForward(*qktv_out, fmha_out);
+      phi::DenseTensor* query_out = config->GetQueryOut();
+      phi::DenseTensor* key_out = config->GetKeyOut();
+      phi::DenseTensor* value_out = config->GetValueOut();
+      ComputeQKVTransposeForward(*query_out,
+                                 *key_out,
+                                 *value_out,
+                                 q_transpose_out,
+                                 k_transpose_out,
+                                 v_transpose_out);
+
+      // q_size != k_size
+      q_ptr = q_transpose_out->data<T>();
+      k_ptr = k_transpose_out->data<T>();
+      v_ptr = v_transpose_out->data<T>();
     }
+    // qk_out = BatchedGEMM(Q, K^T)
+    // [batch_size, seq_len_m, num_heads, seq_len_r, head_dim] *
+    //                [batch_size, seq_len_m, num_heads, m_size, head_dim]
+    // -> [batch_size, seq_len_m, num_heads, seq_len_r, m_size]
+    phi::DenseTensor* qk_out = config->GetQKOut(softmax_out);
+    T* qk_out_ptr = qk_out->data<T>();
+
+    int64_t gemm_batch_size =
+        config->batch_size * config->seq_len_m * config->num_heads;
+    int64_t gemm_m = config->seq_len_r;
+    int64_t gemm_n = config->m_size;
+    int64_t gemm_k = config->head_dim;
+    // attn = torch.matmul(q, k.transpose(-1, -2))
+    T alpha = static_cast<T>(1.0 / sqrt(config->head_dim));
+    // ComputeBatchedGEMM(merge_qkv_ ?
+    //                         phi::slice_ddim(qkv_transpose_out->dims(),
+    //                                         1,
+    //                                         qkv_transpose_out->dims().size()
+    //                                         - 1) : q_transpose_out->dims(),
+    //                    merge_qkv_ ?
+    //                         phi::slice_ddim(qkv_transpose_out->dims(),
+    //                                         1,
+    //                                         qkv_transpose_out->dims().size()
+    //                                         - 1) : k_transpose_out->dims(),
+    //                    q_ptr,
+    //                    k_ptr,
+    //                    qk_out_ptr,
+    //                    false,
+    //                    true,
+    //                    gemm_m,
+    //                    gemm_n,
+    //                    gemm_k,
+    //                    gemm_batch_size,
+    //                    alpha);
+
+    ComputeBatchedGEMM(q_ptr,
+                       k_ptr,
+                       qk_out_ptr,
+                       false,
+                       true,
+                       gemm_m,
+                       gemm_n,
+                       gemm_k,
+                       gemm_batch_size,
+                       alpha);
+    // attn = softmax_dropout(attn, 0, self.training, mask=mask, bias=bias)
+    // softmax_out = softmax(qk_out + nonbatched_bias + src_mask)
+    ComputeBiasMaskSoftmaxForward(
+        nonbatched_bias, src_mask, qk_out, softmax_out);
+    config->ClearQKOut();
+
+    // qktv_out = BatchedGEMM(softmax_out, V)
+    // [batch_size, seq_len_m, num_heads, seq_len_r, m_size] *
+    // [batch_size, seq_len_m, num_heads, m_size,    head_dim]
+    // -> [batch_size, seq_len_m, num_heads, seq_len_r, head_dim]
+    phi::DenseTensor* qktv_out = config->GetQKTVOut(gate_out);
+    T* qktv_out_ptr = qktv_out->data<T>();
+
+    gemm_m = config->seq_len_r;
+    gemm_n = config->head_dim;
+    gemm_k = config->m_size;
+
+    // o = torch.matmul(attn, v)
+    T* softmax_out_ptr = softmax_out->data<T>();
+    // ComputeBatchedGEMM(softmax_out->dims(),
+    //                    merge_qkv_ ?
+    //                         phi::slice_ddim(qkv_transpose_out->dims(),
+    //                                         1,
+    //                                         qkv_transpose_out->dims().size()
+    //                                         - 1) : v_transpose_out->dims(),
+    //                    softmax_out_ptr,
+    //                    v_ptr,
+    //                    qktv_out_ptr,
+    //                    false,
+    //                    false,
+    //                    gemm_m,
+    //                    gemm_n,
+    //                    gemm_k,
+    //                    gemm_batch_size);
+
+    ComputeBatchedGEMM(softmax_out_ptr,
+                       v_ptr,
+                       qktv_out_ptr,
+                       false,
+                       false,
+                       gemm_m,
+                       gemm_n,
+                       gemm_k,
+                       gemm_batch_size);
+
+    // fmha_out = transpose(qktv_out)
+    // o = o.transpose(-2, -3).contiguous()
+    ComputeQKTVTransposeForward(*qktv_out, fmha_out);
 
     config->ClearQKTVOut();
     if (config->has_gating) {
@@ -843,7 +619,7 @@ class FMHAGateRef {
       std::cout << "[Grad]: T is float. \n";
     }
 
-    if (config->UseFlashAttn(merge_qkv_, config->use_flash_attn && is_bf16)) {
+    if (config->CanUseFlashAttn()) {
       PADDLE_ENFORCE_NOT_NULL(
           qkv_transpose_out,
           platform::errors::NotFound("The input qkv_transpose_out can not be"
@@ -1275,15 +1051,6 @@ class FMHAGateRef {
         dev_ctx_, qkv_out, perm, qkv_transpose_out);
   }
 
-  // [batch_size, seq_len_m, seq_len_r, 3, num_heads, head_dim] ->
-  //         [3, batch_size, seq_len_m, seq_len_r, num_heads, head_dim]
-  void ComputeQKVTransposeForwardForFlashAttn(
-      const phi::DenseTensor& qkv_out, phi::DenseTensor* qkv_transpose_out) {
-    std::vector<int> perm = {3, 0, 1, 2, 4, 5};
-    phi::funcs::TransposeGPUKernelDriver<T>(
-        dev_ctx_, qkv_out, perm, qkv_transpose_out);
-  }
-
   void ComputeQKVTransposeBackward(
       const phi::DenseTensor& qkv_transpose_out_grad,
       phi::DenseTensor* qkv_out_grad) {
@@ -1449,5 +1216,268 @@ class FMHAGateRef {
   bool merge_qkv_;
 };
 
+template <typename T>
+class FlashAttnWithGating {
+ public:
+  FlashAttnWithGating(const phi::GPUContext& dev_ctx, bool merge_qkv)
+      : dev_ctx_(dev_ctx), merge_qkv_(merge_qkv) {}
+
+  void ComputeForward(const phi::DenseTensor* nonbatched_bias,
+                      const phi::DenseTensor* src_mask,
+                      phi::DenseTensor* q_transpose_out,
+                      phi::DenseTensor* k_transpose_out,
+                      phi::DenseTensor* v_transpose_out,
+                      phi::DenseTensor* qkv_transpose_out,
+                      phi::DenseTensor* softmax_out,
+                      phi::DenseTensor* softmax_lse,
+                      phi::DenseTensor* fmha_out,
+                      phi::DenseTensor* gate_out,
+                      GateAttentionConfig<T>* config) {
+    T* q_ptr = nullptr;
+    T* k_ptr = nullptr;
+    T* v_ptr = nullptr;
+
+    bool is_bf16 =
+        qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
+
+    if (std::is_same<T, phi::dtype::float16>::value) {
+      LOG(INFO) << "T is phi::dtype::float16.";
+    } else if (std::is_same<T, phi::dtype::bfloat16>::value) {
+      LOG(INFO) << "T is phi::dtype::bfloat16.";
+    } else if (std::is_same<T, float>::value) {
+      LOG(INFO) << "T is float.";
+    }
+
+    LOG(INFO) << "Use flash attention";
+
+    PADDLE_ENFORCE_NOT_NULL(
+        qkv_transpose_out,
+        platform::errors::NotFound("The input qkv_transpose_out can not be "
+                                   "nullptr when merge_qkv is true."));
+
+    // 1. Dealing with qkv_out for flash_attn.
+    phi::DenseTensor* qkv_out = config->GetQKVOut();
+    ComputeQKVTransposeForwardForFlashAttn(*qkv_out, qkv_transpose_out);
+    config->ClearQKVOut();
+
+    int seq_batch_size = static_cast<int>(config->batch_size) *
+                         static_cast<int>(config->seq_len_m);
+    qkv_transpose_out->Resize(
+        {3,
+         seq_batch_size * static_cast<int>(config->seq_len_r),
+         static_cast<int>(config->num_heads),
+         static_cast<int>(config->head_dim)});
+    DBG_WAIT;
+
+    // q_size == k_size
+    int64_t q_size = config->GetQuerySize();
+    q_ptr = qkv_transpose_out->data<T>();
+    k_ptr = q_ptr + q_size;
+    v_ptr = k_ptr + q_size;
+
+    // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
+    phi::DenseTensor cu_seq_q, cu_seq_k;
+    int64_t end_size = (seq_batch_size + 1);
+    int64_t seq_size = 0;
+    int64_t start = 0, end = end_size,
+            step = static_cast<int>(config->seq_len_r);
+    phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
+    cu_seq_q.Resize({end_size});
+    cu_seq_k.Resize({end_size});
+    AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_q", &cu_seq_q);
+    AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_k", &cu_seq_k);
+    int64_t block = std::min(seq_size, static_cast<int64_t>(256));
+    int64_t grid = (seq_size + block - 1) / block;
+    FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
+        start, step, end, cu_seq_q.data<int32_t>(), cu_seq_k.data<int32_t>());
+    VLOG(4) << "[Flash_attn] cu_seq_len : start = " << start
+            << ", step = " << step << ", end = " << end;
+    DBG_WAIT;
+
+    // 3. Dealing with mask and bias for flash_attn.
+    phi::DenseTensor temp_mask, temp_bias;
+    auto dims_merge_func = [&](const phi::DenseTensor* src_tensor,
+                               phi::DenseTensor* dst_tensor,
+                               const std::string& prefix) {
+      if (src_tensor) {
+        int64_t first_dim = 1;
+        dst_tensor->ShareDataWith(*src_tensor);
+        auto dims_ = src_tensor->dims();
+        for (int i = 0; i < dims_.size() - 3; ++i) {
+          first_dim *= dims_[i];
+        }
+        auto dims_rank = dims_.size();
+        dst_tensor->Resize({first_dim,
+                            dims_[dims_rank - 3],
+                            dims_[dims_rank - 2],
+                            dims_[dims_rank - 1]});
+        GetFlashAttnDimsString(prefix, temp_mask.dims());
+      }
+    };
+    auto& qkv_dims = qkv_transpose_out->dims();
+    dims_merge_func(src_mask, &temp_mask, "mask_dim");
+    dims_merge_func(nonbatched_bias, &temp_bias, "bias_dim");
+    GetFlashAttnDimsString("qkv_transpose_out", qkv_dims);
+    DBG_WAIT;
+    // 4. flash_attn parameter setting.
+
+    int batch_size_ = seq_batch_size;
+    int total_q_ = qkv_dims[1];    // q.dims()[0]
+    int total_k_ = qkv_dims[1];    // q.dims()[0]
+    int num_heads_ = qkv_dims[2];  // q.dims()[1]
+    int head_size_ = qkv_dims[3];  // q.dims()[2]
+    int max_seqlen_q_ = batch_size_;
+    int max_seqlen_k_ = batch_size_;
+    int num_splits = 0;  // 0 for an internal heuristic, which is optimal
+    VLOG(6) << "[Flash_attn Fwd] batch_size : " << batch_size_;
+    VLOG(6) << "[Flash_attn Fwd] total_q   : " << total_q_;
+    VLOG(6) << "[Flash_attn Fwd] total_k   : " << total_k_;
+    VLOG(6) << "[Flash_attn Fwd] num_heads : " << num_heads_;
+    VLOG(6) << "[Flash_attn Fwd] head_size : " << head_size_;
+    VLOG(6) << "[Flash_attn Fwd] max_seqlen_q : " << max_seqlen_q_;
+    VLOG(6) << "[Flash_attn Fwd] max_seqlen_k : " << max_seqlen_k_;
+
+    // 5. construct softmax_lse
+    int softmax_lse_last_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
+    softmax_lse->Resize({batch_size_, num_heads_, softmax_lse_last_dim});
+    AllocWithDebugInfo<float>(dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
+
+    DBG_WAIT;
+    // 6. construct random seed
+    auto gen = dev_ctx_.GetGenerator();
+    uint64_t inc = batch_size_ * num_heads_ * 32;
+    auto seed_offset_pair = gen->IncrementOffset(inc);
+    uint64_t seed = seed_offset_pair.first;
+    uint64_t offset = seed_offset_pair.second;
+
+    GetFlashAttnDimsString("softmax_out", softmax_out->dims());
+    GetFlashAttnDimsString("softmax_lse", softmax_lse->dims());
+    GetFlashAttnDimsString("cu_seq_q", cu_seq_q.dims());
+    GetFlashAttnDimsString("cu_seq_k", cu_seq_k.dims());
+    DBG_WAIT;
+
+    // 7. flas_attn part one, get temp worksapce size.
+    float p_dropout = 0.f;
+    float softmax_scale = static_cast<float>(1);
+    cudaStream_t stream = dev_ctx_.stream();
+    uint64_t workspace_size;
+    bool succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
+        static_cast<const void*>(q_ptr),
+        static_cast<const void*>(k_ptr),
+        static_cast<const void*>(v_ptr),
+        nullptr,  // for calculation workspace size
+        cu_seq_q.data<int32_t>(),
+        cu_seq_k.data<int32_t>(),
+        total_q_,
+        total_k_,
+        batch_size_,
+        num_heads_,
+        head_size_,
+        max_seqlen_q_,
+        max_seqlen_k_,
+        p_dropout,
+        softmax_scale,
+        /*zero_tensors=*/false,
+        /*is_causal=*/false,
+        is_bf16,
+        num_splits,
+        softmax_lse->data(),
+        softmax_out->data(),
+        nullptr,
+        &workspace_size,
+        stream,
+        seed,
+        offset,
+        src_mask ? temp_mask.data() : nullptr,
+        nonbatched_bias ? temp_bias.data() : nullptr,
+        temp_mask.dims().Get(),
+        temp_bias.dims().Get());
+    if (!succ) {
+      PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
+    }
+    DBG_WAIT;
+
+    phi::DenseTensor workspace;
+    printf("workspace_size = %d\n", workspace_size);
+    if (workspace_size > 0) {
+      workspace = phi::Empty<float, phi::GPUContext>(
+          dev_ctx_, {int64_t(workspace_size / sizeof(float))});
+      DBGPTR(workspace.data(), "workspace");
+    }
+    DBG_WAIT;
+
+#define DBG_INIT(prefix, x)                                       \
+  do {                                                            \
+    printf("[%s, %d] ", __func__, __LINE__);                      \
+    if (x->initialized()) {                                       \
+      std::cout << prefix << " is initialized." << std::endl;     \
+    } else {                                                      \
+      std::cout << prefix << " is not initialized." << std::endl; \
+    }                                                             \
+  } while (0);
+
+    DBG_INIT("qkv_transpose_out", qkv_transpose_out);
+    DBG_INIT("softmax_out", softmax_out);
+    DBG_INIT("src_mask", src_mask);
+    DBG_INIT("fmha_out", fmha_out);
+    DBG_INIT("gate_out", gate_out);
+
+    // 8. flas_attn part two, run impl.
+    succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
+        static_cast<const void*>(q_ptr),
+        static_cast<const void*>(k_ptr),
+        static_cast<const void*>(v_ptr),
+        static_cast<void*>(fmha_out->data()),  // for calculation workspace size
+        cu_seq_q.data<int32_t>(),
+        cu_seq_k.data<int32_t>(),
+        total_q_,
+        total_k_,
+        batch_size_,
+        num_heads_,
+        head_size_,
+        max_seqlen_q_,
+        max_seqlen_k_,
+        p_dropout,
+        softmax_scale,
+        /*zero_tensors=*/false,
+        /*is_causal=*/false,
+        is_bf16,
+        num_splits,
+        softmax_lse->data(),
+        softmax_out->data(),
+        (workspace_size > 0) ? static_cast<void*>(workspace.data()) : nullptr,
+        &workspace_size,
+        stream,
+        seed,
+        offset,
+        src_mask ? temp_mask.data() : nullptr,
+        nonbatched_bias ? temp_bias.data() : nullptr,
+        temp_mask.dims().Get(),
+        temp_bias.dims().Get());
+    DBG_WAIT;
+    if (!succ) {
+      PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
+    }
+    DBG_WAIT;
+
+    if (config->has_gating) {
+      gate_out->Resize(config->gate_out_dims);
+    }
+  }
+
+ private:
+  // [batch_size, seq_len_m, seq_len_r, 3, num_heads, head_dim] ->
+  //         [3, batch_size, seq_len_m, seq_len_r, num_heads, head_dim]
+  void ComputeQKVTransposeForwardForFlashAttn(
+      const phi::DenseTensor& qkv_out, phi::DenseTensor* qkv_transpose_out) {
+    std::vector<int> perm = {3, 0, 1, 2, 4, 5};
+    phi::funcs::TransposeGPUKernelDriver<T>(
+        dev_ctx_, qkv_out, perm, qkv_transpose_out);
+  }
+
+  const phi::GPUContext& dev_ctx_;
+  bool merge_qkv_;
+};
+
 }  // namespace operators
 }  // namespace paddle
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cc b/paddle/fluid/operators/fused/fused_gate_attention_op.cc
index 0c965ac08745a..a3fd178059fab 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cc
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cc
@@ -157,14 +157,14 @@ class FusedGateAttentionOpMaker : public framework::OpProtoAndCheckerMaker {
         .AsIntermediate()
         .AsDispensable();
     AddOutput("SoftmaxOut", "Result in fmha.").AsIntermediate();
+    AddOutput("SoftmaxLse", "Result of the flash attention.")
+        .AsIntermediate()
+        .AsDispensable();
     AddOutput("FMHAOut", "Result in fmha.").AsIntermediate();
     AddOutput("GateOut", "Result of the gating module.")
         .AsIntermediate()
         .AsDispensable();
     AddOutput("Out", "Result after attention.");
-    AddOutput("SoftmaxLse", "Result of the gating module.")
-        .AsIntermediate()
-        .AsDispensable();
     AddAttr<bool>("has_gating",
                   "if true, the attention op uses gate architecure, "
                   "[default true].")
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cu b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
index 938d0c707d08c..afee194827ceb 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cu
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
@@ -428,17 +428,33 @@ class FusedGateAttentionOpKernel : public framework::OpKernel<T> {
     }
 
     // 2. FMHA
-    auto fmha_compute = FMHAGateRef<T>(dev_ctx, merge_qkv);
-    fmha_compute.ComputeForward(nonbatched_bias,
-                                src_mask,
-                                q_transpose_out,
-                                k_transpose_out,
-                                v_transpose_out,
-                                qkv_transpose_out,
-                                softmax_out,
-                                fmha_out,
-                                gate_out,
-                                &config);
+    if (config.CanUseFlashAttn()) {
+      auto *softmax_lse = ctx.Output<phi::DenseTensor>("SoftmaxLse");
+      auto fmha_compute = FlashAttnWithGating<T>(dev_ctx, merge_qkv);
+      fmha_compute.ComputeForward(nonbatched_bias,
+                                  src_mask,
+                                  q_transpose_out,
+                                  k_transpose_out,
+                                  v_transpose_out,
+                                  qkv_transpose_out,
+                                  softmax_out,
+                                  softmax_lse,
+                                  fmha_out,
+                                  gate_out,
+                                  &config);
+    } else {
+      auto fmha_compute = FMHAGateRef<T>(dev_ctx, merge_qkv);
+      fmha_compute.ComputeForward(nonbatched_bias,
+                                  src_mask,
+                                  q_transpose_out,
+                                  k_transpose_out,
+                                  v_transpose_out,
+                                  qkv_transpose_out,
+                                  softmax_out,
+                                  fmha_out,
+                                  gate_out,
+                                  &config);
+    }
 
     // 3. Gating Linear
     if (has_gating) {

From 562d2daf7529a4faf62d3d9dd20ecf8f6965b66f Mon Sep 17 00:00:00 2001
From: risemeup1 <62429225+risemeup1@users.noreply.github.com>
Date: Mon, 24 Apr 2023 19:20:10 +0800
Subject: [PATCH 033/405] Fix gpu ps compile patch error (#53256)

* fix patch error

* fix patch error
---
 cmake/external/eigen.cmake | 2 +-
 cmake/external/gloo.cmake  | 4 ++--
 cmake/external/gtest.cmake | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cmake/external/eigen.cmake b/cmake/external/eigen.cmake
index 12ccf5df27cae..eda9ae8c4a2ac 100644
--- a/cmake/external/eigen.cmake
+++ b/cmake/external/eigen.cmake
@@ -52,7 +52,7 @@ if(CMAKE_COMPILER_IS_GNUCC)
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Complex.h.patch
          complex_header)
     set(EIGEN_PATCH_COMMAND
-        patch -d ${EIGEN_SOURCE_DIR}/Eigen/src/Core/arch/SSE/ <
+        patch -Nd ${EIGEN_SOURCE_DIR}/Eigen/src/Core/arch/SSE/ <
         ${complex_header})
   endif()
 endif()
diff --git a/cmake/external/gloo.cmake b/cmake/external/gloo.cmake
index 0666d48538b74..63212c974e257 100755
--- a/cmake/external/gloo.cmake
+++ b/cmake/external/gloo.cmake
@@ -55,8 +55,8 @@ if(CMAKE_COMPILER_IS_GNUCC)
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/gloo/types.h.patch
          types_header)
     set(GLOO_PATCH_COMMAND
-        patch -d ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch
-        -d ${GLOO_SOURCE_DIR}/gloo/ < ${types_header})
+        patch -Nd ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst} &&
+        patch -Nd ${GLOO_SOURCE_DIR}/gloo/ < ${types_header})
   endif()
 endif()
 include_directories(${GLOO_INCLUDE_DIR})
diff --git a/cmake/external/gtest.cmake b/cmake/external/gtest.cmake
index be6d9cdde61e7..315f6b5b752b2 100644
--- a/cmake/external/gtest.cmake
+++ b/cmake/external/gtest.cmake
@@ -66,7 +66,7 @@ endif()
 if(NOT WIN32 AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 12.0)
   file(TO_NATIVE_PATH
        ${PADDLE_SOURCE_DIR}/patches/gtest/gtest-death-test.cc.patch native_src)
-  set(GTEST_PATCH_COMMAND patch -d ${GTEST_SOURCE_DIR}/googletest/src <
+  set(GTEST_PATCH_COMMAND patch -Nd ${GTEST_SOURCE_DIR}/googletest/src <
                           ${native_src})
 endif()
 if(WIN32)

From ddd7203987fe8bd464b2a1d7652085e9f4cee6e0 Mon Sep 17 00:00:00 2001
From: Leo Chen <chenqiuliang@baidu.com>
Date: Mon, 24 Apr 2023 19:33:16 +0800
Subject: [PATCH 034/405] fix dist_grad kernel (#53239)

---
 paddle/phi/kernels/dist_grad_kernel.cc        | 48 +++++++++++--------
 .../fluid/tests/unittests/test_dist_op.py     |  9 ++++
 2 files changed, 38 insertions(+), 19 deletions(-)

diff --git a/paddle/phi/kernels/dist_grad_kernel.cc b/paddle/phi/kernels/dist_grad_kernel.cc
index ba468ad299e4c..17c24fa905b5c 100644
--- a/paddle/phi/kernels/dist_grad_kernel.cc
+++ b/paddle/phi/kernels/dist_grad_kernel.cc
@@ -52,6 +52,10 @@ void DistGradKernel(const Context& dev_ctx,
                     float p,
                     DenseTensor* x_grad,
                     DenseTensor* y_grad) {
+  if ((!x_grad) && (!y_grad)) {
+    return;
+  }
+
   auto t = Subtract<T, Context>(dev_ctx, x, y);
   DenseTensor x_grad_tmp;
   x_grad_tmp.Resize(t.dims());
@@ -59,26 +63,32 @@ void DistGradKernel(const Context& dev_ctx,
   y_grad_tmp.Resize(t.dims());
   PNormGradKernel<T, Context>(
       dev_ctx, t, out, out_grad, p, -1, 1e-12, false, true, &x_grad_tmp);
-  ScaleKernel<T, Context>(dev_ctx, x_grad_tmp, -1.0, 0.0, false, &y_grad_tmp);
-  // do reduce, the implemetation of cpu SumKernel has bug, it changes
-  // the dims of output iternally, so we Resize x/y_grad twice.
-  auto res_x = GetReduceDims(x_grad_tmp.dims(), x.dims());
-  if (!std::get<0>(res_x).empty()) {
-    x_grad->Resize(phi::make_ddim(std::get<1>(res_x)));
-    SumKernel<T, Context>(
-        dev_ctx, x_grad_tmp, std::get<0>(res_x), x.dtype(), false, x_grad);
-    x_grad->Resize(x.dims());
-  } else {
-    x_grad->ShareBufferWith(x_grad_tmp);
+
+  if (x_grad) {
+    // do reduce, the implemetation of cpu SumKernel has bug, it changes
+    // the dims of output iternally, so we Resize x/y_grad twice.
+    auto res_x = GetReduceDims(x_grad_tmp.dims(), x.dims());
+    if (!std::get<0>(res_x).empty()) {
+      x_grad->Resize(phi::make_ddim(std::get<1>(res_x)));
+      SumKernel<T, Context>(
+          dev_ctx, x_grad_tmp, std::get<0>(res_x), x.dtype(), false, x_grad);
+      x_grad->Resize(x.dims());
+    } else {
+      x_grad->ShareBufferWith(x_grad_tmp);
+    }
   }
-  auto res_y = GetReduceDims(y_grad_tmp.dims(), y.dims());
-  if (!std::get<0>(res_y).empty()) {
-    y_grad->Resize(phi::make_ddim(std::get<1>(res_y)));
-    SumKernel<T, Context>(
-        dev_ctx, y_grad_tmp, std::get<0>(res_y), y.dtype(), false, y_grad);
-    y_grad->Resize(y.dims());
-  } else {
-    y_grad->ShareBufferWith(y_grad_tmp);
+
+  if (y_grad) {
+    ScaleKernel<T, Context>(dev_ctx, x_grad_tmp, -1.0, 0.0, false, &y_grad_tmp);
+    auto res_y = GetReduceDims(y_grad_tmp.dims(), y.dims());
+    if (!std::get<0>(res_y).empty()) {
+      y_grad->Resize(phi::make_ddim(std::get<1>(res_y)));
+      SumKernel<T, Context>(
+          dev_ctx, y_grad_tmp, std::get<0>(res_y), y.dtype(), false, y_grad);
+      y_grad->Resize(y.dims());
+    } else {
+      y_grad->ShareBufferWith(y_grad_tmp);
+    }
   }
 }
 
diff --git a/python/paddle/fluid/tests/unittests/test_dist_op.py b/python/paddle/fluid/tests/unittests/test_dist_op.py
index 4ec55cb7938df..96c0de915cff2 100644
--- a/python/paddle/fluid/tests/unittests/test_dist_op.py
+++ b/python/paddle/fluid/tests/unittests/test_dist_op.py
@@ -192,6 +192,15 @@ def test_api(self):
             )
             np.testing.assert_allclose(dist(x_i, y_i, p), out[0], rtol=1e-05)
 
+    def test_grad_x(self):
+        paddle.disable_static()
+        a = paddle.rand([2, 2, 3, 2])
+        b = paddle.rand([1, 1, 3, 1])
+        a.stop_gradient = False
+        c = paddle.dist(a, b, 2)
+        c.backward()
+        paddle.enable_static()
+
 
 if __name__ == '__main__':
     paddle.enable_static()

From 9f9cd919a669428fe47e4184b0d334a25e6c6e61 Mon Sep 17 00:00:00 2001
From: Zhou Wei <1183042833@qq.com>
Date: Mon, 24 Apr 2023 20:25:29 +0800
Subject: [PATCH 035/405] [Zero-Dim] Support paddle.max output 0D, test=allcase
 (#53242)

---
 .../operators/reduce_ops/reduce_max_op.cc     |  7 +--
 .../composite_backward_api.h                  |  2 +-
 paddle/phi/api/yaml/legacy_ops.yaml           |  2 +-
 paddle/phi/kernels/cpu/add_n_kernel.cc        |  2 +
 .../kernels/funcs/selected_rows_functor.cc    |  1 +
 paddle/phi/kernels/funcs/unsqueeze.h          | 18 +++----
 .../phi/kernels/onednn/reduce_kernel_impl.h   |  6 ++-
 .../fleet/utils/mix_precision_utils.py        |  2 +-
 python/paddle/fluid/dygraph/math_op_patch.py  |  2 +-
 .../unittests/test_learning_rate_scheduler.py |  8 +--
 .../tests/unittests/test_lr_scheduler.py      |  2 +-
 .../tests/unittests/test_zero_dim_tensor.py   | 49 +++++++++++++++++--
 python/paddle/hapi/progressbar.py             |  9 +++-
 python/paddle/nn/clip.py                      |  2 +-
 python/paddle/nn/quant/lsq.py                 |  4 +-
 test/autograd/utils.py                        |  5 +-
 .../seq2seq_dygraph_model.py                  |  2 +-
 test/dygraph_to_static/test_for_enumerate.py  |  2 +-
 test/dygraph_to_static/test_sentiment.py      |  4 +-
 19 files changed, 88 insertions(+), 41 deletions(-)

diff --git a/paddle/fluid/operators/reduce_ops/reduce_max_op.cc b/paddle/fluid/operators/reduce_ops/reduce_max_op.cc
index f53b6d985e23b..1bb84a054698d 100644
--- a/paddle/fluid/operators/reduce_ops/reduce_max_op.cc
+++ b/paddle/fluid/operators/reduce_ops/reduce_max_op.cc
@@ -54,9 +54,10 @@ class ReduceMaxCompositeGradOpMaker : public prim::CompositeGradOpMakerBase {
 }  // namespace operators
 }  // namespace paddle
 
-DECLARE_INFER_SHAPE_FUNCTOR(reduce_max,
-                            ReduceMaxInferShapeFunctor,
-                            PD_INFER_META(phi::OriginReduceInferMetaBase));
+DECLARE_INFER_SHAPE_FUNCTOR(
+    reduce_max,
+    ReduceMaxInferShapeFunctor,
+    PD_INFER_META(phi::ReduceIntArrayAxisInferMetaBase));
 
 REGISTER_OPERATOR(
     reduce_max,
diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
index c722aa4858388..a42c41c1ba229 100644
--- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
+++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
@@ -1335,7 +1335,7 @@ void max_grad(const Tensor& x,
   } else {
     auto axis_ = std::vector<int64_t>();
     if (reduce_all) {
-      for (int64_t i = 1; i < x_dim_size; i++) {
+      for (int64_t i = 0; i < x_dim_size; i++) {
         axis_.push_back(i);
       }
     } else {
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index a3ff474a69ecc..d23a20b18fcc4 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -744,7 +744,7 @@
   args : (Tensor x, IntArray axis={}, bool keepdim=false)
   output : Tensor(out)
   infer_meta :
-    func : OriginReduceInferMeta
+    func : ReduceIntArrayAxisInferMeta
   kernel :
     func : max
   backward : max_grad
diff --git a/paddle/phi/kernels/cpu/add_n_kernel.cc b/paddle/phi/kernels/cpu/add_n_kernel.cc
index 650d33755dca9..e165f1cc7f24b 100644
--- a/paddle/phi/kernels/cpu/add_n_kernel.cc
+++ b/paddle/phi/kernels/cpu/add_n_kernel.cc
@@ -89,6 +89,7 @@ PD_REGISTER_KERNEL(add_n,
                    double,
                    int,
                    phi::dtype::bfloat16,
+                   phi::dtype::float16,
                    int64_t) {}
 
 PD_REGISTER_KERNEL(add_n_array,
@@ -99,4 +100,5 @@ PD_REGISTER_KERNEL(add_n_array,
                    double,
                    int,
                    phi::dtype::bfloat16,
+                   phi::dtype::float16,
                    int64_t) {}
diff --git a/paddle/phi/kernels/funcs/selected_rows_functor.cc b/paddle/phi/kernels/funcs/selected_rows_functor.cc
index 9fc69ddab4a9c..efeb742cfd2ae 100644
--- a/paddle/phi/kernels/funcs/selected_rows_functor.cc
+++ b/paddle/phi/kernels/funcs/selected_rows_functor.cc
@@ -395,6 +395,7 @@ template struct SelectedRowsAddToTensor<phi::CPUContext, float>;
 template struct SelectedRowsAddToTensor<phi::CPUContext, double>;
 template struct SelectedRowsAddToTensor<phi::CPUContext, int>;
 template struct SelectedRowsAddToTensor<phi::CPUContext, int64_t>;
+template struct SelectedRowsAddToTensor<phi::CPUContext, phi::dtype::float16>;
 template struct SelectedRowsAddToTensor<phi::CPUContext, phi::dtype::bfloat16>;
 
 #ifdef PADDLE_WITH_XPU
diff --git a/paddle/phi/kernels/funcs/unsqueeze.h b/paddle/phi/kernels/funcs/unsqueeze.h
index 136fb9b2924bd..b15e781b25117 100644
--- a/paddle/phi/kernels/funcs/unsqueeze.h
+++ b/paddle/phi/kernels/funcs/unsqueeze.h
@@ -105,19 +105,19 @@ inline DDim GetOutputSqueezeShape(const std::vector<int> squeeze_dims,
 
 inline DDim GetUnsqueezeShape(const std::vector<int64_t> unsqz_dims,
                               const DDim& in_dims) {
-  int output_size = in_dims.size() + static_cast<int>(unsqz_dims.size());
-  int cur_output_size = in_dims.size();
-  std::vector<int64_t> output_shape(output_size, 0);
+  int output_rank = in_dims.size() + static_cast<int>(unsqz_dims.size());
+  int cur_output_rank = in_dims.size();
+  std::vector<int64_t> output_shape(output_rank, 0);
 
   // Validity Check: rank range.
   PADDLE_ENFORCE_LE(
-      output_size,
+      output_rank,
       6,
       phi::errors::InvalidArgument("The output "
                                    "tensor's rank should be less than 6."));
 
   for (int axis : unsqz_dims) {
-    int cur = axis < 0 ? axis + cur_output_size + 1 : axis;
+    int cur = axis < 0 ? axis + cur_output_rank + 1 : axis;
     // Vaildity Check: the axis bound
     PADDLE_ENFORCE_GE(
         cur,
@@ -125,12 +125,12 @@ inline DDim GetUnsqueezeShape(const std::vector<int64_t> unsqz_dims,
         phi::errors::InvalidArgument("The insert dimension value should "
                                      "not be less than 0"));
     PADDLE_ENFORCE_LE(cur,
-                      cur_output_size,
+                      cur_output_rank,
                       phi::errors::InvalidArgument(
                           "The insert dimension value shoule not be larger "
                           "than the dimension size of input tensor"));
     // Move old axis, and insert new axis
-    for (int i = cur_output_size; i >= cur; --i) {
+    for (int i = cur_output_rank; i >= cur; --i) {
       if (output_shape[i] == 1) {
         // Move axis
         output_shape[i + 1] = 1;
@@ -139,11 +139,11 @@ inline DDim GetUnsqueezeShape(const std::vector<int64_t> unsqz_dims,
     }
     output_shape[cur] = 1;
     // Add the output size.
-    cur_output_size++;
+    cur_output_rank++;
   }
 
   // Make output shape
-  for (int in_idx = 0, out_idx = 0; out_idx < output_size; ++out_idx) {
+  for (int in_idx = 0, out_idx = 0; out_idx < output_rank; ++out_idx) {
     if (output_shape[out_idx] == 0) {
       output_shape[out_idx] = in_dims[in_idx++];
     }
diff --git a/paddle/phi/kernels/onednn/reduce_kernel_impl.h b/paddle/phi/kernels/onednn/reduce_kernel_impl.h
index d9e01f79bed6a..69f667c36624b 100644
--- a/paddle/phi/kernels/onednn/reduce_kernel_impl.h
+++ b/paddle/phi/kernels/onednn/reduce_kernel_impl.h
@@ -102,8 +102,10 @@ void ReduceKernel(const Context& dev_ctx,
     reduction_p->execute(astream, reduction_args);
     astream.wait();
 
-    out->set_mem_desc(
-        dst_memory_p->get_desc().reshape(vectorize<int64_t>(out->dims())));
+    const auto reshape_dims = out->dims().size() != 0
+                                  ? vectorize<int64_t>(out->dims())
+                                  : std::vector<int64_t>{1};
+    out->set_mem_desc(dst_memory_p->get_desc().reshape(reshape_dims));
   }
 }
 
diff --git a/python/paddle/distributed/fleet/utils/mix_precision_utils.py b/python/paddle/distributed/fleet/utils/mix_precision_utils.py
index 1ee26bce1fb85..9e172d840bdfc 100644
--- a/python/paddle/distributed/fleet/utils/mix_precision_utils.py
+++ b/python/paddle/distributed/fleet/utils/mix_precision_utils.py
@@ -242,7 +242,7 @@ def unscale_method(self, optimizer):
         paddle.distributed.all_reduce(
             is_found_inf, op=paddle.distributed.ReduceOp.MAX, group=None
         )
-        self._found_inf = is_found_inf.numpy()[0]
+        self._found_inf = int(is_found_inf)
 
 
 class MixPrecisionScaler:
diff --git a/python/paddle/fluid/dygraph/math_op_patch.py b/python/paddle/fluid/dygraph/math_op_patch.py
index 47d3451122943..220d849070d18 100644
--- a/python/paddle/fluid/dygraph/math_op_patch.py
+++ b/python/paddle/fluid/dygraph/math_op_patch.py
@@ -179,7 +179,7 @@ def _ndim_(var):
 
     @property
     def _size_(var):
-        return np.prod(var.shape)
+        return int(np.prod(var.shape))
 
     @property
     def _T_(var):
diff --git a/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py b/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py
index e92f44f5e1b41..126d5bfbf753e 100644
--- a/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py
+++ b/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py
@@ -212,7 +212,7 @@ def test_LR_state_dict(self):
             adam_test.set_dict(opt_state)
             self.assertEqual(
                 adam_test._learning_rate.best_loss,
-                adam3._learning_rate.best_loss.numpy()[0],
+                adam3._learning_rate.best_loss,
                 "best_loss is different before and after set_dict",
             )
             self.assertEqual(
@@ -275,7 +275,7 @@ def test_LinearLrWarmup(self):
                 t = lr()
 
                 np.testing.assert_allclose(
-                    t.numpy()[0].item(), right_result[i], rtol=1e-05
+                    t.numpy().item(), right_result[i], rtol=1e-05
                 )
 
             with self.assertRaises(TypeError):
@@ -342,7 +342,7 @@ def test_StepDecay(self):
                 right_result = step_decay(
                     epoch, learning_rate, step_size, decay_rate
                 )
-                fluid_result = scheduler().numpy()[0]
+                fluid_result = scheduler().numpy().item()
                 scheduler.epoch()
                 self.assertAlmostEqual(
                     right_result,
@@ -371,7 +371,7 @@ def test_LambdaDecay(self):
 
             for epoch in range(30):
                 right_result = lambda_decay(epoch, learning_rate, lr_lambda)
-                fluid_result = scheduler().numpy()[0]
+                fluid_result = scheduler().numpy().item()
                 scheduler.epoch()
                 self.assertAlmostEqual(
                     right_result,
diff --git a/python/paddle/fluid/tests/unittests/test_lr_scheduler.py b/python/paddle/fluid/tests/unittests/test_lr_scheduler.py
index 56cf520b1c997..0202d7540d234 100644
--- a/python/paddle/fluid/tests/unittests/test_lr_scheduler.py
+++ b/python/paddle/fluid/tests/unittests/test_lr_scheduler.py
@@ -208,7 +208,7 @@ def _test_dygraph(self, place, kwargs):
         self.assertEqual(
             scheduler.cooldown_counter, scheduler1.cooldown_counter
         )
-        self.assertEqual(scheduler.best.numpy()[0], scheduler1.best)
+        self.assertEqual(scheduler.best, scheduler1.best)
         self.assertEqual(scheduler.num_bad_epochs, scheduler1.num_bad_epochs)
         self.assertEqual(scheduler.last_epoch, scheduler1.last_epoch)
         self.assertEqual(scheduler.last_lr, scheduler1.last_lr)
diff --git a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
index 965bcae57d9db..9c049ddbf435d 100644
--- a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
+++ b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
@@ -219,17 +219,19 @@ def test_dygraph_reduce(self):
                 self.assertEqual(x.grad.shape, [])
                 np.testing.assert_allclose(x.grad.numpy(), np.array(3.0))
 
-            # 2) x is ND
             if api in [
                 paddle.sum,
                 paddle.mean,
                 paddle.nanmean,
                 paddle.nansum,
-                paddle.max,
             ]:
                 return
 
-            x = paddle.rand([3, 5])
+            # 2) x is ND, reduce to 0D
+            if api in [paddle.all, paddle.any]:
+                x = paddle.randint(0, 2, [3, 5]).astype('bool')
+            else:
+                x = paddle.rand([3, 5])
             x.stop_gradient = False
             out = api(x, None)
             out.retain_grads()
@@ -240,6 +242,21 @@ def test_dygraph_reduce(self):
                 self.assertEqual(out.grad.shape, [])
                 self.assertEqual(x.grad.shape, [3, 5])
 
+            # 3) x is 1D, axis=0, reduce to 0D
+            if api in [paddle.all, paddle.any]:
+                x = paddle.randint(0, 2, [5]).astype('bool')
+            else:
+                x = paddle.rand([5])
+            x.stop_gradient = False
+            out = api(x, 0)
+            out.retain_grads()
+            out.backward()
+
+            self.assertEqual(out.shape, [])
+            if x.grad is not None:
+                self.assertEqual(out.grad.shape, [])
+                self.assertEqual(x.grad.shape, [5])
+
         paddle.enable_static()
 
     def test_static_reduce(self):
@@ -284,16 +301,19 @@ def test_static_reduce(self):
                     np.testing.assert_allclose(res[2], np.array(1.0))
                     np.testing.assert_allclose(res[3], np.array(1.0))
 
-                # 2) x is ND
                 if api in [
                     paddle.sum,
                     paddle.mean,
                     paddle.nanmean,
                     paddle.nansum,
-                    paddle.max,
                 ]:
                     return
 
+                # 2) x is ND, reduce to 0D
+                if api in [paddle.all, paddle.any]:
+                    x = paddle.randint(0, 2, [3, 5]).astype('bool')
+                else:
+                    x = paddle.rand([3, 5])
                 x = paddle.rand([3, 5])
                 x.stop_gradient = False
                 out = api(x, None)
@@ -309,6 +329,25 @@ def test_static_reduce(self):
                     self.assertEqual(res[1].shape, ())
                     self.assertEqual(res[2].shape, (3, 5))
 
+                # 3) x is 1D, axis=0, reduce to 0D
+                if api in [paddle.all, paddle.any]:
+                    x = paddle.randint(0, 2, [5]).astype('bool')
+                else:
+                    x = paddle.rand([5])
+                x.stop_gradient = False
+                out = api(x, 0)
+                paddle.static.append_backward(out)
+
+                fetch_list = [out]
+                if block.has_var(x.grad_name):
+                    fetch_list.extend([out.grad_name, x.grad_name])
+
+                res = exe.run(main_prog, fetch_list=fetch_list)
+                self.assertEqual(res[0].shape, ())
+                if len(res) > 1:
+                    self.assertEqual(res[1].shape, ())
+                    self.assertEqual(res[2].shape, (5,))
+
         paddle.disable_static()
 
 
diff --git a/python/paddle/hapi/progressbar.py b/python/paddle/hapi/progressbar.py
index e63bb913334ea..08d6cf2b12181 100644
--- a/python/paddle/hapi/progressbar.py
+++ b/python/paddle/hapi/progressbar.py
@@ -81,8 +81,13 @@ def convert_uint16_to_float(in_list):
 
         for i, (k, val) in enumerate(values):
             if k == "loss":
-                val = val if isinstance(val, (list, np.ndarray)) else [val]
-                if isinstance(val[0], np.uint16):
+                if isinstance(val, list):
+                    scalar_val = val[0]
+                elif isinstance(val, np.ndarray):
+                    scalar_val = val.item()
+                else:
+                    scalar_val = val
+                if isinstance(scalar_val, np.uint16):
                     values[i] = ("loss", list(convert_uint16_to_float(val)))
 
         if current_num:
diff --git a/python/paddle/nn/clip.py b/python/paddle/nn/clip.py
index 58332802efa29..1952ea0514b50 100644
--- a/python/paddle/nn/clip.py
+++ b/python/paddle/nn/clip.py
@@ -700,7 +700,7 @@ def _dygraph_clip(self, params_grads):
         global_norm_var = paddle.add_n(global_norm_var)
         global_norm_var = paddle.sqrt(global_norm_var)
         max_global_norm = paddle.full(
-            shape=[1], dtype=global_norm_var.dtype, fill_value=self.clip_norm
+            shape=[], dtype=global_norm_var.dtype, fill_value=self.clip_norm
         )
 
         need_clip = False
diff --git a/python/paddle/nn/quant/lsq.py b/python/paddle/nn/quant/lsq.py
index 8f225e9265980..4fa8f55266a38 100644
--- a/python/paddle/nn/quant/lsq.py
+++ b/python/paddle/nn/quant/lsq.py
@@ -178,7 +178,7 @@ def __init__(
         s_attr = ParamAttr(
             name=self._scale_name, initializer=Constant(1.0), trainable=True
         )
-        self.s = self.create_parameter(shape=[1], attr=s_attr, dtype='float32')
+        self.s = self.create_parameter(shape=[], attr=s_attr, dtype='float32')
         self.s.stop_gradient = False
 
         if not self.symmetric:
@@ -189,7 +189,7 @@ def __init__(
                 name=self._beta_name, initializer=Constant(0.0), trainable=True
             )
             self.beta = self.create_parameter(
-                shape=[1], attr=beta_attr, dtype='float32'
+                shape=[], attr=beta_attr, dtype='float32'
             )
             self.beta.stop_gradient = False
 
diff --git a/test/autograd/utils.py b/test/autograd/utils.py
index 7fbc456b6b5df..de1db9f2a19f5 100644
--- a/test/autograd/utils.py
+++ b/test/autograd/utils.py
@@ -26,10 +26,7 @@
 # Finite Difference Utils
 ##########################################################
 def _product(t):
-    if isinstance(t, int):
-        return t
-    else:
-        return np.product(t)
+    return int(np.product(t))
 
 
 def _get_item(t, idx):
diff --git a/test/dygraph_to_static/seq2seq_dygraph_model.py b/test/dygraph_to_static/seq2seq_dygraph_model.py
index df9746a57986a..3b9f343ca9fad 100644
--- a/test/dygraph_to_static/seq2seq_dygraph_model.py
+++ b/test/dygraph_to_static/seq2seq_dygraph_model.py
@@ -407,7 +407,7 @@ def beam_search(self, inputs):
         parent_ids = []
 
         for step_idx in range(paddle.to_tensor(self.beam_max_step_num)):
-            if paddle.sum(1 - beam_finished).numpy()[0] == 0:
+            if paddle.sum(1 - beam_finished) == 0:
                 break
             step_input = self._merge_batch_beams(step_input)
             new_dec_hidden, new_dec_cell = [], []
diff --git a/test/dygraph_to_static/test_for_enumerate.py b/test/dygraph_to_static/test_for_enumerate.py
index 58fd9a0b03e8b..1aca0549213c2 100644
--- a/test/dygraph_to_static/test_for_enumerate.py
+++ b/test/dygraph_to_static/test_for_enumerate.py
@@ -28,7 +28,7 @@
 def for_in_range(x):
     z = paddle.tensor.fill_constant([1], 'int32', 0)
     x = fluid.dygraph.to_variable(x)
-    for i in range(x.numpy()[0]):
+    for i in range(x.numpy().item()):
         z = z + i
     return z
 
diff --git a/test/dygraph_to_static/test_sentiment.py b/test/dygraph_to_static/test_sentiment.py
index 5de0d47387931..3ad8ce6333433 100644
--- a/test/dygraph_to_static/test_sentiment.py
+++ b/test/dygraph_to_static/test_sentiment.py
@@ -342,7 +342,7 @@ def train(args, to_static):
 
                 model.train()
                 avg_cost, prediction, acc = model(doc, label)
-                loss_data.append(avg_cost.numpy()[0])
+                loss_data.append(float(avg_cost))
 
                 avg_cost.backward()
                 sgd_optimizer.minimize(avg_cost)
@@ -358,7 +358,7 @@ def train(args, to_static):
                         "step: %d, ave loss: %f, speed: %f steps/s"
                         % (
                             batch_id,
-                            avg_cost.numpy()[0],
+                            float(avg_cost),
                             args.log_step / used_time,
                         )
                     )

From 66f07bcdb5dc118303c1ebf446b272f752019863 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Mon, 24 Apr 2023 21:12:13 +0800
Subject: [PATCH 036/405] Fix forward.

---
 .../operators/fused/fused_gate_attention.h    | 126 +++++++++---------
 .../fused/fused_gate_attention_op.cu          |   7 +-
 2 files changed, 69 insertions(+), 64 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index d0310383abe8b..45dd62a0d4ce7 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -56,6 +56,21 @@ void AllocWithDebugInfo(const phi::GPUContext& dev_ctx,
   VLOG(4) << info << ": " << MemoryDebugString(*t);
 }
 
+inline std::string TensorDebugString(const phi::DenseTensor* t) {
+  if (t && t->initialized()) {
+    std::stringstream ss;
+    ss << "shape=[" << t->dims() << "], ptr=" << t->data();
+    return ss.str();
+  } else {
+    return "nullptr";
+  }
+}
+
+inline std::string WaitWithDebugInfo(const phi::GPUContext& dev_ctx) {
+  dev_ctx.Wait();
+  return "[Synchronize] ";
+}
+
 template <typename T>
 struct TernaryAddFunctor {
   inline HOSTDEVICE T operator()(T a, T b, T c) const { return a + b + c; }
@@ -391,7 +406,7 @@ static void GetFlashAttnDimsString(const std::string& prefix,
                                    const phi::DDim dim_val) {
   // if (VLOG_IS_ON(4)) {
   std::ostringstream out_string;
-  out_string << "FlashAttn - " << prefix << ".dims() is [ ";
+  out_string << "FlashAttn - " << prefix << ".dims() is [";
   for (int i = 0; i < dim_val.size(); ++i) {
     out_string << dim_val[i] << ", ";
   }
@@ -411,12 +426,6 @@ static void GetFlashAttnDimsString(const std::string& prefix,
     std::cout << out_string.str();                                       \
   } while (0);
 
-#define DBG_WAIT                                        \
-  do {                                                  \
-    printf("[%s, %d] Run here.\n", __func__, __LINE__); \
-    dev_ctx_.Wait();                                    \
-  } while (0);
-
 template <typename T>
 class FMHAGateRef {
  public:
@@ -635,7 +644,7 @@ class FMHAGateRef {
 
       int seq_batch_size = static_cast<int>(config->batch_size) *
                            static_cast<int>(config->seq_len_m);
-      DBG_WAIT;
+      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
       // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
       phi::DenseTensor cu_seq_q, cu_seq_k;
@@ -655,7 +664,7 @@ class FMHAGateRef {
           start, step, end, cu_seq_q.data<int32_t>(), cu_seq_k.data<int32_t>());
       VLOG(4) << "[Flash_attn] cu_seq_len : start = " << start
               << ", step = " << step << ", end = " << end;
-      DBG_WAIT;
+      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
       // 3. Dealing with mask and bias for flash_attn.
       phi::DenseTensor temp_mask, temp_bias;
@@ -704,18 +713,18 @@ class FMHAGateRef {
       // softmax_lse->Resize({batch_size_, num_heads_, last_q_dim});
       // AllocWithDebugInfo<float>(
       //     dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
-      DBG_WAIT;
+      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
       phi::DenseTensor softmax_d = phi::Empty<float, phi::GPUContext>(
           dev_ctx_, {batch_size_, num_heads_, last_q_dim});
-      DBG_WAIT;
+      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
       phi::DenseTensor bias_d;
       if (nonbatched_bias) {
         bias_d = phi::Empty<T, phi::GPUContext>(
             dev_ctx_, {batch_size_, num_heads_, max_seqlen_q_, max_seqlen_k_});
       }
-      DBG_WAIT;
+      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
       q_ptr = q_transpose_out->data<T>();
       k_ptr = k_transpose_out->data<T>();
@@ -783,7 +792,7 @@ class FMHAGateRef {
       if (!succ) {
         PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
       }
-      DBG_WAIT;
+      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
       phi::DenseTensor workspace;
       printf("workspace_size = %d\n", workspace_size);
@@ -792,7 +801,7 @@ class FMHAGateRef {
             dev_ctx_, {int64_t(workspace_size / sizeof(float))});
         DBGPTR(workspace.data(), "workspace");
       }
-      DBG_WAIT;
+      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
       succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
           static_cast<const void*>(q_ptr),
@@ -833,7 +842,7 @@ class FMHAGateRef {
       if (!succ) {
         PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
       }
-      DBG_WAIT;
+      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
       if (nonbatched_bias) {
         // compare block reduce
@@ -1228,7 +1237,6 @@ class FlashAttnWithGating {
                       phi::DenseTensor* k_transpose_out,
                       phi::DenseTensor* v_transpose_out,
                       phi::DenseTensor* qkv_transpose_out,
-                      phi::DenseTensor* softmax_out,
                       phi::DenseTensor* softmax_lse,
                       phi::DenseTensor* fmha_out,
                       phi::DenseTensor* gate_out,
@@ -1248,7 +1256,7 @@ class FlashAttnWithGating {
       LOG(INFO) << "T is float.";
     }
 
-    LOG(INFO) << "Use flash attention";
+    LOG(INFO) << "Use flash attention: merge_qkv=" << merge_qkv_;
 
     PADDLE_ENFORCE_NOT_NULL(
         qkv_transpose_out,
@@ -1267,7 +1275,10 @@ class FlashAttnWithGating {
          seq_batch_size * static_cast<int>(config->seq_len_r),
          static_cast<int>(config->num_heads),
          static_cast<int>(config->head_dim)});
-    DBG_WAIT;
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
+              << "1: Reshape qkv_transpose_out: ["
+              << config->qkv_transpose_out_dims << "] -> ["
+              << qkv_transpose_out->dims() << "]";
 
     // q_size == k_size
     int64_t q_size = config->GetQuerySize();
@@ -1277,10 +1288,11 @@ class FlashAttnWithGating {
 
     // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
     phi::DenseTensor cu_seq_q, cu_seq_k;
-    int64_t end_size = (seq_batch_size + 1);
+    int64_t end_size = seq_batch_size + 1;
     int64_t seq_size = 0;
-    int64_t start = 0, end = end_size,
-            step = static_cast<int>(config->seq_len_r);
+    int64_t start = 0;
+    int64_t end = end_size;
+    int64_t step = static_cast<int>(config->seq_len_r);
     phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
     cu_seq_q.Resize({end_size});
     cu_seq_k.Resize({end_size});
@@ -1292,7 +1304,7 @@ class FlashAttnWithGating {
         start, step, end, cu_seq_q.data<int32_t>(), cu_seq_k.data<int32_t>());
     VLOG(4) << "[Flash_attn] cu_seq_len : start = " << start
             << ", step = " << step << ", end = " << end;
-    DBG_WAIT;
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_) << "2: Init cu_seq_q and cu_seq_k";
 
     // 3. Dealing with mask and bias for flash_attn.
     phi::DenseTensor temp_mask, temp_bias;
@@ -1318,9 +1330,10 @@ class FlashAttnWithGating {
     dims_merge_func(src_mask, &temp_mask, "mask_dim");
     dims_merge_func(nonbatched_bias, &temp_bias, "bias_dim");
     GetFlashAttnDimsString("qkv_transpose_out", qkv_dims);
-    DBG_WAIT;
-    // 4. flash_attn parameter setting.
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
+              << "3: Merge dimensions for mask and bias";
 
+    // 4. flash_attn parameter setting.
     int batch_size_ = seq_batch_size;
     int total_q_ = qkv_dims[1];    // q.dims()[0]
     int total_k_ = qkv_dims[1];    // q.dims()[0]
@@ -1329,20 +1342,24 @@ class FlashAttnWithGating {
     int max_seqlen_q_ = batch_size_;
     int max_seqlen_k_ = batch_size_;
     int num_splits = 0;  // 0 for an internal heuristic, which is optimal
-    VLOG(6) << "[Flash_attn Fwd] batch_size : " << batch_size_;
-    VLOG(6) << "[Flash_attn Fwd] total_q   : " << total_q_;
-    VLOG(6) << "[Flash_attn Fwd] total_k   : " << total_k_;
-    VLOG(6) << "[Flash_attn Fwd] num_heads : " << num_heads_;
-    VLOG(6) << "[Flash_attn Fwd] head_size : " << head_size_;
-    VLOG(6) << "[Flash_attn Fwd] max_seqlen_q : " << max_seqlen_q_;
-    VLOG(6) << "[Flash_attn Fwd] max_seqlen_k : " << max_seqlen_k_;
+    LOG(INFO) << "[Flash_attn Fwd] batch_size : " << batch_size_;
+    LOG(INFO) << "[Flash_attn Fwd] total_q   : " << total_q_;
+    LOG(INFO) << "[Flash_attn Fwd] total_k   : " << total_k_;
+    LOG(INFO) << "[Flash_attn Fwd] num_heads : " << num_heads_;
+    LOG(INFO) << "[Flash_attn Fwd] head_size : " << head_size_;
+    LOG(INFO) << "[Flash_attn Fwd] max_seqlen_q : " << max_seqlen_q_;
+    LOG(INFO) << "[Flash_attn Fwd] max_seqlen_k : " << max_seqlen_k_;
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
+              << "4: Init flash-attention parameters";
 
     // 5. construct softmax_lse
     int softmax_lse_last_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
     softmax_lse->Resize({batch_size_, num_heads_, softmax_lse_last_dim});
     AllocWithDebugInfo<float>(dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
+              << "5: Allocate softmax_lse: shape=[" << softmax_lse->dims()
+              << "]";
 
-    DBG_WAIT;
     // 6. construct random seed
     auto gen = dev_ctx_.GetGenerator();
     uint64_t inc = batch_size_ * num_heads_ * 32;
@@ -1350,11 +1367,10 @@ class FlashAttnWithGating {
     uint64_t seed = seed_offset_pair.first;
     uint64_t offset = seed_offset_pair.second;
 
-    GetFlashAttnDimsString("softmax_out", softmax_out->dims());
     GetFlashAttnDimsString("softmax_lse", softmax_lse->dims());
     GetFlashAttnDimsString("cu_seq_q", cu_seq_q.dims());
     GetFlashAttnDimsString("cu_seq_k", cu_seq_k.dims());
-    DBG_WAIT;
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_) << "6: Construct random seed";
 
     // 7. flas_attn part one, get temp worksapce size.
     float p_dropout = 0.f;
@@ -1382,7 +1398,7 @@ class FlashAttnWithGating {
         is_bf16,
         num_splits,
         softmax_lse->data(),
-        softmax_out->data(),
+        nullptr,  // softmax out,
         nullptr,
         &workspace_size,
         stream,
@@ -1390,44 +1406,33 @@ class FlashAttnWithGating {
         offset,
         src_mask ? temp_mask.data() : nullptr,
         nonbatched_bias ? temp_bias.data() : nullptr,
-        temp_mask.dims().Get(),
-        temp_bias.dims().Get());
+        src_mask ? temp_mask.dims().Get() : nullptr,
+        nonbatched_bias ? temp_bias.dims().Get() : nullptr);
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
-    DBG_WAIT;
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
+              << "7: Get workspace_size=" << workspace_size;
 
     phi::DenseTensor workspace;
-    printf("workspace_size = %d\n", workspace_size);
     if (workspace_size > 0) {
       workspace = phi::Empty<float, phi::GPUContext>(
           dev_ctx_, {int64_t(workspace_size / sizeof(float))});
       DBGPTR(workspace.data(), "workspace");
     }
-    DBG_WAIT;
-
-#define DBG_INIT(prefix, x)                                       \
-  do {                                                            \
-    printf("[%s, %d] ", __func__, __LINE__);                      \
-    if (x->initialized()) {                                       \
-      std::cout << prefix << " is initialized." << std::endl;     \
-    } else {                                                      \
-      std::cout << prefix << " is not initialized." << std::endl; \
-    }                                                             \
-  } while (0);
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_) << "Allocate workspace";
 
-    DBG_INIT("qkv_transpose_out", qkv_transpose_out);
-    DBG_INIT("softmax_out", softmax_out);
-    DBG_INIT("src_mask", src_mask);
-    DBG_INIT("fmha_out", fmha_out);
-    DBG_INIT("gate_out", gate_out);
+    LOG(INFO) << "qkv_transpose_out: " << TensorDebugString(qkv_transpose_out);
+    LOG(INFO) << "src_mask: " << TensorDebugString(src_mask);
+    LOG(INFO) << "fmha_out: " << TensorDebugString(fmha_out);
+    LOG(INFO) << "gate_out: " << TensorDebugString(gate_out);
 
     // 8. flas_attn part two, run impl.
     succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
         static_cast<const void*>(q_ptr),
         static_cast<const void*>(k_ptr),
         static_cast<const void*>(v_ptr),
-        static_cast<void*>(fmha_out->data()),  // for calculation workspace size
+        static_cast<void*>(fmha_out->data()),
         cu_seq_q.data<int32_t>(),
         cu_seq_k.data<int32_t>(),
         total_q_,
@@ -1444,7 +1449,7 @@ class FlashAttnWithGating {
         is_bf16,
         num_splits,
         softmax_lse->data(),
-        softmax_out->data(),
+        nullptr,  // softmax out
         (workspace_size > 0) ? static_cast<void*>(workspace.data()) : nullptr,
         &workspace_size,
         stream,
@@ -1452,13 +1457,12 @@ class FlashAttnWithGating {
         offset,
         src_mask ? temp_mask.data() : nullptr,
         nonbatched_bias ? temp_bias.data() : nullptr,
-        temp_mask.dims().Get(),
-        temp_bias.dims().Get());
-    DBG_WAIT;
+        src_mask ? temp_mask.dims().Get() : nullptr,
+        nonbatched_bias ? temp_bias.dims().Get() : nullptr);
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
-    DBG_WAIT;
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_) << "8: Run SUCCESS";
 
     if (config->has_gating) {
       gate_out->Resize(config->gate_out_dims);
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cu b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
index afee194827ceb..3d10047bf23a0 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cu
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
@@ -371,7 +371,6 @@ class FusedGateAttentionOpKernel : public framework::OpKernel<T> {
     auto *v_transpose_out = ctx.Output<phi::DenseTensor>("ValueTransposeOut");
     auto *qkv_transpose_out = ctx.Output<phi::DenseTensor>("QKVTransposeOut");
 
-    auto *softmax_out = ctx.Output<phi::DenseTensor>("SoftmaxOut");
     auto *fmha_out = ctx.Output<phi::DenseTensor>("FMHAOut");
     auto *gate_out = ctx.Output<phi::DenseTensor>("GateOut");
     auto *out = ctx.Output<phi::DenseTensor>("Out");
@@ -382,7 +381,6 @@ class FusedGateAttentionOpKernel : public framework::OpKernel<T> {
 
     bool use_fused_matmul_bias = true;
     auto &dev_ctx = ctx.template device_context<phi::GPUContext>();
-    AllocWithDebugInfo<T>(dev_ctx, "softmax_out", softmax_out);
     AllocWithDebugInfo<T>(dev_ctx, "fmha_out", fmha_out);
     if (has_gating) {
       AllocWithDebugInfo<T>(dev_ctx, "gate_out", gate_out);
@@ -430,6 +428,7 @@ class FusedGateAttentionOpKernel : public framework::OpKernel<T> {
     // 2. FMHA
     if (config.CanUseFlashAttn()) {
       auto *softmax_lse = ctx.Output<phi::DenseTensor>("SoftmaxLse");
+
       auto fmha_compute = FlashAttnWithGating<T>(dev_ctx, merge_qkv);
       fmha_compute.ComputeForward(nonbatched_bias,
                                   src_mask,
@@ -437,12 +436,14 @@ class FusedGateAttentionOpKernel : public framework::OpKernel<T> {
                                   k_transpose_out,
                                   v_transpose_out,
                                   qkv_transpose_out,
-                                  softmax_out,
                                   softmax_lse,
                                   fmha_out,
                                   gate_out,
                                   &config);
     } else {
+      auto *softmax_out = ctx.Output<phi::DenseTensor>("SoftmaxOut");
+      AllocWithDebugInfo<T>(dev_ctx, "softmax_out", softmax_out);
+
       auto fmha_compute = FMHAGateRef<T>(dev_ctx, merge_qkv);
       fmha_compute.ComputeForward(nonbatched_bias,
                                   src_mask,

From 387d26f3c20533fb5ea102ce5cf5f8862d6a5adb Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Mon, 24 Apr 2023 21:28:10 +0800
Subject: [PATCH 037/405] Remove some noused codes.

---
 .../operators/fused/fused_gate_attention.h    | 42 ++-----------------
 1 file changed, 3 insertions(+), 39 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 45dd62a0d4ce7..0b67382edc4b0 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -503,29 +503,8 @@ class FMHAGateRef {
     int64_t gemm_m = config->seq_len_r;
     int64_t gemm_n = config->m_size;
     int64_t gemm_k = config->head_dim;
-    // attn = torch.matmul(q, k.transpose(-1, -2))
     T alpha = static_cast<T>(1.0 / sqrt(config->head_dim));
-    // ComputeBatchedGEMM(merge_qkv_ ?
-    //                         phi::slice_ddim(qkv_transpose_out->dims(),
-    //                                         1,
-    //                                         qkv_transpose_out->dims().size()
-    //                                         - 1) : q_transpose_out->dims(),
-    //                    merge_qkv_ ?
-    //                         phi::slice_ddim(qkv_transpose_out->dims(),
-    //                                         1,
-    //                                         qkv_transpose_out->dims().size()
-    //                                         - 1) : k_transpose_out->dims(),
-    //                    q_ptr,
-    //                    k_ptr,
-    //                    qk_out_ptr,
-    //                    false,
-    //                    true,
-    //                    gemm_m,
-    //                    gemm_n,
-    //                    gemm_k,
-    //                    gemm_batch_size,
-    //                    alpha);
-
+    // attn = matmul(q, k.transpose(-1, -2))
     ComputeBatchedGEMM(q_ptr,
                        k_ptr,
                        qk_out_ptr,
@@ -536,6 +515,7 @@ class FMHAGateRef {
                        gemm_k,
                        gemm_batch_size,
                        alpha);
+
     // attn = softmax_dropout(attn, 0, self.training, mask=mask, bias=bias)
     // softmax_out = softmax(qk_out + nonbatched_bias + src_mask)
     ComputeBiasMaskSoftmaxForward(
@@ -553,24 +533,8 @@ class FMHAGateRef {
     gemm_n = config->head_dim;
     gemm_k = config->m_size;
 
-    // o = torch.matmul(attn, v)
+    // o = matmul(attn, v)
     T* softmax_out_ptr = softmax_out->data<T>();
-    // ComputeBatchedGEMM(softmax_out->dims(),
-    //                    merge_qkv_ ?
-    //                         phi::slice_ddim(qkv_transpose_out->dims(),
-    //                                         1,
-    //                                         qkv_transpose_out->dims().size()
-    //                                         - 1) : v_transpose_out->dims(),
-    //                    softmax_out_ptr,
-    //                    v_ptr,
-    //                    qktv_out_ptr,
-    //                    false,
-    //                    false,
-    //                    gemm_m,
-    //                    gemm_n,
-    //                    gemm_k,
-    //                    gemm_batch_size);
-
     ComputeBatchedGEMM(softmax_out_ptr,
                        v_ptr,
                        qktv_out_ptr,

From cf4a1c84205dbbe0a0b7c91e71ce9dda05542c67 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Mon, 24 Apr 2023 22:27:36 +0800
Subject: [PATCH 038/405] Change backward.

---
 .../operators/fused/fused_gate_attention.h    | 751 ++++++++++--------
 .../fused/fused_gate_attention_op.cc          |  27 +-
 .../fused/fused_gate_attention_op.cu          |  37 +-
 3 files changed, 440 insertions(+), 375 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 0b67382edc4b0..07b4b5973cac8 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -563,11 +563,7 @@ class FMHAGateRef {
                        const phi::DenseTensor* fmha_out_grad,
                        phi::DenseTensor* src_mask_grad,
                        phi::DenseTensor* nonbatched_bias_grad,
-                       GateAttentionGradConfig<T>* config,
-                       const phi::DenseTensor* fmha_out = nullptr,
-                       const phi::DenseTensor* softmax_lse = nullptr,
-                       const phi::DenseTensor* nonbatched_bias = nullptr,
-                       const phi::DenseTensor* src_mask = nullptr) {
+                       GateAttentionGradConfig<T>* config) {
     const T* q_ptr = nullptr;
     const T* k_ptr = nullptr;
     const T* v_ptr = nullptr;
@@ -581,22 +577,12 @@ class FMHAGateRef {
     phi::DenseTensor v_transpose_out_grad;
     phi::DenseTensor qkv_transpose_out_grad;
 
-    bool is_bf16 =
-        qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
-
-    if (std::is_same<T, phi::dtype::float16>::value) {
-      std::cout << "[Grad]: T is phi::dtype::float16. \n";
-    } else if (std::is_same<T, phi::dtype::bfloat16>::value) {
-      std::cout << "[Grad]: T is phi::dtype::bfloat16. \n";
-    } else if (std::is_same<T, float>::value) {
-      std::cout << "[Grad]: T is float. \n";
-    }
-
-    if (config->CanUseFlashAttn()) {
+    if (merge_qkv_) {
       PADDLE_ENFORCE_NOT_NULL(
           qkv_transpose_out,
-          platform::errors::NotFound("The input qkv_transpose_out can not be"
+          platform::errors::NotFound("The input qkv_transpose_out can not be "
                                      "nullptr when merge_qkv is true."));
+
       int64_t q_size = config->GetQuerySize();
       q_ptr = qkv_transpose_out->data<T>();
       k_ptr = q_ptr + q_size;
@@ -606,93 +592,27 @@ class FMHAGateRef {
       AllocWithDebugInfo<T>(
           dev_ctx_, "qkv_transpose_out_grad", &qkv_transpose_out_grad);
 
-      int seq_batch_size = static_cast<int>(config->batch_size) *
-                           static_cast<int>(config->seq_len_m);
-      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
-
-      // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
-      phi::DenseTensor cu_seq_q, cu_seq_k;
-      int64_t start = 0;
-      int64_t step = static_cast<int>(config->seq_len_r);
-      int64_t end_size = (seq_batch_size + 1);
-      int64_t end = end_size;
-      int64_t seq_size = 0;
-      phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
-      cu_seq_q.Resize({end_size});
-      cu_seq_k.Resize({end_size});
-      AllocWithDebugInfo<int32_t>(dev_ctx_, "Grad: cu_seq_q", &cu_seq_q);
-      AllocWithDebugInfo<int32_t>(dev_ctx_, "Grad: cu_seq_k", &cu_seq_k);
-      int64_t block = std::min(seq_size, static_cast<int64_t>(256));
-      int64_t grid = (seq_size + block - 1) / block;
-      FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
-          start, step, end, cu_seq_q.data<int32_t>(), cu_seq_k.data<int32_t>());
-      VLOG(4) << "[Flash_attn] cu_seq_len : start = " << start
-              << ", step = " << step << ", end = " << end;
-      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
-
-      // 3. Dealing with mask and bias for flash_attn.
-      phi::DenseTensor temp_mask, temp_bias;
-      auto dims_merge_func = [&](const phi::DenseTensor* src_tensor,
-                                 phi::DenseTensor* dst_tensor,
-                                 const std::string& prefix) {
-        if (src_tensor) {
-          int64_t first_dim = 1;
-          dst_tensor->ShareDataWith(*src_tensor);
-          auto dims_ = src_tensor->dims();
-          for (int i = 0; i < dims_.size() - 3; ++i) {
-            first_dim *= dims_[i];
-          }
-          auto dims_rank = dims_.size();
-          dst_tensor->Resize({first_dim,
-                              dims_[dims_rank - 3],
-                              dims_[dims_rank - 2],
-                              dims_[dims_rank - 1]});
-          GetFlashAttnDimsString(prefix, temp_mask.dims());
-        }
-      };
-      dims_merge_func(src_mask, &temp_mask, "[Grad] mask_dim");
-      dims_merge_func(nonbatched_bias, &temp_bias, "[Grad] bias_dim");
-
-      phi::DDim qkv_dims({3,
-                          seq_batch_size * static_cast<int>(config->seq_len_r),
-                          static_cast<int>(config->num_heads),
-                          static_cast<int>(config->head_dim)});
-      int batch_size_ = seq_batch_size;
-      int total_q_ = qkv_dims[1];    // q.dims()[0]
-      int total_k_ = qkv_dims[1];    // q.dims()[0]
-      int num_heads_ = qkv_dims[2];  // q.dims()[1]
-      int head_size_ = qkv_dims[3];  // q.dims()[2]
-      int max_seqlen_q_ = batch_size_;
-      int max_seqlen_k_ = batch_size_;
-      VLOG(6) << "[Flash_attn Grad] batch_size : " << batch_size_;
-      VLOG(6) << "[Flash_attn Grad] total_q   : " << total_q_;
-      VLOG(6) << "[Flash_attn Grad] total_k   : " << total_k_;
-      VLOG(6) << "[Flash_attn Grad] num_heads : " << num_heads_;
-      VLOG(6) << "[Flash_attn Grad] head_size : " << head_size_;
-      VLOG(6) << "[Flash_attn Grad] max_seqlen_q : " << max_seqlen_q_;
-      VLOG(6) << "[Flash_attn Grad] max_seqlen_k : " << max_seqlen_k_;
-
-      // 5. construct softmax_lse
-      int last_q_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
-      // softmax_lse->Resize({batch_size_, num_heads_, last_q_dim});
-      // AllocWithDebugInfo<float>(
-      //     dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
-      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
-
-      phi::DenseTensor softmax_d = phi::Empty<float, phi::GPUContext>(
-          dev_ctx_, {batch_size_, num_heads_, last_q_dim});
-      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
-
-      phi::DenseTensor bias_d;
-      if (nonbatched_bias) {
-        bias_d = phi::Empty<T, phi::GPUContext>(
-            dev_ctx_, {batch_size_, num_heads_, max_seqlen_q_, max_seqlen_k_});
-      }
-      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+      q_grad_ptr = qkv_transpose_out_grad.data<T>();
+      k_grad_ptr = q_grad_ptr + q_size;
+      v_grad_ptr = k_grad_ptr + q_size;
+    } else {
+      PADDLE_ENFORCE_NOT_NULL(
+          q_transpose_out,
+          platform::errors::NotFound("The input q_transpose_out can not be "
+                                     "nullptr when merge_qkv is false."));
+      PADDLE_ENFORCE_NOT_NULL(
+          k_transpose_out,
+          platform::errors::NotFound("The input k_transpose_out can not be "
+                                     "nullptr when merge_qkv is false."));
+      PADDLE_ENFORCE_NOT_NULL(
+          v_transpose_out,
+          platform::errors::NotFound("The input v_transpose_out can not be "
+                                     "nullptr when merge_qkv is false."));
 
       q_ptr = q_transpose_out->data<T>();
       k_ptr = k_transpose_out->data<T>();
       v_ptr = v_transpose_out->data<T>();
+
       q_transpose_out_grad.Resize(config->q_transpose_out_dims);
       k_transpose_out_grad.Resize(config->kv_transpose_out_dims);
       v_transpose_out_grad.Resize(config->kv_transpose_out_dims);
@@ -703,273 +623,99 @@ class FMHAGateRef {
                                      k_transpose_out_grad.numel() * sizeof(T));
       v_grad_ptr = dev_ctx_.Alloc<T>(&v_transpose_out_grad,
                                      v_transpose_out_grad.numel() * sizeof(T));
+    }
 
-      // 6. construct random seed
-      auto gen = dev_ctx_.GetGenerator();
-      uint64_t inc = batch_size_ * num_heads_ * 32;
-      auto seed_offset_pair = gen->IncrementOffset(inc);
-      uint64_t seed = seed_offset_pair.first;
-      uint64_t offset = seed_offset_pair.second;
-
-      // 7. flas_attn part one, get temp worksapce size.
-      uint64_t workspace_size;
-      float p_dropout = 0.f;
-      float softmax_scale = static_cast<float>(1);
-      cudaStream_t stream = dev_ctx_.stream();
-      int num_splits = 0;  // 0 for an internal heuristic, which is optimal
-      bool succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
-          static_cast<const void*>(q_ptr),
-          static_cast<const void*>(k_ptr),
-          static_cast<const void*>(v_ptr),
-          static_cast<void*>(q_grad_ptr),
-          static_cast<void*>(k_grad_ptr),
-          static_cast<void*>(v_grad_ptr),
-          static_cast<const void*>(fmha_out->data()),
-          static_cast<const void*>(fmha_out_grad->data()),
-          cu_seq_q.data<int32_t>(),
-          cu_seq_k.data<int32_t>(),
-          total_q_,
-          total_k_,
-          batch_size_,
-          num_heads_,
-          head_size_,
-          max_seqlen_q_,
-          max_seqlen_k_,
-          p_dropout,
-          softmax_scale,
-          /*zero_tensors=*/false,
-          /*is_causal=*/false,
-          is_bf16,
-          num_splits,
-          softmax_lse->data(),
-          softmax_d.data(),
-          bias_d.data(),
-          nullptr,
-          &workspace_size,
-          stream,
-          seed,
-          offset,
-          src_mask ? temp_mask.data() : nullptr,
-          nonbatched_bias ? temp_bias.data() : nullptr,
-          temp_mask.dims().Get(),
-          temp_bias.dims().Get());
-      if (!succ) {
-        PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
-      }
-      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
-
-      phi::DenseTensor workspace;
-      printf("workspace_size = %d\n", workspace_size);
-      if (workspace_size > 0) {
-        workspace = phi::Empty<float, phi::GPUContext>(
-            dev_ctx_, {int64_t(workspace_size / sizeof(float))});
-        DBGPTR(workspace.data(), "workspace");
-      }
-      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
-
-      succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
-          static_cast<const void*>(q_ptr),
-          static_cast<const void*>(k_ptr),
-          static_cast<const void*>(v_ptr),
-          static_cast<void*>(q_grad_ptr),
-          static_cast<void*>(k_grad_ptr),
-          static_cast<void*>(v_grad_ptr),
-          static_cast<const void*>(fmha_out->data()),
-          static_cast<const void*>(fmha_out_grad->data()),
-          cu_seq_q.data<int32_t>(),
-          cu_seq_k.data<int32_t>(),
-          total_q_,
-          total_k_,
-          batch_size_,
-          num_heads_,
-          head_size_,
-          max_seqlen_q_,
-          max_seqlen_k_,
-          p_dropout,
-          softmax_scale,
-          /*zero_tensors=*/false,
-          /*is_causal=*/false,
-          is_bf16,
-          num_splits,
-          softmax_lse->data(),
-          softmax_d.data(),
-          bias_d.data(),
-          workspace.data(),
-          &workspace_size,
-          stream,
-          seed,
-          offset,
-          src_mask ? temp_mask.data() : nullptr,
-          nonbatched_bias ? temp_bias.data() : nullptr,
-          temp_mask.dims().Get(),
-          temp_bias.dims().Get());
-      if (!succ) {
-        PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
-      }
-      LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
-
-      if (nonbatched_bias) {
-        // compare block reduce
-        // auto size = attn_bias->sizes();
-        // dbias = ds.reshape({ -1, size[0], size[1], size[2], size[3] }).sum({
-        // 0 }); result.push_back( dbias );
-        const auto temp_bias_num = temp_bias.numel();
-        const auto bias_d_num = bias_d.numel();
-        auto dbias_first_dim = bias_d_num / temp_bias_num;
-        bias_d.Resize({dbias_first_dim,
-                       temp_bias.dims()[0],
-                       temp_bias.dims()[1],
-                       temp_bias.dims()[2],
-                       temp_bias.dims()[3]});
-        phi::funcs::
-            ReduceKernel<T, T, kps::AddFunctor, kps::IdentityFunctor<T>>(
-                dev_ctx_,
-                bias_d,
-                nonbatched_bias_grad,
-                kps::IdentityFunctor<T>(),
-                {0});
-      }
-    } else {
-      if (merge_qkv_) {
-        PADDLE_ENFORCE_NOT_NULL(
-            qkv_transpose_out,
-            platform::errors::NotFound("The input qkv_transpose_out can not be "
-                                       "nullptr when merge_qkv is true."));
-
-        int64_t q_size = config->GetQuerySize();
-        q_ptr = qkv_transpose_out->data<T>();
-        k_ptr = q_ptr + q_size;
-        v_ptr = k_ptr + q_size;
-
-        qkv_transpose_out_grad.Resize(config->qkv_transpose_out_dims);
-        AllocWithDebugInfo<T>(
-            dev_ctx_, "qkv_transpose_out_grad", &qkv_transpose_out_grad);
-
-        q_grad_ptr = qkv_transpose_out_grad.data<T>();
-        k_grad_ptr = q_grad_ptr + q_size;
-        v_grad_ptr = k_grad_ptr + q_size;
-      } else {
-        PADDLE_ENFORCE_NOT_NULL(
-            q_transpose_out,
-            platform::errors::NotFound("The input q_transpose_out can not be "
-                                       "nullptr when merge_qkv is false."));
-        PADDLE_ENFORCE_NOT_NULL(
-            k_transpose_out,
-            platform::errors::NotFound("The input k_transpose_out can not be "
-                                       "nullptr when merge_qkv is false."));
-        PADDLE_ENFORCE_NOT_NULL(
-            v_transpose_out,
-            platform::errors::NotFound("The input v_transpose_out can not be "
-                                       "nullptr when merge_qkv is false."));
-
-        q_ptr = q_transpose_out->data<T>();
-        k_ptr = k_transpose_out->data<T>();
-        v_ptr = v_transpose_out->data<T>();
-
-        q_transpose_out_grad.Resize(config->q_transpose_out_dims);
-        k_transpose_out_grad.Resize(config->kv_transpose_out_dims);
-        v_transpose_out_grad.Resize(config->kv_transpose_out_dims);
-
-        q_grad_ptr = dev_ctx_.Alloc<T>(
-            &q_transpose_out_grad, q_transpose_out_grad.numel() * sizeof(T));
-        k_grad_ptr = dev_ctx_.Alloc<T>(
-            &k_transpose_out_grad, k_transpose_out_grad.numel() * sizeof(T));
-        v_grad_ptr = dev_ctx_.Alloc<T>(
-            &v_transpose_out_grad, v_transpose_out_grad.numel() * sizeof(T));
-      }
-
-      phi::DenseTensor softmax_out_grad;
-      softmax_out_grad.Resize(config->softmax_out_dims);
-      AllocWithDebugInfo<T>(dev_ctx_, "softmax_out_grad", &softmax_out_grad);
-
-      int64_t gemm_batch_size =
-          config->batch_size * config->seq_len_m * config->num_heads;
-      {
-        // Forward: fmha_out = transpose(qktv_out)
-        phi::DenseTensor qktv_out_grad;
-        qktv_out_grad.Resize(config->qktv_out_dims);
-        AllocWithDebugInfo<T>(dev_ctx_, "qktv_out_grad", &qktv_out_grad);
-        ComputeQKTVTransposeBackward(*fmha_out_grad, &qktv_out_grad);
-
-        // Forward: qktv_out = BatchedGEMM(softmax_out, V)
-        // Backward:
-        //  V_grad = BatchedGEMM(softmax_out^T, qktv_out_grad) (dy = x^T * dout)
-        int64_t gemm_m = config->m_size;
-        int64_t gemm_n = config->head_dim;
-        int64_t gemm_k = config->seq_len_r;
-
-        const T* softmax_out_ptr = softmax_out->data<T>();
-        const T* qktv_out_grad_ptr = qktv_out_grad.data<T>();
-        ComputeBatchedGEMM(softmax_out_ptr,
-                           qktv_out_grad_ptr,
-                           v_grad_ptr,
-                           true,
-                           false,
-                           gemm_m,
-                           gemm_n,
-                           gemm_k,
-                           gemm_batch_size);
-
-        // Backward: softmax_out_grad = qktv_out_grad * V^T (dx = dout * y^T)
-        gemm_m = config->seq_len_r;
-        gemm_n = config->m_size;
-        gemm_k = config->head_dim;
-
-        T* softmax_out_grad_ptr = softmax_out_grad.data<T>();
-        ComputeBatchedGEMM(qktv_out_grad_ptr,
-                           v_ptr,
-                           softmax_out_grad_ptr,
-                           false,
-                           true,
-                           gemm_m,
-                           gemm_n,
-                           gemm_k,
-                           gemm_batch_size);
-      }
-
-      phi::DenseTensor* qk_out_grad = config->GetQKOutGrad(&softmax_out_grad);
-      ComputeBiasMaskSoftmaxBackward(&softmax_out_grad,
-                                     softmax_out,
-                                     src_mask_grad,
-                                     qk_out_grad,
-                                     nonbatched_bias_grad);
+    phi::DenseTensor softmax_out_grad;
+    softmax_out_grad.Resize(config->softmax_out_dims);
+    AllocWithDebugInfo<T>(dev_ctx_, "softmax_out_grad", &softmax_out_grad);
 
-      // Forward: qk_out = BatchedGEMM(Q, K^T)
-      // Backward: k_grad = BatchedGEMM(qk_out_grad^T, Q) (dy = dout^t * x)
+    int64_t gemm_batch_size =
+        config->batch_size * config->seq_len_m * config->num_heads;
+    {
+      // Forward: fmha_out = transpose(qktv_out)
+      phi::DenseTensor qktv_out_grad;
+      qktv_out_grad.Resize(config->qktv_out_dims);
+      AllocWithDebugInfo<T>(dev_ctx_, "qktv_out_grad", &qktv_out_grad);
+      ComputeQKTVTransposeBackward(*fmha_out_grad, &qktv_out_grad);
+
+      // Forward: qktv_out = BatchedGEMM(softmax_out, V)
+      // Backward:
+      //  V_grad = BatchedGEMM(softmax_out^T, qktv_out_grad) (dy = x^T * dout)
       int64_t gemm_m = config->m_size;
       int64_t gemm_n = config->head_dim;
       int64_t gemm_k = config->seq_len_r;
-      T alpha = static_cast<T>(1.0 / sqrt(config->head_dim));
 
-      T* qk_out_grad_ptr = qk_out_grad->data<T>();
-      ComputeBatchedGEMM(qk_out_grad_ptr,
-                         q_ptr,
-                         k_grad_ptr,
+      const T* softmax_out_ptr = softmax_out->data<T>();
+      const T* qktv_out_grad_ptr = qktv_out_grad.data<T>();
+      ComputeBatchedGEMM(softmax_out_ptr,
+                         qktv_out_grad_ptr,
+                         v_grad_ptr,
                          true,
                          false,
                          gemm_m,
                          gemm_n,
                          gemm_k,
-                         gemm_batch_size,
-                         alpha);
+                         gemm_batch_size);
 
-      // Backward: q_grad = BatchedGEMM(qk_out_grad, K) (dx = dout * y)
+      // Backward: softmax_out_grad = qktv_out_grad * V^T (dx = dout * y^T)
       gemm_m = config->seq_len_r;
-      gemm_n = config->head_dim;
-      gemm_k = config->m_size;
-      ComputeBatchedGEMM(qk_out_grad_ptr,
-                         k_ptr,
-                         q_grad_ptr,
-                         false,
+      gemm_n = config->m_size;
+      gemm_k = config->head_dim;
+
+      T* softmax_out_grad_ptr = softmax_out_grad.data<T>();
+      ComputeBatchedGEMM(qktv_out_grad_ptr,
+                         v_ptr,
+                         softmax_out_grad_ptr,
                          false,
+                         true,
                          gemm_m,
                          gemm_n,
                          gemm_k,
-                         gemm_batch_size,
-                         alpha);
+                         gemm_batch_size);
     }
 
-    if (merge_qkv_ || config->use_flash_attn) {
+    phi::DenseTensor* qk_out_grad = config->GetQKOutGrad(&softmax_out_grad);
+    ComputeBiasMaskSoftmaxBackward(&softmax_out_grad,
+                                   softmax_out,
+                                   src_mask_grad,
+                                   qk_out_grad,
+                                   nonbatched_bias_grad);
+
+    // Forward: qk_out = BatchedGEMM(Q, K^T)
+    // Backward: k_grad = BatchedGEMM(qk_out_grad^T, Q) (dy = dout^t * x)
+    int64_t gemm_m = config->m_size;
+    int64_t gemm_n = config->head_dim;
+    int64_t gemm_k = config->seq_len_r;
+    T alpha = static_cast<T>(1.0 / sqrt(config->head_dim));
+
+    T* qk_out_grad_ptr = qk_out_grad->data<T>();
+    ComputeBatchedGEMM(qk_out_grad_ptr,
+                       q_ptr,
+                       k_grad_ptr,
+                       true,
+                       false,
+                       gemm_m,
+                       gemm_n,
+                       gemm_k,
+                       gemm_batch_size,
+                       alpha);
+
+    // Backward: q_grad = BatchedGEMM(qk_out_grad, K) (dx = dout * y)
+    gemm_m = config->seq_len_r;
+    gemm_n = config->head_dim;
+    gemm_k = config->m_size;
+    ComputeBatchedGEMM(qk_out_grad_ptr,
+                       k_ptr,
+                       q_grad_ptr,
+                       false,
+                       false,
+                       gemm_m,
+                       gemm_n,
+                       gemm_k,
+                       gemm_batch_size,
+                       alpha);
+
+    if (merge_qkv_) {
       phi::DenseTensor* qkv_out_grad = config->GetQKVOutGrad();
       ComputeQKVTransposeBackward(qkv_transpose_out_grad, qkv_out_grad);
     } else {
@@ -1433,6 +1179,290 @@ class FlashAttnWithGating {
     }
   }
 
+  void ComputeBackward(const phi::DenseTensor* q_transpose_out,
+                       const phi::DenseTensor* k_transpose_out,
+                       const phi::DenseTensor* v_transpose_out,
+                       const phi::DenseTensor* qkv_transpose_out,
+                       const phi::DenseTensor* fmha_out_grad,
+                       phi::DenseTensor* src_mask_grad,
+                       phi::DenseTensor* nonbatched_bias_grad,
+                       GateAttentionGradConfig<T>* config,
+                       const phi::DenseTensor* fmha_out = nullptr,
+                       const phi::DenseTensor* softmax_lse = nullptr,
+                       const phi::DenseTensor* nonbatched_bias = nullptr,
+                       const phi::DenseTensor* src_mask = nullptr) {
+    T* q_grad_ptr = nullptr;
+    T* k_grad_ptr = nullptr;
+    T* v_grad_ptr = nullptr;
+
+    phi::DenseTensor q_transpose_out_grad;
+    phi::DenseTensor k_transpose_out_grad;
+    phi::DenseTensor v_transpose_out_grad;
+    phi::DenseTensor qkv_transpose_out_grad;
+
+    bool is_bf16 =
+        qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
+
+    if (std::is_same<T, phi::dtype::float16>::value) {
+      std::cout << "[Grad]: T is phi::dtype::float16. \n";
+    } else if (std::is_same<T, phi::dtype::bfloat16>::value) {
+      std::cout << "[Grad]: T is phi::dtype::bfloat16. \n";
+    } else if (std::is_same<T, float>::value) {
+      std::cout << "[Grad]: T is float. \n";
+    }
+
+    PADDLE_ENFORCE_NOT_NULL(
+        qkv_transpose_out,
+        platform::errors::NotFound("The input qkv_transpose_out can not be"
+                                   "nullptr when merge_qkv is true."));
+    int64_t q_size = config->GetQuerySize();
+    const T* q_ptr = qkv_transpose_out->data<T>();
+    const T* k_ptr = q_ptr + q_size;
+    const T* v_ptr = k_ptr + q_size;
+
+    qkv_transpose_out_grad.Resize(config->qkv_transpose_out_dims);
+    AllocWithDebugInfo<T>(
+        dev_ctx_, "qkv_transpose_out_grad", &qkv_transpose_out_grad);
+
+    int seq_batch_size = static_cast<int>(config->batch_size) *
+                         static_cast<int>(config->seq_len_m);
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+
+    // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
+    phi::DenseTensor cu_seq_q, cu_seq_k;
+    int64_t start = 0;
+    int64_t step = static_cast<int>(config->seq_len_r);
+    int64_t end_size = (seq_batch_size + 1);
+    int64_t end = end_size;
+    int64_t seq_size = 0;
+    phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
+    cu_seq_q.Resize({end_size});
+    cu_seq_k.Resize({end_size});
+    AllocWithDebugInfo<int32_t>(dev_ctx_, "Grad: cu_seq_q", &cu_seq_q);
+    AllocWithDebugInfo<int32_t>(dev_ctx_, "Grad: cu_seq_k", &cu_seq_k);
+    int64_t block = std::min(seq_size, static_cast<int64_t>(256));
+    int64_t grid = (seq_size + block - 1) / block;
+    FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
+        start, step, end, cu_seq_q.data<int32_t>(), cu_seq_k.data<int32_t>());
+    VLOG(4) << "[Flash_attn] cu_seq_len : start = " << start
+            << ", step = " << step << ", end = " << end;
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+
+    // 3. Dealing with mask and bias for flash_attn.
+    phi::DenseTensor temp_mask, temp_bias;
+    auto dims_merge_func = [&](const phi::DenseTensor* src_tensor,
+                               phi::DenseTensor* dst_tensor,
+                               const std::string& prefix) {
+      if (src_tensor) {
+        int64_t first_dim = 1;
+        dst_tensor->ShareDataWith(*src_tensor);
+        auto dims_ = src_tensor->dims();
+        for (int i = 0; i < dims_.size() - 3; ++i) {
+          first_dim *= dims_[i];
+        }
+        auto dims_rank = dims_.size();
+        dst_tensor->Resize({first_dim,
+                            dims_[dims_rank - 3],
+                            dims_[dims_rank - 2],
+                            dims_[dims_rank - 1]});
+        GetFlashAttnDimsString(prefix, temp_mask.dims());
+      }
+    };
+    dims_merge_func(src_mask, &temp_mask, "[Grad] mask_dim");
+    dims_merge_func(nonbatched_bias, &temp_bias, "[Grad] bias_dim");
+
+    phi::DDim qkv_dims({3,
+                        seq_batch_size * static_cast<int>(config->seq_len_r),
+                        static_cast<int>(config->num_heads),
+                        static_cast<int>(config->head_dim)});
+    int batch_size_ = seq_batch_size;
+    int total_q_ = qkv_dims[1];    // q.dims()[0]
+    int total_k_ = qkv_dims[1];    // q.dims()[0]
+    int num_heads_ = qkv_dims[2];  // q.dims()[1]
+    int head_size_ = qkv_dims[3];  // q.dims()[2]
+    int max_seqlen_q_ = batch_size_;
+    int max_seqlen_k_ = batch_size_;
+    VLOG(6) << "[Flash_attn Grad] batch_size : " << batch_size_;
+    VLOG(6) << "[Flash_attn Grad] total_q   : " << total_q_;
+    VLOG(6) << "[Flash_attn Grad] total_k   : " << total_k_;
+    VLOG(6) << "[Flash_attn Grad] num_heads : " << num_heads_;
+    VLOG(6) << "[Flash_attn Grad] head_size : " << head_size_;
+    VLOG(6) << "[Flash_attn Grad] max_seqlen_q : " << max_seqlen_q_;
+    VLOG(6) << "[Flash_attn Grad] max_seqlen_k : " << max_seqlen_k_;
+
+    // 5. construct softmax_lse
+    int last_q_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
+    // softmax_lse->Resize({batch_size_, num_heads_, last_q_dim});
+    // AllocWithDebugInfo<float>(
+    //     dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+
+    phi::DenseTensor softmax_d = phi::Empty<float, phi::GPUContext>(
+        dev_ctx_, {batch_size_, num_heads_, last_q_dim});
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+
+    phi::DenseTensor bias_d;
+    if (nonbatched_bias) {
+      bias_d = phi::Empty<T, phi::GPUContext>(
+          dev_ctx_, {batch_size_, num_heads_, max_seqlen_q_, max_seqlen_k_});
+    }
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+
+    q_ptr = q_transpose_out->data<T>();
+    k_ptr = k_transpose_out->data<T>();
+    v_ptr = v_transpose_out->data<T>();
+    q_transpose_out_grad.Resize(config->q_transpose_out_dims);
+    k_transpose_out_grad.Resize(config->kv_transpose_out_dims);
+    v_transpose_out_grad.Resize(config->kv_transpose_out_dims);
+
+    q_grad_ptr = dev_ctx_.Alloc<T>(&q_transpose_out_grad,
+                                   q_transpose_out_grad.numel() * sizeof(T));
+    k_grad_ptr = dev_ctx_.Alloc<T>(&k_transpose_out_grad,
+                                   k_transpose_out_grad.numel() * sizeof(T));
+    v_grad_ptr = dev_ctx_.Alloc<T>(&v_transpose_out_grad,
+                                   v_transpose_out_grad.numel() * sizeof(T));
+
+    // 6. construct random seed
+    auto gen = dev_ctx_.GetGenerator();
+    uint64_t inc = batch_size_ * num_heads_ * 32;
+    auto seed_offset_pair = gen->IncrementOffset(inc);
+    uint64_t seed = seed_offset_pair.first;
+    uint64_t offset = seed_offset_pair.second;
+
+    // 7. flas_attn part one, get temp worksapce size.
+    uint64_t workspace_size;
+    float p_dropout = 0.f;
+    float softmax_scale = static_cast<float>(1);
+    cudaStream_t stream = dev_ctx_.stream();
+    int num_splits = 0;  // 0 for an internal heuristic, which is optimal
+    bool succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
+        static_cast<const void*>(q_ptr),
+        static_cast<const void*>(k_ptr),
+        static_cast<const void*>(v_ptr),
+        static_cast<void*>(q_grad_ptr),
+        static_cast<void*>(k_grad_ptr),
+        static_cast<void*>(v_grad_ptr),
+        static_cast<const void*>(fmha_out->data()),
+        static_cast<const void*>(fmha_out_grad->data()),
+        cu_seq_q.data<int32_t>(),
+        cu_seq_k.data<int32_t>(),
+        total_q_,
+        total_k_,
+        batch_size_,
+        num_heads_,
+        head_size_,
+        max_seqlen_q_,
+        max_seqlen_k_,
+        p_dropout,
+        softmax_scale,
+        /*zero_tensors=*/false,
+        /*is_causal=*/false,
+        is_bf16,
+        num_splits,
+        softmax_lse->data(),
+        softmax_d.data(),
+        bias_d.data(),
+        nullptr,
+        &workspace_size,
+        stream,
+        seed,
+        offset,
+        src_mask ? temp_mask.data() : nullptr,
+        nonbatched_bias ? temp_bias.data() : nullptr,
+        src_mask ? temp_mask.dims().Get() : nullptr,
+        nonbatched_bias ? temp_bias.dims().Get() : nullptr);
+    if (!succ) {
+      PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
+    }
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+
+    phi::DenseTensor workspace;
+    printf("workspace_size = %d\n", workspace_size);
+    if (workspace_size > 0) {
+      workspace = phi::Empty<float, phi::GPUContext>(
+          dev_ctx_, {int64_t(workspace_size / sizeof(float))});
+      DBGPTR(workspace.data(), "workspace");
+    }
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+
+    succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
+        static_cast<const void*>(q_ptr),
+        static_cast<const void*>(k_ptr),
+        static_cast<const void*>(v_ptr),
+        static_cast<void*>(q_grad_ptr),
+        static_cast<void*>(k_grad_ptr),
+        static_cast<void*>(v_grad_ptr),
+        static_cast<const void*>(fmha_out->data()),
+        static_cast<const void*>(fmha_out_grad->data()),
+        cu_seq_q.data<int32_t>(),
+        cu_seq_k.data<int32_t>(),
+        total_q_,
+        total_k_,
+        batch_size_,
+        num_heads_,
+        head_size_,
+        max_seqlen_q_,
+        max_seqlen_k_,
+        p_dropout,
+        softmax_scale,
+        /*zero_tensors=*/false,
+        /*is_causal=*/false,
+        is_bf16,
+        num_splits,
+        softmax_lse->data(),
+        softmax_d.data(),
+        bias_d.data(),
+        workspace.data(),
+        &workspace_size,
+        stream,
+        seed,
+        offset,
+        src_mask ? temp_mask.data() : nullptr,
+        nonbatched_bias ? temp_bias.data() : nullptr,
+        src_mask ? temp_mask.dims().Get() : nullptr,
+        nonbatched_bias ? temp_bias.dims().Get() : nullptr);
+    if (!succ) {
+      PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
+    }
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+
+    if (nonbatched_bias) {
+      // compare block reduce
+      // auto size = attn_bias->sizes();
+      // dbias = ds.reshape({ -1, size[0], size[1], size[2], size[3] }).sum({
+      // 0 }); result.push_back( dbias );
+      const auto temp_bias_num = temp_bias.numel();
+      const auto bias_d_num = bias_d.numel();
+      auto dbias_first_dim = bias_d_num / temp_bias_num;
+      bias_d.Resize({dbias_first_dim,
+                     temp_bias.dims()[0],
+                     temp_bias.dims()[1],
+                     temp_bias.dims()[2],
+                     temp_bias.dims()[3]});
+      phi::funcs::ReduceKernel<T, T, kps::AddFunctor, kps::IdentityFunctor<T>>(
+          dev_ctx_,
+          bias_d,
+          nonbatched_bias_grad,
+          kps::IdentityFunctor<T>(),
+          {0});
+    }
+
+    if (merge_qkv_) {
+      phi::DenseTensor* qkv_out_grad = config->GetQKVOutGrad();
+      ComputeQKVTransposeBackward(qkv_transpose_out_grad, qkv_out_grad);
+    } else {
+      phi::DenseTensor* q_out_grad = config->GetQueryOutGrad();
+      phi::DenseTensor* k_out_grad = config->GetKeyOutGrad();
+      phi::DenseTensor* v_out_grad = config->GetValueOutGrad();
+      ComputeQKVTransposeBackward(q_transpose_out_grad,
+                                  k_transpose_out_grad,
+                                  v_transpose_out_grad,
+                                  q_out_grad,
+                                  k_out_grad,
+                                  v_out_grad);
+    }
+  }
+
  private:
   // [batch_size, seq_len_m, seq_len_r, 3, num_heads, head_dim] ->
   //         [3, batch_size, seq_len_m, seq_len_r, num_heads, head_dim]
@@ -1443,6 +1473,29 @@ class FlashAttnWithGating {
         dev_ctx_, qkv_out, perm, qkv_transpose_out);
   }
 
+  void ComputeQKVTransposeBackward(const phi::DenseTensor& q_transpose_out_grad,
+                                   const phi::DenseTensor& k_transpose_out_grad,
+                                   const phi::DenseTensor& v_transpose_out_grad,
+                                   phi::DenseTensor* q_out_grad,
+                                   phi::DenseTensor* k_out_grad,
+                                   phi::DenseTensor* v_out_grad) {
+    std::vector<int> perm = {0, 1, 3, 2, 4};
+    phi::funcs::TransposeGPUKernelDriver<T>(
+        dev_ctx_, q_transpose_out_grad, perm, q_out_grad);
+    phi::funcs::TransposeGPUKernelDriver<T>(
+        dev_ctx_, k_transpose_out_grad, perm, k_out_grad);
+    phi::funcs::TransposeGPUKernelDriver<T>(
+        dev_ctx_, v_transpose_out_grad, perm, v_out_grad);
+  }
+
+  void ComputeQKVTransposeBackward(
+      const phi::DenseTensor& qkv_transpose_out_grad,
+      phi::DenseTensor* qkv_out_grad) {
+    std::vector<int> perm = {1, 2, 4, 0, 3, 5};
+    phi::funcs::TransposeGPUKernelDriver<T>(
+        dev_ctx_, qkv_transpose_out_grad, perm, qkv_out_grad);
+  }
+
   const phi::GPUContext& dev_ctx_;
   bool merge_qkv_;
 };
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cc b/paddle/fluid/operators/fused/fused_gate_attention_op.cc
index a3fd178059fab..9743bc33fd055 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cc
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cc
@@ -231,15 +231,15 @@ class FusedGateAttentionGradOp : public framework::OperatorWithKernel {
       OP_INOUT_CHECK(ctx->HasInput("QueryWeight"),
                      "Input",
                      "QueryWeight",
-                     "fused_aate_attention_arad");
+                     "fused_gate_attention_arad");
       OP_INOUT_CHECK(ctx->HasInput("KeyWeight"),
                      "Input",
                      "KeyWeight",
-                     "fused_aate_attention_arad");
+                     "fused_gate_attention_arad");
       OP_INOUT_CHECK(ctx->HasInput("ValueWeight"),
                      "Input",
                      "ValueWeight",
-                     "fused_aate_attention_arad");
+                     "fused_gate_attention_arad");
 
       for (auto& name : {"QueryWeight", "KeyWeight", "ValueWeight"}) {
         ctx->SetOutputDim(framework::GradVarName(name), ctx->GetInputDim(name));
@@ -267,6 +267,27 @@ class FusedGateAttentionGradOp : public framework::OperatorWithKernel {
     ctx->SetOutputDim(framework::GradVarName("OutLinearBias"),
                       ctx->GetInputDim("OutLinearBias"));
   }
+
+ protected:
+  phi::KernelKey GetExpectedKernelType(
+      const framework::ExecutionContext& ctx) const override {
+    auto input = ctx.Input<phi::DenseTensor>("Query");
+    auto input_data_type = framework::TransToProtoVarType(input->dtype());
+    return phi::KernelKey(input_data_type, ctx.GetPlace());
+  }
+
+  phi::KernelKey GetKernelTypeForVar(
+      const std::string& var_name,
+      const phi::DenseTensor& tensor,
+      const phi::KernelKey& expected_kernel_type) const override {
+    if (var_name == "SoftmaxLse") {
+      return phi::KernelKey(phi::Backend::ALL_BACKEND,
+                            expected_kernel_type.layout(),
+                            expected_kernel_type.dtype());
+    }
+    return phi::KernelKey(
+        tensor.place(), tensor.layout(), expected_kernel_type.dtype());
+  }
 };
 
 template <typename T>
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cu b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
index 3d10047bf23a0..a0f97ce59109b 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cu
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
@@ -474,6 +474,8 @@ template <typename T, typename DeviceContext>
 class FusedGateAttentionGradKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext &ctx) const override {
+    LOG(INFO) << "In FusedGateAttentionGradKernel";
+
     // forward input
     const auto *query = ctx.Input<phi::DenseTensor>("Query");
     const auto *key = ctx.Input<phi::DenseTensor>("Key");
@@ -489,7 +491,6 @@ class FusedGateAttentionGradKernel : public framework::OpKernel<T> {
         ctx.Input<phi::DenseTensor>("ValueTransposeOut");
     const auto *qkv_transpose_out =
         ctx.Input<phi::DenseTensor>("QKVTransposeOut");
-    const auto *softmax_out = ctx.Input<phi::DenseTensor>("SoftmaxOut");
     const auto *fmha_out = ctx.Input<phi::DenseTensor>("FMHAOut");
     const auto *gate_out = ctx.Input<phi::DenseTensor>("GateOut");
 
@@ -516,17 +517,6 @@ class FusedGateAttentionGradKernel : public framework::OpKernel<T> {
                                       has_gating,
                                       use_flash_attn);
 
-    const phi::DenseTensor *fwd_out = nullptr;
-    const phi::DenseTensor *fwd_bias = nullptr;
-    const phi::DenseTensor *fwd_mask = nullptr;
-    const phi::DenseTensor *fwd_softmax_lse = nullptr;
-    if (merge_qkv && use_flash_attn) {
-      fwd_bias = ctx.Input<phi::DenseTensor>("NonbatchedBias");
-      fwd_mask = ctx.Input<phi::DenseTensor>("SrcMask");
-      fwd_softmax_lse = ctx.Input<phi::DenseTensor>("SoftmaxLse");
-      fwd_out = fmha_out;
-    }
-
     phi::DenseTensor fmha_out_grad;
     fmha_out_grad.Resize(config.gate_out_dims);
     AllocWithDebugInfo<T>(dev_ctx, "fmha_out_grad", &fmha_out_grad);
@@ -560,28 +550,29 @@ class FusedGateAttentionGradKernel : public framework::OpKernel<T> {
           dev_ctx, "nonbatched_bias_grad", nonbatched_bias_grad);
     }
 
-    auto fmha_compute = FMHAGateRef<T>(dev_ctx, merge_qkv);
-
-    if (use_flash_attn) {
-      const phi::DenseTensor *fwd_bias =
+    if (config.CanUseFlashAttn()) {
+      const auto *non_batched_bias =
           ctx.Input<phi::DenseTensor>("NonbatchedBias");
-      const phi::DenseTensor *fwd_mask = ctx.Input<phi::DenseTensor>("SrcMask");
-      const phi::DenseTensor *fwd_softmax_lse =
-          ctx.Input<phi::DenseTensor>("SoftmaxLse");
+      const auto *src_mask = ctx.Input<phi::DenseTensor>("SrcMask");
+      const auto *softmax_lse = ctx.Input<phi::DenseTensor>("SoftmaxLse");
+
+      auto fmha_compute = FlashAttnWithGating<T>(dev_ctx, merge_qkv);
       fmha_compute.ComputeBackward(q_transpose_out,
                                    k_transpose_out,
                                    v_transpose_out,
                                    qkv_transpose_out,
-                                   softmax_out,
                                    &fmha_out_grad,
                                    nullptr,
                                    nonbatched_bias_grad,
                                    &config,
                                    fmha_out,
-                                   fwd_softmax_lse,
-                                   fwd_bias,
-                                   fwd_mask);
+                                   softmax_lse,
+                                   non_batched_bias,
+                                   src_mask);
     } else {
+      const auto *softmax_out = ctx.Input<phi::DenseTensor>("SoftmaxOut");
+
+      auto fmha_compute = FMHAGateRef<T>(dev_ctx, merge_qkv);
       fmha_compute.ComputeBackward(q_transpose_out,
                                    k_transpose_out,
                                    v_transpose_out,

From 37a095390bf3fea5c1190d9fd9a2bf66ff420161 Mon Sep 17 00:00:00 2001
From: Ruibiao Chen <chenruibiao@baidu.com>
Date: Tue, 25 Apr 2023 09:12:17 +0800
Subject: [PATCH 039/405] Remove managed memory msg in cuda allocator (#53263)

---
 paddle/fluid/memory/allocation/cuda_allocator.cc | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/paddle/fluid/memory/allocation/cuda_allocator.cc b/paddle/fluid/memory/allocation/cuda_allocator.cc
index 4ea182cb722ce..781addd7dba60 100644
--- a/paddle/fluid/memory/allocation/cuda_allocator.cc
+++ b/paddle/fluid/memory/allocation/cuda_allocator.cc
@@ -70,29 +70,20 @@ phi::Allocation* CUDAAllocator::AllocateImpl(size_t size) {
         limit_size);
   }
 
-  std::string managed_memory_msg;
-  if (platform::IsGPUManagedMemoryOversubscriptionSupported(place_.device)) {
-    managed_memory_msg = string::Sprintf(
-        "If the above ways do not solve the out of memory problem, you can try "
-        "to use CUDA managed memory. The command is `export "
-        "FLAGS_use_cuda_managed_memory=true`.");
-  }
-
   PADDLE_THROW_BAD_ALLOC(platform::errors::ResourceExhausted(
       "\n\nOut of memory error on GPU %d. "
       "Cannot allocate %s memory on GPU %d, %s memory has been allocated and "
       "available memory is only %s.\n\n"
       "Please check whether there is any other process using GPU %d.\n"
       "1. If yes, please stop them, or start PaddlePaddle on another GPU.\n"
-      "2. If no, please decrease the batch size of your model. %s\n%s\n",
+      "2. If no, please decrease the batch size of your model. %s\n",
       place_.device,
       string::HumanReadableSize(size),
       place_.device,
       string::HumanReadableSize(allocated),
       string::HumanReadableSize(avail),
       place_.device,
-      err_msg,
-      managed_memory_msg));
+      err_msg));
 }
 
 }  // namespace allocation

From ade7a070c5b7012d1e6a7a6d810cf1fe13984f38 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Tue, 25 Apr 2023 09:33:38 +0800
Subject: [PATCH 040/405] Simplify codes.

---
 .../operators/fused/fused_gate_attention.h    | 155 +++++++-----------
 1 file changed, 61 insertions(+), 94 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 07b4b5973cac8..8e91eca7b2e8a 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -998,23 +998,8 @@ class FlashAttnWithGating {
 
     // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
     phi::DenseTensor cu_seq_q, cu_seq_k;
-    int64_t end_size = seq_batch_size + 1;
-    int64_t seq_size = 0;
-    int64_t start = 0;
-    int64_t end = end_size;
-    int64_t step = static_cast<int>(config->seq_len_r);
-    phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
-    cu_seq_q.Resize({end_size});
-    cu_seq_k.Resize({end_size});
-    AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_q", &cu_seq_q);
-    AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_k", &cu_seq_k);
-    int64_t block = std::min(seq_size, static_cast<int64_t>(256));
-    int64_t grid = (seq_size + block - 1) / block;
-    FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
-        start, step, end, cu_seq_q.data<int32_t>(), cu_seq_k.data<int32_t>());
-    VLOG(4) << "[Flash_attn] cu_seq_len : start = " << start
-            << ", step = " << step << ", end = " << end;
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_) << "2: Init cu_seq_q and cu_seq_k";
+    int64_t step = static_cast<int64_t>(config->seq_len_r);
+    AllocAndInitSeqQK(seq_batch_size, step, &cu_seq_q, &cu_seq_k);
 
     // 3. Dealing with mask and bias for flash_attn.
     phi::DenseTensor temp_mask, temp_bias;
@@ -1071,9 +1056,7 @@ class FlashAttnWithGating {
               << "]";
 
     // 6. construct random seed
-    auto gen = dev_ctx_.GetGenerator();
-    uint64_t inc = batch_size_ * num_heads_ * 32;
-    auto seed_offset_pair = gen->IncrementOffset(inc);
+    auto seed_offset_pair = GenerateSeedOffsetPair(batch_size_, num_heads_);
     uint64_t seed = seed_offset_pair.first;
     uint64_t offset = seed_offset_pair.second;
 
@@ -1124,13 +1107,7 @@ class FlashAttnWithGating {
     LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
               << "7: Get workspace_size=" << workspace_size;
 
-    phi::DenseTensor workspace;
-    if (workspace_size > 0) {
-      workspace = phi::Empty<float, phi::GPUContext>(
-          dev_ctx_, {int64_t(workspace_size / sizeof(float))});
-      DBGPTR(workspace.data(), "workspace");
-    }
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_) << "Allocate workspace";
+    phi::DenseTensor workspace = CreateWorkspace(workspace_size);
 
     LOG(INFO) << "qkv_transpose_out: " << TensorDebugString(qkv_transpose_out);
     LOG(INFO) << "src_mask: " << TensorDebugString(src_mask);
@@ -1191,15 +1168,6 @@ class FlashAttnWithGating {
                        const phi::DenseTensor* softmax_lse = nullptr,
                        const phi::DenseTensor* nonbatched_bias = nullptr,
                        const phi::DenseTensor* src_mask = nullptr) {
-    T* q_grad_ptr = nullptr;
-    T* k_grad_ptr = nullptr;
-    T* v_grad_ptr = nullptr;
-
-    phi::DenseTensor q_transpose_out_grad;
-    phi::DenseTensor k_transpose_out_grad;
-    phi::DenseTensor v_transpose_out_grad;
-    phi::DenseTensor qkv_transpose_out_grad;
-
     bool is_bf16 =
         qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
 
@@ -1215,38 +1183,29 @@ class FlashAttnWithGating {
         qkv_transpose_out,
         platform::errors::NotFound("The input qkv_transpose_out can not be"
                                    "nullptr when merge_qkv is true."));
+
     int64_t q_size = config->GetQuerySize();
     const T* q_ptr = qkv_transpose_out->data<T>();
     const T* k_ptr = q_ptr + q_size;
     const T* v_ptr = k_ptr + q_size;
 
+    phi::DenseTensor qkv_transpose_out_grad;
     qkv_transpose_out_grad.Resize(config->qkv_transpose_out_dims);
     AllocWithDebugInfo<T>(
         dev_ctx_, "qkv_transpose_out_grad", &qkv_transpose_out_grad);
 
+    T* q_grad_ptr = qkv_transpose_out_grad.data<T>();
+    T* k_grad_ptr = q_grad_ptr + q_size;
+    T* v_grad_ptr = k_grad_ptr + q_size;
+
     int seq_batch_size = static_cast<int>(config->batch_size) *
                          static_cast<int>(config->seq_len_m);
     LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
-    // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
+    // 2. Init with cu_seq_q and cu_seq_k for flash_attn.
     phi::DenseTensor cu_seq_q, cu_seq_k;
-    int64_t start = 0;
-    int64_t step = static_cast<int>(config->seq_len_r);
-    int64_t end_size = (seq_batch_size + 1);
-    int64_t end = end_size;
-    int64_t seq_size = 0;
-    phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
-    cu_seq_q.Resize({end_size});
-    cu_seq_k.Resize({end_size});
-    AllocWithDebugInfo<int32_t>(dev_ctx_, "Grad: cu_seq_q", &cu_seq_q);
-    AllocWithDebugInfo<int32_t>(dev_ctx_, "Grad: cu_seq_k", &cu_seq_k);
-    int64_t block = std::min(seq_size, static_cast<int64_t>(256));
-    int64_t grid = (seq_size + block - 1) / block;
-    FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
-        start, step, end, cu_seq_q.data<int32_t>(), cu_seq_k.data<int32_t>());
-    VLOG(4) << "[Flash_attn] cu_seq_len : start = " << start
-            << ", step = " << step << ", end = " << end;
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+    int64_t step = static_cast<int64_t>(config->seq_len_r);
+    AllocAndInitSeqQK(seq_batch_size, step, &cu_seq_q, &cu_seq_k);
 
     // 3. Dealing with mask and bias for flash_attn.
     phi::DenseTensor temp_mask, temp_bias;
@@ -1308,24 +1267,8 @@ class FlashAttnWithGating {
     }
     LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
-    q_ptr = q_transpose_out->data<T>();
-    k_ptr = k_transpose_out->data<T>();
-    v_ptr = v_transpose_out->data<T>();
-    q_transpose_out_grad.Resize(config->q_transpose_out_dims);
-    k_transpose_out_grad.Resize(config->kv_transpose_out_dims);
-    v_transpose_out_grad.Resize(config->kv_transpose_out_dims);
-
-    q_grad_ptr = dev_ctx_.Alloc<T>(&q_transpose_out_grad,
-                                   q_transpose_out_grad.numel() * sizeof(T));
-    k_grad_ptr = dev_ctx_.Alloc<T>(&k_transpose_out_grad,
-                                   k_transpose_out_grad.numel() * sizeof(T));
-    v_grad_ptr = dev_ctx_.Alloc<T>(&v_transpose_out_grad,
-                                   v_transpose_out_grad.numel() * sizeof(T));
-
     // 6. construct random seed
-    auto gen = dev_ctx_.GetGenerator();
-    uint64_t inc = batch_size_ * num_heads_ * 32;
-    auto seed_offset_pair = gen->IncrementOffset(inc);
+    auto seed_offset_pair = GenerateSeedOffsetPair(batch_size_, num_heads_);
     uint64_t seed = seed_offset_pair.first;
     uint64_t offset = seed_offset_pair.second;
 
@@ -1376,15 +1319,7 @@ class FlashAttnWithGating {
     }
     LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
-    phi::DenseTensor workspace;
-    printf("workspace_size = %d\n", workspace_size);
-    if (workspace_size > 0) {
-      workspace = phi::Empty<float, phi::GPUContext>(
-          dev_ctx_, {int64_t(workspace_size / sizeof(float))});
-      DBGPTR(workspace.data(), "workspace");
-    }
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
-
+    phi::DenseTensor workspace = CreateWorkspace(workspace_size);
     succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
         static_cast<const void*>(q_ptr),
         static_cast<const void*>(k_ptr),
@@ -1447,23 +1382,55 @@ class FlashAttnWithGating {
           {0});
     }
 
-    if (merge_qkv_) {
-      phi::DenseTensor* qkv_out_grad = config->GetQKVOutGrad();
-      ComputeQKVTransposeBackward(qkv_transpose_out_grad, qkv_out_grad);
-    } else {
-      phi::DenseTensor* q_out_grad = config->GetQueryOutGrad();
-      phi::DenseTensor* k_out_grad = config->GetKeyOutGrad();
-      phi::DenseTensor* v_out_grad = config->GetValueOutGrad();
-      ComputeQKVTransposeBackward(q_transpose_out_grad,
-                                  k_transpose_out_grad,
-                                  v_transpose_out_grad,
-                                  q_out_grad,
-                                  k_out_grad,
-                                  v_out_grad);
-    }
+    phi::DenseTensor* qkv_out_grad = config->GetQKVOutGrad();
+    ComputeQKVTransposeBackward(qkv_transpose_out_grad, qkv_out_grad);
   }
 
  private:
+  void AllocAndInitSeqQK(int64_t seq_batch_size,
+                         int64_t step,
+                         phi::DenseTensor* cu_seq_q,
+                         phi::DenseTensor* cu_seq_k) {
+    int64_t start = 0;
+    int64_t end_size = seq_batch_size + 1;
+    int64_t end = end_size;
+    int64_t seq_size = 0;
+    phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
+
+    cu_seq_q->Resize({end_size});
+    cu_seq_k->Resize({end_size});
+    AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_q", cu_seq_q);
+    AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_k", cu_seq_k);
+
+    int64_t block = std::min(seq_size, static_cast<int64_t>(256));
+    int64_t grid = (seq_size + block - 1) / block;
+    FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
+        start, step, end, cu_seq_q->data<int32_t>(), cu_seq_k->data<int32_t>());
+
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
+              << "AllocAndInit cu_seq_q and cu_seq_k: start=" << start
+              << ", step=" << step << ", end=" << end;
+  }
+
+  phi::DenseTensor CreateWorkspace(uint64_t workspace_size) {
+    phi::DenseTensor workspace;
+    if (workspace_size > 0) {
+      workspace = phi::Empty<float, phi::GPUContext>(
+          dev_ctx_, {int64_t(workspace_size / sizeof(float))});
+      DBGPTR(workspace.data(), "workspace");
+    }
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
+              << "Allocate workspace: workspace_size=" << workspace_size;
+    return workspace;
+  }
+
+  std::pair<uint64_t, uint64_t> GenerateSeedOffsetPair(int64_t batch_size,
+                                                       int64_t num_heads) {
+    auto gen = dev_ctx_.GetGenerator();
+    uint64_t inc = batch_size * num_heads * 32;
+    return gen->IncrementOffset(inc);
+  }
+
   // [batch_size, seq_len_m, seq_len_r, 3, num_heads, head_dim] ->
   //         [3, batch_size, seq_len_m, seq_len_r, num_heads, head_dim]
   void ComputeQKVTransposeForwardForFlashAttn(

From b7565222beee8bfaec05048c05c5b17a71de289a Mon Sep 17 00:00:00 2001
From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com>
Date: Tue, 25 Apr 2023 11:04:05 +0800
Subject: [PATCH 041/405] add syncthreads (#53149)

---
 paddle/phi/kernels/funcs/dropout_impl.cu.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/paddle/phi/kernels/funcs/dropout_impl.cu.h b/paddle/phi/kernels/funcs/dropout_impl.cu.h
index 0b47febb0d30e..006784eb64c76 100644
--- a/paddle/phi/kernels/funcs/dropout_impl.cu.h
+++ b/paddle/phi/kernels/funcs/dropout_impl.cu.h
@@ -272,6 +272,7 @@ __global__ void DropOutNdForwardKernel(
       idx = fast_divmoder.val[0];
       offset += broadcast_config.strides[j] * fast_divmoder.val[1];
     }
+    __syncthreads();
     y[i] = dst_functor(src[i], mask[offset]);
   }
 }

From a1f8f411c24ccb8c7f345d102ac8c777789a8624 Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Tue, 25 Apr 2023 11:22:27 +0800
Subject: [PATCH 042/405] change gpu_time_diff (#53252)

---
 tools/check_op_benchmark_result.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/check_op_benchmark_result.py b/tools/check_op_benchmark_result.py
index 4396734a85491..30d417b2fb771 100644
--- a/tools/check_op_benchmark_result.py
+++ b/tools/check_op_benchmark_result.py
@@ -74,7 +74,7 @@ def check_speed_result(case_name, develop_data, pr_data, pr_result):
         gpu_time_diff = (pr_gpu_time - develop_gpu_time) / develop_gpu_time
         gpu_time_diff_str = "{:.5f}".format(gpu_time_diff * 100)
     else:
-        gpu_time_diff = None
+        gpu_time_diff = 0
         gpu_time_diff_str = ""
 
     pr_total_time = pr_data.get("total")

From 5ca3bc6d981ce29b625c5127d87f5a22d2d6352e Mon Sep 17 00:00:00 2001
From: Meteor Liu <liudongxue01@baidu.com>
Date: Tue, 25 Apr 2023 11:25:13 +0800
Subject: [PATCH 043/405] rename monkey_patch_{math_}varbase as
 monkey_patch_{math_}tensor (#53191)

* rename monkey_patch_varbase as monkey_patch_tensor & monkey_patch_math_varbase as monkey_patch_math_tensor

* rename monkey_patch_varbase as monkey_patch_tensor & monkey_patch_math_varbase as monkey_patch_math_tensor

* rename monkey_patch_varbase as monkey_patch_tensor & monkey_patch_math_varbase as monkey_patch_math_tensor v2

* rename monkey_patch_varbase as monkey_patch_tensor & monkey_patch_math_varbase as monkey_patch_math_tensor fixed bug
---
 python/paddle/__init__.py                     |  8 +++--
 python/paddle/fluid/__init__.py               |  4 +--
 python/paddle/fluid/dygraph/__init__.py       |  2 --
 python/paddle/fluid/dygraph/math_op_patch.py  |  8 ++---
 ...tch_methods.py => tensor_patch_methods.py} | 22 ++++++-------
 python/paddle/fluid/framework.py              | 31 -------------------
 .../unittests/test_deprecated_decorator.py    |  2 +-
 python/paddle/framework/__init__.py           |  7 +++--
 8 files changed, 29 insertions(+), 55 deletions(-)
 rename python/paddle/fluid/dygraph/{varbase_patch_methods.py => tensor_patch_methods.py} (98%)

diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py
index ca237df8e53fe..f319ab27c063a 100644
--- a/python/paddle/__init__.py
+++ b/python/paddle/__init__.py
@@ -24,11 +24,15 @@
     )
 
 from .batch import batch  # noqa: F401
+
+# Do the *DUPLICATED* monkey-patch for the tensor object.
+# We need remove the duplicated code here once we fix
+# the illogical implement in the monkey-patch methods later.
 from .framework import monkey_patch_variable
-from .framework import monkey_patch_math_varbase
+from .framework import monkey_patch_math_tensor
 
 monkey_patch_variable()
-monkey_patch_math_varbase()
+monkey_patch_math_tensor()
 
 from .framework import disable_signal_handler  # noqa: F401
 from .framework import get_flags  # noqa: F401
diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py
index c33406dba8e6a..a82febe3ad6d7 100644
--- a/python/paddle/fluid/__init__.py
+++ b/python/paddle/fluid/__init__.py
@@ -79,7 +79,7 @@
 from .compiler import *
 from paddle.fluid.layers.math_op_patch import monkey_patch_variable
 from .dygraph.base import enable_dygraph, disable_dygraph
-from .dygraph.varbase_patch_methods import monkey_patch_varbase
+from .dygraph.tensor_patch_methods import monkey_patch_tensor
 from .core import _cuda_synchronize
 from .trainer_desc import (
     TrainerDesc,
@@ -211,7 +211,7 @@ def remove_flag_if_exists(name):
 # Consider paddle.init(args) or paddle.main(args)
 monkey_patch_variable()
 __bootstrap__()
-monkey_patch_varbase()
+monkey_patch_tensor()
 
 # NOTE(Aurelius84): clean up ExecutorCacheInfo in advance manually.
 atexit.register(core.clear_executor_cache)
diff --git a/python/paddle/fluid/dygraph/__init__.py b/python/paddle/fluid/dygraph/__init__.py
index 36dbf90c52b8b..c40262a45d7c3 100644
--- a/python/paddle/fluid/dygraph/__init__.py
+++ b/python/paddle/fluid/dygraph/__init__.py
@@ -21,8 +21,6 @@
 from . import learning_rate_scheduler
 from .learning_rate_scheduler import *
 
-from .math_op_patch import monkey_patch_math_varbase
-
 __all__ = []
 __all__ += base.__all__
 __all__ += learning_rate_scheduler.__all__
diff --git a/python/paddle/fluid/dygraph/math_op_patch.py b/python/paddle/fluid/dygraph/math_op_patch.py
index 220d849070d18..7b20e626b8e95 100644
--- a/python/paddle/fluid/dygraph/math_op_patch.py
+++ b/python/paddle/fluid/dygraph/math_op_patch.py
@@ -65,7 +65,7 @@
 _already_patch_eager_tensor = False
 
 
-def monkey_patch_math_varbase():
+def monkey_patch_math_tensor():
     """
     Similar to monkey_patch_variable.
     The difference is, in dygraph mode, use auto-generated op functions for better performance.
@@ -248,7 +248,7 @@ def __impl__(self, other_var):
                 # do nothing
                 pass
 
-            # 2. create varbase for scalar
+            # 2. create Tensor for scalar
             lhs_dtype = self.dtype
             other_var_should_be = core.eager.Tensor
             if not isinstance(other_var, other_var_should_be):
@@ -343,7 +343,7 @@ def __impl__(self, other_var):
         __impl__.__name__ = method_name
         return __impl__
 
-    varbase_methods = [
+    tensor_methods = [
         ('__neg__', _neg_),
         ('__float__', _float_),
         ('__long__', _long_),
@@ -498,7 +498,7 @@ def __impl__(self, other_var):
                 setattr(local_tensor, method_name, method_impl)
 
         else:
-            for method in varbase_methods:
+            for method in tensor_methods:
                 method_name = method[0]
                 method_impl = method[1]
                 setattr(local_tensor, method_name, method_impl)
diff --git a/python/paddle/fluid/dygraph/varbase_patch_methods.py b/python/paddle/fluid/dygraph/tensor_patch_methods.py
similarity index 98%
rename from python/paddle/fluid/dygraph/varbase_patch_methods.py
rename to python/paddle/fluid/dygraph/tensor_patch_methods.py
index 5e64cd6bad3cb..882a333b5ebf7 100644
--- a/python/paddle/fluid/dygraph/varbase_patch_methods.py
+++ b/python/paddle/fluid/dygraph/tensor_patch_methods.py
@@ -32,7 +32,7 @@
     in_dygraph_mode,
 )
 from .base import switch_to_static_graph
-from .math_op_patch import monkey_patch_math_varbase
+from .math_op_patch import monkey_patch_math_tensor
 from paddle.fluid.data_feeder import convert_dtype, _PADDLE_DTYPE_2_NUMPY_DTYPE
 import paddle.utils.deprecated as deprecated
 import paddle.profiler as profiler
@@ -86,7 +86,7 @@ def remove(self):
 _already_patch_repr = False
 
 
-def monkey_patch_varbase():
+def monkey_patch_tensor():
     @switch_to_static_graph
     def _to_static_var(self, to_parameter=False, **kwargs):
         """
@@ -110,8 +110,8 @@ def _to_static_var(self, to_parameter=False, **kwargs):
 
                 data = np.ones([3, 1024], dtype='float32')
                 with fluid.dygraph.guard():
-                    var_base = to_variable(data)
-                    static_var = var_base._to_static_var()
+                    tensor = to_variable(data)
+                    static_var = tensor._to_static_var()
 
         """
 
@@ -700,11 +700,11 @@ def __deepcopy__(self, memo):
             raise RuntimeError(
                 "Only Leaf Tensor support the deepcopy at the moment, non-Leaf Tensors contains graph information that does't support deepcopy"
             )
-        new_varbase = core.eager.Tensor()
-        new_varbase.name = self.name + unique_name.generate("_deepcopy")
-        memo[id(self)] = new_varbase
-        new_varbase.copy_(self, True)
-        return new_varbase
+        new_tensor = core.eager.Tensor()
+        new_tensor.name = self.name + unique_name.generate("_deepcopy")
+        memo[id(self)] = new_tensor
+        new_tensor.copy_(self, True)
+        return new_tensor
 
     @property
     def block(self):
@@ -1073,5 +1073,5 @@ def dtype_str(dtype):
         setattr(core.VarDesc.VarType, "__str__", dtype_str)
         _already_patch_repr = True
 
-    # patch math methods for varbase
-    monkey_patch_math_varbase()
+    # patch math methods for tensor
+    monkey_patch_math_tensor()
diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py
index a699a2f0fe720..63ab3a65bb6b9 100644
--- a/python/paddle/fluid/framework.py
+++ b/python/paddle/fluid/framework.py
@@ -112,8 +112,6 @@ def __setattr__(self, name, val):
 _current_device = None
 global_prog_seed = 0
 _current_pipeline_stage = None
-_already_patch_eager_tensor = False
-_already_patch_varbase = False
 _current_cuda_graph_mode = None
 _global_flags_ = core.globals()
 
@@ -182,35 +180,6 @@ def __setattr__(self, name, val):
 # to make sure in most case, we find new dygraph mode first with only one if statement.
 
 
-def _update_monkey_methods():
-    """
-    Update monkey methods of Tensor or eager.Tensor while
-    switching eager mode and legacy mode.
-    """
-    from paddle import _C_ops, _legacy_C_ops
-    from .dygraph.varbase_patch_methods import monkey_patch_varbase
-    from .dygraph import monkey_patch_math_varbase
-
-    global _already_patch_eager_tensor
-    global _already_patch_varbase
-
-    if not _already_patch_eager_tensor:
-        monkey_patch_varbase()
-        monkey_patch_math_varbase()
-
-        _already_patch_eager_tensor = True
-
-    # switch Paddle.Tensor bind type
-    _switch_tensor_bind_type()
-
-
-def _switch_tensor_bind_type():
-    import paddle
-
-    paddle.Tensor = core.eager.Tensor
-    paddle.Tensor.__qualname__ = 'Tensor'
-
-
 def _in_eager_without_dygraph_check():
     return global_var._in_eager_mode_
 
diff --git a/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py b/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py
index 4624897f0168e..1e4bfe50515dd 100755
--- a/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py
+++ b/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py
@@ -118,7 +118,7 @@ def test_tensor_gradient(self):
         with warnings.catch_warnings(record=True) as w:
             grad = x.gradient()
             assert (
-                'API "paddle.fluid.dygraph.varbase_patch_methods.gradient" is '
+                'API "paddle.fluid.dygraph.tensor_patch_methods.gradient" is '
                 'deprecated since 2.1.0'
             ) in str(w[-1].message)
 
diff --git a/python/paddle/framework/__init__.py b/python/paddle/framework/__init__.py
index 2b9449b7c0902..3c6338e345ddb 100755
--- a/python/paddle/framework/__init__.py
+++ b/python/paddle/framework/__init__.py
@@ -44,8 +44,11 @@
 from .io_utils import _unpack_saved_dict
 from .io_utils import _load_program_scope
 
-from ..fluid import monkey_patch_variable
-from ..fluid.dygraph import monkey_patch_math_varbase
+# Do the *DUPLICATED* monkey-patch for the tensor object.
+# We need remove the duplicated code here once we fix
+# the illogical implement in the monkey-patch methods later.
+from ..fluid.layers.math_op_patch import monkey_patch_variable
+from ..fluid.dygraph.math_op_patch import monkey_patch_math_tensor
 from ..fluid.framework import disable_signal_handler  # noqa: F401
 from ..fluid.framework import get_flags  # noqa: F401
 from ..fluid.framework import set_flags  # noqa: F401

From 4695122492eee3cc9e9c585e33429c0f98dbdbb0 Mon Sep 17 00:00:00 2001
From: Difer <707065510@qq.com>
Date: Tue, 25 Apr 2023 11:58:34 +0800
Subject: [PATCH 044/405] =?UTF-8?q?=E3=80=90Hackathon=20No57=E3=80=91add?=
 =?UTF-8?q?=20fp16=20&=20bf16=20for=20max=5Fpool2d=5Fwith=5Findex,=20max?=
 =?UTF-8?q?=5Fpool3d=5Fwith=5Findex=20(#52314)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add fp_bf for pool_max_withidx

* fix some error

* fix error

* codestyle error

* fix masktype

* fix input bf type

* input bf dtype convert error

* back to convert input to bf16 first

* fix convert error

* fix bf16 grad check
---
 paddle/phi/kernels/funcs/pooling.cu           |  24 +-
 paddle/phi/kernels/gpu/pool_grad_kernel.cu    |   8 +-
 paddle/phi/kernels/gpu/pool_kernel.cu         |   8 +-
 .../fluid/tests/unittests/test_pool_max_op.py | 205 +++++++++++++++++-
 4 files changed, 227 insertions(+), 18 deletions(-)

diff --git a/paddle/phi/kernels/funcs/pooling.cu b/paddle/phi/kernels/funcs/pooling.cu
index d0a0416994169..2f89b51815e64 100644
--- a/paddle/phi/kernels/funcs/pooling.cu
+++ b/paddle/phi/kernels/funcs/pooling.cu
@@ -1963,7 +1963,7 @@ __global__ void KernelMaxPool2dWithIdx(const int nthreads,
       wstart = max(wstart, 0);
     }
 
-    T1 ele = -FLT_MAX;
+    T1 ele = static_cast<T1>(-FLT_MAX);
     int max_index = -1;
     for (int h = hstart; h < hend; ++h) {
       for (int w = wstart; w < wend; ++w) {
@@ -2015,7 +2015,7 @@ __global__ void AdaptiveKernelMaxPool2dWithIdx(const int nthreads,
     wstart = AdaptStartIndex(w_offset, input_width, output_width);
     wend = AdaptEndIndex(w_offset, input_width, output_width);
 
-    T1 ele = -FLT_MAX;
+    T1 ele = static_cast<T1>(-FLT_MAX);
     int max_index = -1;
     for (int h = hstart; h < hend; ++h) {
       for (int w = wstart; w < wend; ++w) {
@@ -2089,7 +2089,7 @@ __global__ void KernelMaxPool2DWithIdxGrad(const int nthreads,
       pwend = min((w_offset + padding_width) / stride_width + 1, output_width);
     }
 
-    T1 input_grad_data = 0;
+    T1 input_grad_data = static_cast<T1>(0);
     int input_current_featuremap_idx = h_offset * input_width + w_offset;
     for (int ph = phstart; ph < phend; ++ph) {
       for (int pw = pwstart; pw < pwend; ++pw) {
@@ -2259,6 +2259,14 @@ template class MaxPool2dWithIndexFunctor<phi::GPUContext, float, int>;
 template class MaxPool2dWithIndexGradFunctor<phi::GPUContext, float, int>;
 template class MaxPool2dWithIndexFunctor<phi::GPUContext, double, int>;
 template class MaxPool2dWithIndexGradFunctor<phi::GPUContext, double, int>;
+template class MaxPool2dWithIndexFunctor<phi::GPUContext, dtype::float16, int>;
+template class MaxPool2dWithIndexGradFunctor<phi::GPUContext,
+                                             dtype::float16,
+                                             int>;
+template class MaxPool2dWithIndexFunctor<phi::GPUContext, dtype::bfloat16, int>;
+template class MaxPool2dWithIndexGradFunctor<phi::GPUContext,
+                                             dtype::bfloat16,
+                                             int>;
 
 template <typename T1, typename T2>
 __global__ void KernelMaxPool3DWithIdx(const int ncd,
@@ -2324,7 +2332,7 @@ __global__ void KernelMaxPool3DWithIdx(const int ncd,
         wstart = max(wstart, 0);
       }
 
-      T1 ele = -FLT_MAX;
+      T1 ele = static_cast<T1>(-FLT_MAX);
       int max_index = -1;
       for (int d = dstart; d < dend; ++d) {
         for (int h = hstart; h < hend; ++h) {
@@ -2560,6 +2568,14 @@ template class MaxPool3dWithIndexFunctor<phi::GPUContext, float, int>;
 template class MaxPool3dWithIndexGradFunctor<phi::GPUContext, float, int>;
 template class MaxPool3dWithIndexFunctor<phi::GPUContext, double, int>;
 template class MaxPool3dWithIndexGradFunctor<phi::GPUContext, double, int>;
+template class MaxPool3dWithIndexFunctor<phi::GPUContext, dtype::float16, int>;
+template class MaxPool3dWithIndexGradFunctor<phi::GPUContext,
+                                             dtype::float16,
+                                             int>;
+template class MaxPool3dWithIndexFunctor<phi::GPUContext, dtype::bfloat16, int>;
+template class MaxPool3dWithIndexGradFunctor<phi::GPUContext,
+                                             dtype::bfloat16,
+                                             int>;
 
 }  // namespace funcs
 }  // namespace phi
diff --git a/paddle/phi/kernels/gpu/pool_grad_kernel.cu b/paddle/phi/kernels/gpu/pool_grad_kernel.cu
index e4cfcb23b730e..c625977543558 100644
--- a/paddle/phi/kernels/gpu/pool_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/pool_grad_kernel.cu
@@ -38,7 +38,9 @@ PD_REGISTER_KERNEL(max_pool2d_with_index_grad,
                    ALL_LAYOUT,
                    phi::MaxPool2dWithIndexGradKernel,
                    float,
-                   double) {
+                   double,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {
   kernel->InputAt(1).SetDataType(phi::CppTypeToDataType<int>::Type());
 }
 
@@ -55,6 +57,8 @@ PD_REGISTER_KERNEL(max_pool3d_with_index_grad,
                    ALL_LAYOUT,
                    phi::MaxPool3dWithIndexGradKernel,
                    float,
-                   double) {
+                   double,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {
   kernel->InputAt(1).SetDataType(phi::CppTypeToDataType<int>::Type());
 }
diff --git a/paddle/phi/kernels/gpu/pool_kernel.cu b/paddle/phi/kernels/gpu/pool_kernel.cu
index 65d0ef4bdc916..511cc263bc760 100644
--- a/paddle/phi/kernels/gpu/pool_kernel.cu
+++ b/paddle/phi/kernels/gpu/pool_kernel.cu
@@ -32,7 +32,9 @@ PD_REGISTER_KERNEL(max_pool2d_with_index,
                    ALL_LAYOUT,
                    phi::MaxPool2dWithIndexKernel,
                    float,
-                   double) {
+                   double,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {
   kernel->OutputAt(1).SetDataType(phi::CppTypeToDataType<int>::Type());
 }
 
@@ -49,6 +51,8 @@ PD_REGISTER_KERNEL(max_pool3d_with_index,
                    ALL_LAYOUT,
                    phi::MaxPool3dWithIndexKernel,
                    float,
-                   double) {
+                   double,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {
   kernel->OutputAt(1).SetDataType(phi::CppTypeToDataType<int>::Type());
 }
diff --git a/python/paddle/fluid/tests/unittests/test_pool_max_op.py b/python/paddle/fluid/tests/unittests/test_pool_max_op.py
index d8d61f4fb2904..16d1f356537bc 100644
--- a/python/paddle/fluid/tests/unittests/test_pool_max_op.py
+++ b/python/paddle/fluid/tests/unittests/test_pool_max_op.py
@@ -15,9 +15,16 @@
 import unittest
 
 import numpy as np
-from eager_op_test import OpTest
+from eager_op_test import (
+    OpTest,
+    convert_float_to_uint16,
+    convert_uint16_to_float,
+    get_numeric_gradient,
+)
 
 import paddle
+from paddle.fluid import core
+from paddle.fluid.tests.unittests.testsuite import create_op
 
 
 def adaptive_start_index(index, input_size, output_size):
@@ -149,9 +156,18 @@ def setUp(self):
         self.init_test_case()
         self.init_global()
         self.init_adaptive()
+        self.init_dtype()
+
+        if self.is_bfloat16_op():
+            input = np.random.random(self.shape).astype(np.float32)
+            input = convert_uint16_to_float(
+                convert_float_to_uint16(np.round(input * 100.0, 2))
+            )
+
+        else:
+            input = np.random.random(self.shape).astype(self.dtype)
+            input = np.round(input * 100.0, 2)
 
-        input = np.random.random(self.shape).astype("float64")
-        input = np.round(input * 100.0, 2)
         output, mask = self.pool_forward_naive(
             input,
             self.ksize,
@@ -160,8 +176,11 @@ def setUp(self):
             self.global_pool,
             self.adaptive,
         )
-        output = output.astype("float64")
         mask = mask.astype("int32")
+        if self.is_bfloat16_op():
+            output = output.astype(np.float32)
+        else:
+            output = output.astype(self.dtype)
 
         self.attrs = {
             'strides': self.strides,
@@ -171,8 +190,20 @@ def setUp(self):
             'adaptive': self.adaptive,
         }
 
-        self.inputs = {'X': input}
-        self.outputs = {'Out': output, "Mask": mask}
+        if self.is_bfloat16_op():
+            self.inputs = {'X': convert_float_to_uint16(input)}
+            self.outputs = {
+                'Out': convert_float_to_uint16(output),
+                "Mask": mask,
+            }
+            self.inputs_fp32 = {'X': input}
+
+        else:
+            self.inputs = {'X': input}
+            self.outputs = {'Out': output, "Mask": mask}
+
+    def init_dtype(self):
+        self.dtype = np.float64
 
     def test_check_output(self):
         self.check_output()
@@ -220,9 +251,90 @@ def init_global(self):
         self.global_pool = False
 
 
-# ----------------max_pool2d_with_index----------------
+class TestCastAdaptive3d(TestMaxPoolWithIndex_Op):
+    def init_adaptive(self):
+        self.adaptive = True
 
 
+# ----------------max_pool3d_with_index_fp16----------------
+def create_test_fp16_class(parent):
+    @unittest.skipIf(
+        not core.is_compiled_with_cuda(), "core is not compiled with CUDA"
+    )
+    class TestMaxPool3dFP16(parent):
+        def init_dtype(self):
+            self.dtype = np.float16
+
+        def test_check_output(self):
+            if core.is_compiled_with_cuda():
+                place = core.CUDAPlace(0)
+                if core.is_float16_supported(place):
+                    self.check_output_with_place(place)
+
+        def test_check_grad(self):
+            place = core.CUDAPlace(0)
+            if core.is_float16_supported(place):
+                self.check_grad_with_place(place, {'X'}, ['Out'])
+
+    cls_name = "{}_{}".format(parent.__name__, "FP16OP")
+    TestMaxPool3dFP16.__name__ = cls_name
+    globals()[cls_name] = TestMaxPool3dFP16
+
+
+create_test_fp16_class(TestMaxPoolWithIndex_Op)
+create_test_fp16_class(TestCase1)
+create_test_fp16_class(TestCase2)
+create_test_fp16_class(TestCase3)
+create_test_fp16_class(TestCastAdaptive3d)
+
+
+# ----------------max_pool3d_with_index_bf16----------------
+def create_test_bf16_class(parent):
+    @unittest.skipIf(
+        not core.is_compiled_with_cuda()
+        or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+        "core is not compiled with CUDA and do not support bfloat16",
+    )
+    class TestMaxPool3dBF16(parent):
+        def init_dtype(self):
+            self.dtype = np.uint16
+
+        def get_numeric_grad(self, place, check_name):
+            scope = core.Scope()
+            self._check_grad_helper()
+            op = create_op(
+                scope, self.op_type, self.inputs, self.outputs, self.attrs
+            )
+            return get_numeric_gradient(
+                place, scope, op, self.inputs_fp32, check_name, ['Out']
+            )
+
+        def test_check_output(self):
+            place = core.CUDAPlace(0)
+            if core.is_bfloat16_supported(place):
+                self.check_output_with_place(place)
+
+        def test_check_grad(self):
+            place = core.CUDAPlace(0)
+            numeric_grads = self.get_numeric_grad(place, 'X')
+            if core.is_bfloat16_supported(place):
+                self.check_grad_with_place(
+                    place, {'X'}, ['Out'], user_defined_grads=[numeric_grads]
+                )
+
+    cls_name = "{}_{}".format(parent.__name__, "BF16OP")
+    TestMaxPool3dBF16.__name__ = cls_name
+    globals()[cls_name] = TestMaxPool3dBF16
+
+
+create_test_bf16_class(TestMaxPoolWithIndex_Op)
+create_test_bf16_class(TestCase1)
+create_test_bf16_class(TestCase2)
+create_test_bf16_class(TestCase3)
+create_test_bf16_class(TestCastAdaptive3d)
+
+
+# ----------------max_pool2d_with_index----------------
 def max_pool2d_with_index_wapper(
     x,
     kernel_size=[],
@@ -279,9 +391,82 @@ def init_adaptive(self):
         self.adaptive = True
 
 
-class TestCastAdaptive3d(TestMaxPoolWithIndex_Op):
-    def init_adaptive(self):
-        self.adaptive = True
+# ----------------max_pool2d_with_index_fp16----------------
+def create_test_fp16_class(parent):
+    @unittest.skipIf(
+        not core.is_compiled_with_cuda(), "core is not compiled with CUDA"
+    )
+    class TestMaxPool2dFP16(parent):
+        def init_dtype(self):
+            self.dtype = np.float16
+
+        def test_check_output(self):
+            if core.is_compiled_with_cuda():
+                place = core.CUDAPlace(0)
+                if core.is_float16_supported(place):
+                    self.check_output_with_place(place)
+
+        def test_check_grad(self):
+            place = core.CUDAPlace(0)
+            if core.is_float16_supported(place):
+                self.check_grad_with_place(place, {'X'}, ['Out'])
+
+    cls_name = "{}_{}".format(parent.__name__, "FP16OP")
+    TestMaxPool2dFP16.__name__ = cls_name
+    globals()[cls_name] = TestMaxPool2dFP16
+
+
+create_test_fp16_class(TestCase4)
+create_test_fp16_class(TestCase5)
+create_test_fp16_class(TestCase6)
+create_test_fp16_class(TestCase7)
+create_test_fp16_class(TestCastAdaptive2d)
+
+
+# ----------------max_pool2d_with_index_bf16----------------
+def create_test_bf16_class(parent):
+    @unittest.skipIf(
+        not core.is_compiled_with_cuda()
+        or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+        "core is not compiled with CUDA and do not support bfloat16",
+    )
+    class TestMaxPool2dBF16(parent):
+        def init_dtype(self):
+            self.dtype = np.uint16
+
+        def get_numeric_grad(self, place, check_name):
+            scope = core.Scope()
+            self._check_grad_helper()
+            op = create_op(
+                scope, self.op_type, self.inputs, self.outputs, self.attrs
+            )
+            return get_numeric_gradient(
+                place, scope, op, self.inputs_fp32, check_name, ['Out']
+            )
+
+        def test_check_output(self):
+            place = core.CUDAPlace(0)
+            if core.is_bfloat16_supported(place):
+                self.check_output_with_place(place)
+
+        def test_check_grad(self):
+            place = core.CUDAPlace(0)
+            numeric_grads = self.get_numeric_grad(place, 'X')
+            if core.is_bfloat16_supported(place):
+                self.check_grad_with_place(
+                    place, {'X'}, ['Out'], user_defined_grads=[numeric_grads]
+                )
+
+    cls_name = "{}_{}".format(parent.__name__, "BF16OP")
+    TestMaxPool2dBF16.__name__ = cls_name
+    globals()[cls_name] = TestMaxPool2dBF16
+
+
+create_test_bf16_class(TestCase4)
+create_test_bf16_class(TestCase5)
+create_test_bf16_class(TestCase6)
+create_test_bf16_class(TestCase7)
+create_test_bf16_class(TestCastAdaptive2d)
 
 
 if __name__ == '__main__':

From c7c5635e0752e3cdc4aa4cc50c10294cf154000e Mon Sep 17 00:00:00 2001
From: zhoutianzi666 <39978853+zhoutianzi666@users.noreply.github.com>
Date: Tue, 25 Apr 2023 13:24:10 +0800
Subject: [PATCH 045/405] [Paddle-TRT] The Graph uses OpConverterType for op
 converter (#53214)

* add ```converter_type``` for op converter
---
 paddle/fluid/inference/tensorrt/convert/op_converter.h   | 5 +++--
 .../tensorrt/convert/test_custom_plugin_creater.cc       | 4 ++--
 .../inference/tensorrt/convert/test_op_converter.cc      | 2 +-
 paddle/fluid/inference/tensorrt/op_teller.cc             | 6 +++---
 paddle/fluid/inference/tensorrt/op_teller.h              | 9 ++-------
 .../fluid/operators/tensorrt/tensorrt_engine_op_test.cc  | 7 +++++--
 6 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/convert/op_converter.h b/paddle/fluid/inference/tensorrt/convert/op_converter.h
index ee8cc0c8681c0..87ad887cef383 100644
--- a/paddle/fluid/inference/tensorrt/convert/op_converter.h
+++ b/paddle/fluid/inference/tensorrt/convert/op_converter.h
@@ -56,8 +56,9 @@ class OpConverter {
 
     OpConverter* it{nullptr};
 
-    auto op_converter_type_map = OpTeller::Global().GetOpConverterTypeMap();
-    switch (op_converter_type_map.at(op_desc.Type())) {
+    auto converter_type = static_cast<OpConverterType>(
+        PADDLE_GET_CONST(int, op_desc.GetAttr("converter_type")));
+    switch (converter_type) {
       case OpConverterType::Default:
         if (op_desc.Type().find("elementwise") != std::string::npos) {
           static std::unordered_set<std::string> add_tensor_op_set{
diff --git a/paddle/fluid/inference/tensorrt/convert/test_custom_plugin_creater.cc b/paddle/fluid/inference/tensorrt/convert/test_custom_plugin_creater.cc
index 47c3793ab9f0b..eee4d0c12edbe 100644
--- a/paddle/fluid/inference/tensorrt/convert/test_custom_plugin_creater.cc
+++ b/paddle/fluid/inference/tensorrt/convert/test_custom_plugin_creater.cc
@@ -109,7 +109,7 @@ TEST(CustomPluginCreater, StaticShapePlugin) {
   framework::OpDesc custom_op(*op_desc, nullptr);
   CHECK_EQ((*custom_plugin_tell)(custom_op, false, false), true);
 
-  OpTeller::Global().SetOpConverterType("custom_op",
+  OpTeller::Global().SetOpConverterType(&custom_op,
                                         OpConverterType::CustomPluginCreater);
 
   OpConverter converter;
@@ -196,7 +196,7 @@ TEST(CustomPluginCreater, DynamicShapePlugin) {
   framework::OpDesc custom_op(*op_desc, nullptr);
   CHECK_EQ((*custom_plugin_tell)(custom_op, false, true), true);
 
-  OpTeller::Global().SetOpConverterType("custom_op",
+  OpTeller::Global().SetOpConverterType(&custom_op,
                                         OpConverterType::CustomPluginCreater);
 
   OpConverter converter;
diff --git a/paddle/fluid/inference/tensorrt/convert/test_op_converter.cc b/paddle/fluid/inference/tensorrt/convert/test_op_converter.cc
index ee45b602b3ab9..3a15af255e5bc 100644
--- a/paddle/fluid/inference/tensorrt/convert/test_op_converter.cc
+++ b/paddle/fluid/inference/tensorrt/convert/test_op_converter.cc
@@ -57,7 +57,7 @@ TEST(OpConverter, ConvertBlock) {
   x_tensor->Resize(phi::make_ddim(dim_vec));
   x_tensor->mutable_data<float>(platform::CUDAPlace(0));
 
-  OpTeller::Global().SetOpConverterType("conv2d", OpConverterType::Default);
+  OpTeller::Global().SetOpConverterType(conv2d_op, OpConverterType::Default);
   OpConverter converter;
   converter.ConvertBlock(
       *block->Proto(), {"conv2d-Y"}, scope, engine_.get() /*TensorRTEngine*/);
diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index b17aca9e8cb4d..78e300a8d730d 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -3080,17 +3080,17 @@ bool OpTeller::Tell(const framework::ir::Node* node,
     return false;
   auto& default_teller = GetDefaultTeller();
   if ((*default_teller)(desc, use_no_calib_int8, with_dynamic_shape)) {
-    SetOpConverterType(op_type, OpConverterType::Default);
+    SetOpConverterType(node->Op(), OpConverterType::Default);
     return true;
   }
   auto& generic_plugin_teller = GetGenericPluginTeller();
   if ((*generic_plugin_teller)(desc, use_no_calib_int8, with_dynamic_shape)) {
-    SetOpConverterType(op_type, OpConverterType::GenericPluginCreater);
+    SetOpConverterType(node->Op(), OpConverterType::GenericPluginCreater);
     return true;
   }
   auto& custom_plugin_teller = GetCustomPluginTeller();
   if ((*custom_plugin_teller)(desc, use_no_calib_int8, with_dynamic_shape)) {
-    SetOpConverterType(op_type, OpConverterType::CustomPluginCreater);
+    SetOpConverterType(node->Op(), OpConverterType::CustomPluginCreater);
     return true;
   }
   return false;
diff --git a/paddle/fluid/inference/tensorrt/op_teller.h b/paddle/fluid/inference/tensorrt/op_teller.h
index 2fa3dc361217e..cb879cb5f9f61 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.h
+++ b/paddle/fluid/inference/tensorrt/op_teller.h
@@ -82,12 +82,8 @@ class OpTeller {
 
   std::unique_ptr<Teller>& GetCustomPluginTeller() { return tellers_.at(2); }
 
-  void SetOpConverterType(std::string name, OpConverterType type) {
-    op_converter_type_map_[name] = type;
-  }
-
-  const std::map<std::string, OpConverterType>& GetOpConverterTypeMap() const {
-    return op_converter_type_map_;
+  void SetOpConverterType(framework::OpDesc* op_desc, OpConverterType type) {
+    op_desc->SetAttr("converter_type", static_cast<int>(type));
   }
 
  private:
@@ -95,7 +91,6 @@ class OpTeller {
 
  private:
   std::vector<std::unique_ptr<Teller>> tellers_;
-  std::map<std::string, OpConverterType> op_converter_type_map_;
 };
 
 }  // namespace tensorrt
diff --git a/paddle/fluid/operators/tensorrt/tensorrt_engine_op_test.cc b/paddle/fluid/operators/tensorrt/tensorrt_engine_op_test.cc
index 5a9fa7241e853..3d96361d89f04 100644
--- a/paddle/fluid/operators/tensorrt/tensorrt_engine_op_test.cc
+++ b/paddle/fluid/operators/tensorrt/tensorrt_engine_op_test.cc
@@ -94,6 +94,11 @@ void DynamicShapeTest(bool allow_build_at_runtime) {
       "Out", std::vector<std::string>({"z0"}));  // 2 x 4 x 4 x 4
   elementwise_add1->SetAttr("axis", static_cast<int32_t>(0));
 
+  inference::tensorrt::OpTeller::Global().SetOpConverterType(
+      elementwise_add0, inference::tensorrt::OpConverterType::Default);
+  inference::tensorrt::OpTeller::Global().SetOpConverterType(
+      elementwise_add1, inference::tensorrt::OpConverterType::Default);
+
   // Set inputs' variable shape in BlockDesc
   // the batch size is 2, so the dims of 'x' is {2, 4}
   AddTensorToBlockDesc(block_, "x", std::vector<int64_t>({2, 4, 4, 4}));
@@ -170,8 +175,6 @@ void DynamicShapeTest(bool allow_build_at_runtime) {
 
   // Execute them.
   LOG(INFO) << "engine_op run";
-  inference::tensorrt::OpTeller::Global().SetOpConverterType(
-      "elementwise_add", inference::tensorrt::OpConverterType::Default);
   engine_op->Run(scope, place);
 }
 

From 336bc20b18cc3f30dea0819ee99ddc47109a4282 Mon Sep 17 00:00:00 2001
From: zhangyikun02 <48021248+zhangyk0314@users.noreply.github.com>
Date: Tue, 25 Apr 2023 13:44:29 +0800
Subject: [PATCH 046/405] tile op support 0D input for xpu (#53237)

---
 paddle/phi/kernels/xpu/tile_kernel.cc | 41 +++++++++++-------------
 test/xpu/test_tile_op_xpu.py          | 46 +++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 23 deletions(-)

diff --git a/paddle/phi/kernels/xpu/tile_kernel.cc b/paddle/phi/kernels/xpu/tile_kernel.cc
index 419ff72e640ff..f6bc716a7d58a 100644
--- a/paddle/phi/kernels/xpu/tile_kernel.cc
+++ b/paddle/phi/kernels/xpu/tile_kernel.cc
@@ -31,13 +31,15 @@ void TileKernel(const Context& dev_ctx,
                 DenseTensor* out) {
   using XPUType = typename XPUTypeTrait<T>::Type;
   auto rank = x.dims().size();
-  PADDLE_ENFORCE_GE(
-      rank,
-      1,
-      errors::InvalidArgument(
-          "The rank of the input 'x' for tile op must be a positive "
-          "integer, but the value received is %d.",
-          rank));
+  std::vector<int64_t> repeat_times = repeat_times_arr.GetData();
+  int repeat_times_size = repeat_times.size();
+  rank = std::max(rank, repeat_times_size);
+  PADDLE_ENFORCE_GE(rank,
+                    0,
+                    errors::InvalidArgument(
+                        "The rank of the input 'x' for tile op must be a >=0 "
+                        "integer, but the value received is %d.",
+                        rank));
   PADDLE_ENFORCE_LE(
       rank,
       MAX_RANK_SUPPORTED,
@@ -46,14 +48,12 @@ void TileKernel(const Context& dev_ctx,
           "must be less than or equal to %d, but the value received is %d.",
           MAX_RANK_SUPPORTED,
           rank));
-  std::vector<int64_t> repeat_times = repeat_times_arr.GetData();
-  int repeat_times_size = repeat_times.size();
   PADDLE_ENFORCE_GE(
       repeat_times_size,
-      1,
+      0,
       errors::InvalidArgument(
           "The number of elements of the input 'repeat_times' for tile "
-          "op must be positive, but the value received is %d.",
+          "op must be >=0, but the value received is %d.",
           repeat_times_size));
   PADDLE_ENFORCE_LE(
       repeat_times_size,
@@ -102,20 +102,15 @@ void TileKernel(const Context& dev_ctx,
   dev_ctx.template Alloc<T>(out);
 
   std::vector<int64_t> temp(repeat_times.size(), 1);
-  if (repeat_times == temp) {
+  if (rank == 0 || repeat_times == temp) {
     out->Resize(x.dims());
     dev_ctx.template Alloc<T>(out);
-    if (std::is_same<T, double>::value) {
-      int r = xpu::copy(dev_ctx.x_context(),
-                        reinterpret_cast<const int8_t*>(x.data<double>()),
-                        reinterpret_cast<int8_t*>(out->data<double>()),
-                        8 * x.numel());
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "copy");
-    } else {
-      int r = xpu::copy(
-          dev_ctx.x_context(), x.data<T>(), out->data<T>(), x.numel());
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "copy");
-    }
+    int64_t count = x.numel() * sizeof(T);
+    int r = xpu::copy(dev_ctx.x_context(),
+                      reinterpret_cast<const int8_t*>(x.data<T>()),
+                      reinterpret_cast<int8_t*>(out->data<T>()),
+                      count);
+    PADDLE_ENFORCE_XDNN_SUCCESS(r, "copy");
     return;
   }
 
diff --git a/test/xpu/test_tile_op_xpu.py b/test/xpu/test_tile_op_xpu.py
index dc2b0d7f0edcd..2e661199a0928 100644
--- a/test/xpu/test_tile_op_xpu.py
+++ b/test/xpu/test_tile_op_xpu.py
@@ -90,6 +90,21 @@ def init_data(self):
             self.ori_shape = (2, 4, 5, 7)
             self.repeat_times = (3, 2, 1, 2)
 
+    class TestTileOpRank_ZeroDim1(TestTileOpRank1):
+        def init_data(self):
+            self.ori_shape = []
+            self.repeat_times = []
+
+    class TestTileOpRank_ZeroDim2(TestTileOpRank1):
+        def init_data(self):
+            self.ori_shape = []
+            self.repeat_times = [2]
+
+    class TestTileOpRank_ZeroDim3(TestTileOpRank1):
+        def init_data(self):
+            self.ori_shape = []
+            self.repeat_times = [2, 3]
+
 
 # Situation 2: repeat_times is a list (with tensor)
 class XPUTestTileOpRank1_tensor_attr(XPUOpTestWrapper):
@@ -209,5 +224,36 @@ def test_api(self):
             assert np.array_equal(out_3.numpy(), np.tile(np_x, (2, 3)))
 
 
+class TestTileAPI_ZeroDim(unittest.TestCase):
+    def test_dygraph(self):
+        paddle.disable_static()
+
+        x = paddle.rand([])
+        x.stop_gradient = False
+
+        out = paddle.tile(x, [])
+        out.retain_grads()
+        out.backward()
+        self.assertEqual(out.shape, [])
+        self.assertEqual(x.grad.shape, [])
+        self.assertEqual(out.grad.shape, [])
+
+        out = paddle.tile(x, [3])
+        out.retain_grads()
+        out.backward()
+        self.assertEqual(out.shape, [3])
+        self.assertEqual(x.grad.shape, [])
+        self.assertEqual(out.grad.shape, [3])
+
+        out = paddle.tile(x, [2, 3])
+        out.retain_grads()
+        out.backward()
+        self.assertEqual(out.shape, [2, 3])
+        self.assertEqual(x.grad.shape, [])
+        self.assertEqual(out.grad.shape, [2, 3])
+
+        paddle.enable_static()
+
+
 if __name__ == "__main__":
     unittest.main()

From bddeecd1a8e57d69110fbc0f5896a60bb25baa46 Mon Sep 17 00:00:00 2001
From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com>
Date: Tue, 25 Apr 2023 14:07:45 +0800
Subject: [PATCH 047/405] test,test=develop (#53301)

---
 .../distributed/collective/process_group.h    | 267 +++++++++---------
 1 file changed, 137 insertions(+), 130 deletions(-)

diff --git a/paddle/fluid/distributed/collective/process_group.h b/paddle/fluid/distributed/collective/process_group.h
index df9cfdfbc794f..447fc5d1b3c7b 100644
--- a/paddle/fluid/distributed/collective/process_group.h
+++ b/paddle/fluid/distributed/collective/process_group.h
@@ -59,20 +59,20 @@ class ProcessGroup {
     virtual ~Task() = default;
 
     virtual bool IsCompleted();
-    virtual bool Wait(std::chrono::milliseconds timeout = kWaitTimeout) {
+    virtual bool Wait(std::chrono::milliseconds timeout UNUSED = kWaitTimeout) {
       return false;
     }
     virtual void Synchronize() {}
-    virtual void UpdateWaitChain(const phi::DeviceContext& ctx) {}
+    virtual void UpdateWaitChain(const phi::DeviceContext& ctx UNUSED) {}
     bool IsSync() const { return sync_op_; }
 
     // TODO(sunyilun): methods below will be removed later
     Task(int rank,
-         const std::vector<phi::DenseTensor>& inputs,
+         const std::vector<phi::DenseTensor>& inputs UNUSED,
          CommType comm_type)
         : rank_(rank), comm_type_(comm_type) {}
     Task(int rank,
-         const std::vector<phi::DenseTensor>& inputs,
+         const std::vector<phi::DenseTensor>& inputs UNUSED,
          CommType comm_type,
          bool sync_op)
         : rank_(rank), comm_type_(comm_type), sync_op_(sync_op) {}
@@ -97,14 +97,15 @@ class ProcessGroup {
 
   virtual std::string GetBackendName() const = 0;
 
-  virtual phi::DeviceContext* GetDeviceContext(const Place& place) const {
+  virtual phi::DeviceContext* GetDeviceContext(
+      const Place& place UNUSED) const {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support get device_context.",
         GetBackendName()));
   }
 
-  virtual phi::DeviceContext* GetDeviceContext(const Place& place,
-                                               bool use_calc_stream) const {
+  virtual phi::DeviceContext* GetDeviceContext(
+      const Place& place UNUSED, bool use_calc_stream UNUSED) const {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support get device_context.",
         GetBackendName()));
@@ -112,123 +113,127 @@ class ProcessGroup {
 
   // without stream APIs
   virtual std::shared_ptr<ProcessGroup::Task> AllGather(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      bool sync_op) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      bool sync_op UNUSED) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support all_gather with sync_op flag.",
         GetBackendName()));
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> AllGather(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      int64_t offset,
-      int64_t numel,
-      bool sync_op) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      int64_t offset UNUSED,
+      int64_t numel UNUSED,
+      bool sync_op UNUSED) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support all_gather with sync_op flag.",
         GetBackendName()));
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> AllReduce(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const AllreduceOptions& opts,
-      bool sync_op) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const AllreduceOptions& opts UNUSED,
+      bool sync_op UNUSED) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support all_reduce with sync_op flag.",
         GetBackendName()));
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> AllToAll(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const std::vector<int64_t>& out_size_each_rank,
-      const std::vector<int64_t>& in_size_each_rank,
-      bool sync_op) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const std::vector<int64_t>& out_size_each_rank UNUSED,
+      const std::vector<int64_t>& in_size_each_rank UNUSED,
+      bool sync_op UNUSED) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support all_to_all with sync_op flag.",
         GetBackendName()));
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> Barrier(
-      const BarrierOptions& = BarrierOptions()) {
+      const BarrierOptions& UNUSED = BarrierOptions()) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support barrier.", GetBackendName()));
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> Broadcast(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const BroadcastOptions& opts,
-      bool sync_op) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const BroadcastOptions& opts UNUSED,
+      bool sync_op UNUSED) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support broadcast with sync_op flag",
         GetBackendName()));
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> Reduce(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const ReduceOptions& opts,
-      bool sync_op) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const ReduceOptions& opts UNUSED,
+      bool sync_op UNUSED) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support reduce with sync_op flag.",
         GetBackendName()));
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> ReduceScatter(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const ReduceScatterOptions& opts,
-      bool sync_op) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const ReduceScatterOptions& opts UNUSED,
+      bool sync_op UNUSED) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support reduce_scatter with sync_op flag.",
         GetBackendName()));
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> Scatter(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const ScatterOptions& opts,
-      bool sync_op) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const ScatterOptions& opts UNUSED,
+      bool sync_op UNUSED) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support scatter with sync_op flag.",
         GetBackendName()));
   }
 
-  virtual std::shared_ptr<ProcessGroup::Task> Recv(phi::DenseTensor* tensor,
-                                                   int src_rank,
-                                                   bool sync_op) {
+  virtual std::shared_ptr<ProcessGroup::Task> Recv(phi::DenseTensor* tensor
+                                                       UNUSED,
+                                                   int src_rank UNUSED,
+                                                   bool sync_op UNUSED) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support recv with sync_op flag.",
         GetBackendName()));
   }
 
-  virtual std::shared_ptr<ProcessGroup::Task> Recv(phi::DenseTensor* tensor,
-                                                   int src_rank,
-                                                   int64_t offset,
-                                                   int64_t numel,
-                                                   bool sync_op) {
+  virtual std::shared_ptr<ProcessGroup::Task> Recv(phi::DenseTensor* tensor
+                                                       UNUSED,
+                                                   int src_rank UNUSED,
+                                                   int64_t offset UNUSED,
+                                                   int64_t numel UNUSED,
+                                                   bool sync_op UNUSED) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support recv with sync_op flag.",
         GetBackendName()));
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> Send(
-      const phi::DenseTensor& tensor, int dst_rank, bool sync_op) {
+      const phi::DenseTensor& tensor UNUSED,
+      int dst_rank UNUSED,
+      bool sync_op UNUSED) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support send with sync_op flag.",
         GetBackendName()));
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> Send(
-      const phi::DenseTensor& tensor,
-      int dst_rank,
-      int64_t offset,
-      int64_t numel,
-      bool sync_op) {
+      const phi::DenseTensor& tensor UNUSED,
+      int dst_rank UNUSED,
+      int64_t offset UNUSED,
+      int64_t numel UNUSED,
+      bool sync_op UNUSED) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support send with sync_op flag.",
         GetBackendName()));
@@ -236,10 +241,10 @@ class ProcessGroup {
 
   // stream APIs
   virtual std::shared_ptr<ProcessGroup::Task> AllGather(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      bool sync_op,
-      bool use_calc_stream) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("ProcessGroup%s does not support all_gather "
                                    "with sync_op and use_calc_stream flag.",
@@ -247,12 +252,12 @@ class ProcessGroup {
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> AllGather(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      int64_t offset,
-      int64_t numel,
-      bool sync_op,
-      bool use_calc_stream) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      int64_t offset UNUSED,
+      int64_t numel UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("ProcessGroup%s does not support all_gather "
                                    "with sync_op and use_calc_stream flag.",
@@ -260,11 +265,11 @@ class ProcessGroup {
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> AllReduce(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const AllreduceOptions& opts,
-      bool sync_op,
-      bool use_calc_stream) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const AllreduceOptions& opts UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("ProcessGroup%s does not support all_reduce "
                                    "with sync_op and use_calc_stream flag.",
@@ -272,12 +277,12 @@ class ProcessGroup {
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> AllToAll(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const std::vector<int64_t>& out_size_each_rank,
-      const std::vector<int64_t>& in_size_each_rank,
-      bool sync_op,
-      bool use_calc_stream) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const std::vector<int64_t>& out_size_each_rank UNUSED,
+      const std::vector<int64_t>& in_size_each_rank UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("ProcessGroup%s does not support all_to_all "
                                    "with sync_op and use_calc_stream flag.",
@@ -285,11 +290,11 @@ class ProcessGroup {
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> Broadcast(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const BroadcastOptions& opts,
-      bool sync_op,
-      bool use_calc_stream) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const BroadcastOptions& opts UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("ProcessGroup%s does not support broadcast "
                                    "with sync_op and use_calc_stream flag.",
@@ -297,11 +302,11 @@ class ProcessGroup {
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> Reduce(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const ReduceOptions& opts,
-      bool sync_op,
-      bool use_calc_stream) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const ReduceOptions& opts UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("ProcessGroup%s does not support reduce "
                                    "with sync_op and use_calc_stream flag.",
@@ -309,11 +314,11 @@ class ProcessGroup {
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> ReduceScatter(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const ReduceScatterOptions& opts,
-      bool sync_op,
-      bool use_calc_stream) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const ReduceScatterOptions& opts UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroup%s does not support reduce_scatter "
         "with sync_op and use_calc_stream flag.",
@@ -321,11 +326,11 @@ class ProcessGroup {
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> Scatter(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const ScatterOptions& opts,
-      bool sync_op,
-      bool use_calc_stream) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const ScatterOptions& opts UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("ProcessGroup%s does not support scatter "
                                    "with sync_op and use_calc_stream flag.",
@@ -333,11 +338,11 @@ class ProcessGroup {
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> Gather(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const GatherOptions& opts,
-      bool sync_op,
-      bool use_calc_stream) {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const GatherOptions& opts UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("ProcessGroup%s does not support gather "
                                    "with sync_op and use_calc_stream flag.",
@@ -345,33 +350,35 @@ class ProcessGroup {
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> Gather(
-      std::vector<phi::DenseTensor>* gather_tensors_ptr,
-      const phi::DenseTensor& in_tensor,
-      const GatherOptions& opts,
-      bool sync_op,
-      bool use_calc_stream) {
+      std::vector<phi::DenseTensor>* gather_tensors_ptr UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const GatherOptions& opts UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("ProcessGroup%s does not support gather "
                                    "with sync_op and use_calc_stream flag.",
                                    GetBackendName()));
   }
 
-  virtual std::shared_ptr<ProcessGroup::Task> Recv(phi::DenseTensor* tensor,
-                                                   int src_rank,
-                                                   bool sync_op,
-                                                   bool use_calc_stream) {
+  virtual std::shared_ptr<ProcessGroup::Task> Recv(
+      phi::DenseTensor* tensor UNUSED,
+      int src_rank UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("ProcessGroup%s does not support recv with "
                                    "sync_op and use_calc_stream flag.",
                                    GetBackendName()));
   }
 
-  virtual std::shared_ptr<ProcessGroup::Task> Recv(phi::DenseTensor* tensor,
-                                                   int src_rank,
-                                                   int64_t offset,
-                                                   int64_t numel,
-                                                   bool sync_op,
-                                                   bool use_calc_stream) {
+  virtual std::shared_ptr<ProcessGroup::Task> Recv(
+      phi::DenseTensor* tensor UNUSED,
+      int src_rank UNUSED,
+      int64_t offset UNUSED,
+      int64_t numel UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("ProcessGroup%s does not support recv "
                                    "with sync_op and use_calc_stream flag.",
@@ -379,10 +386,10 @@ class ProcessGroup {
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> Send(
-      const phi::DenseTensor& tensor,
-      int dst_rank,
-      bool sync_op,
-      bool use_calc_stream) {
+      const phi::DenseTensor& tensor UNUSED,
+      int dst_rank UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("ProcessGroup%s does not support send "
                                    "with sync_op and use_calc_stream flag.",
@@ -390,12 +397,12 @@ class ProcessGroup {
   }
 
   virtual std::shared_ptr<ProcessGroup::Task> Send(
-      const phi::DenseTensor& tensor,
-      int dst_rank,
-      int64_t offset,
-      int64_t numel,
-      bool sync_op,
-      bool use_calc_stream) {
+      const phi::DenseTensor& tensor UNUSED,
+      int dst_rank UNUSED,
+      int64_t offset UNUSED,
+      int64_t numel UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("ProcessGroup%s does not support send "
                                    "with sync_op and use_calc_stream flag.",
@@ -407,7 +414,7 @@ class ProcessGroup {
   virtual std::shared_ptr<ProcessGroup::Task> AllReduce(
       std::vector<phi::DenseTensor>& /* input tensors */,   // NOLINT
       std::vector<phi::DenseTensor>& /* output tensors */,  // NOLINT
-      const AllreduceOptions& = AllreduceOptions()) {
+      const AllreduceOptions& UNUSED = AllreduceOptions()) {
     PADDLE_THROW(phi::errors::InvalidArgument(
         "ProcessGroup%s does not support allreduce", GetBackendName()));
   }
@@ -415,7 +422,7 @@ class ProcessGroup {
   virtual std::shared_ptr<ProcessGroup::Task> AllReduce(
       std::vector<phi::DenseTensor>& /* input tensors */,   // NOLINT
       std::vector<phi::DenseTensor>& /* output tensors */,  // NOLINT
-      const AllreduceOptions&,
+      const AllreduceOptions& UNUSED,
       bool) {
     PADDLE_THROW(phi::errors::InvalidArgument(
         "ProcessGroup%s does not support allreduce with sync_op flag",
@@ -426,7 +433,7 @@ class ProcessGroup {
   virtual std::shared_ptr<ProcessGroup::Task> Broadcast(
       std::vector<phi::DenseTensor>& /* input tensors */,   // NOLINT
       std::vector<phi::DenseTensor>& /* output tensors */,  // NOLINT
-      const BroadcastOptions& = BroadcastOptions()) {
+      const BroadcastOptions& UNUSED = BroadcastOptions()) {
     PADDLE_THROW(phi::errors::InvalidArgument(
         "ProcessGroup%s does not support broadcast", GetBackendName()));
   }
@@ -434,7 +441,7 @@ class ProcessGroup {
   virtual std::shared_ptr<ProcessGroup::Task> Broadcast(
       std::vector<phi::DenseTensor>& /* input tensors */,   // NOLINT
       std::vector<phi::DenseTensor>& /* output tensors */,  // NOLINT
-      const BroadcastOptions&,
+      const BroadcastOptions& UNUSED,
       bool) {
     PADDLE_THROW(phi::errors::InvalidArgument(
         "ProcessGroup%s does not support broadcast with sync_op flag",

From ed45ecc626c07d50c0a1128b92797a4247d69aa9 Mon Sep 17 00:00:00 2001
From: qizhaoaoe <10208099+qizhaoaoe@users.noreply.github.com>
Date: Tue, 25 Apr 2023 14:25:03 +0800
Subject: [PATCH 048/405] [fluid clean] remove Print. (#51778)

* fluid clean: remove print/switch from fluid to static

* remove Switch in static.__init__

* fix conflicts.

* replace Switch by case.

* fix piecewise_lr decay.

* fix typo

* fix conflicts.

* fix lr dtype

* keep Switch in paddle.static.nn.control_flow and fix piecewise_lr.

* fix conflicts.

* keep Switch in the fluid.

* fix Switch doc

* fix example in Switch doc

* fix Switch doc.

* fix static/__init__.
---
 python/paddle/fluid/layers/control_flow.py    | 100 +-----------------
 .../fluid/layers/learning_rate_scheduler.py   |  37 ++++---
 python/paddle/fluid/optimizer.py              |  69 ++++++------
 .../dist_fleet_heter_pipeline_ctr.py          |   2 +-
 .../fluid/tests/unittests/test_switch.py      |  34 +++---
 .../paddle/jit/dy2static/convert_operators.py |   4 +-
 python/paddle/nn/layer/rnn.py                 |   2 -
 python/paddle/reader/decorator.py             |   3 +-
 python/paddle/static/__init__.py              |   2 +-
 python/paddle/static/nn/control_flow.py       | 100 +++++++++++++++++-
 .../test_multi_precision_fp16_train.py        |  10 +-
 test/ipu/test_print_op_ipu.py                 |   2 +-
 12 files changed, 185 insertions(+), 180 deletions(-)

diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py
index a0ad94df79d67..6d402df9f3cd4 100755
--- a/python/paddle/fluid/layers/control_flow.py
+++ b/python/paddle/fluid/layers/control_flow.py
@@ -50,7 +50,6 @@
 __all__ = [
     'Switch',
     'StaticRNN',
-    'Print',
     'while_loop',
 ]
 
@@ -141,104 +140,6 @@ def select_input(inputs, mask):
     return out
 
 
-@static_only
-def Print(
-    input,
-    first_n=-1,
-    message=None,
-    summarize=20,
-    print_tensor_name=True,
-    print_tensor_type=True,
-    print_tensor_shape=True,
-    print_tensor_layout=True,
-    print_tensor_lod=True,
-    print_phase='both',
-):
-    '''
-    :api_attr: Static Graph
-
-    **Print operator**
-
-    This creates a print op that will print when a tensor is accessed.
-
-    Wraps the tensor passed in so that whenever that a tensor is accessed,
-    the message `message` is printed, along with the current value of the
-    tensor `t`.
-
-    Args:
-        input (Tensor): A Tensor to print.
-        first_n (int, optional): Only log `first_n` number of times. Default: -1.
-        message (str, optional): A string message to print as a prefix. Default: None.
-        summarize (int, optional): Number of elements in the tensor to be print. If
-                it's value is -1, then all elements in the tensor will be print.
-        print_tensor_name (bool, optional): Print the tensor name. Default: True.
-        print_tensor_type (bool, optional): Print the tensor type. Defaultt: True.
-        print_tensor_shape (bool, optional): Print the tensor shape. Default: True.
-        print_tensor_layout (bool, optional): Print the tensor layout. Default: True.
-        print_tensor_lod (bool, optional): Print the tensor lod. Default: True.
-        print_phase (str, optional): Which phase to displace, including 'forward',
-                'backward' and 'both'. Default: 'both'. If set to 'backward', will
-                only print the gradients of input tensor; If set to 'both', will
-                both print the input tensor itself and the gradients of input tensor.
-
-    Returns:
-        Tensor: Output tensor.
-
-    NOTES:
-        The input and output are two different Tensor, and in the
-        following process, you should use the output Tensor but not the input,
-        otherwise, the print layer doesn't have backward.
-
-    Examples:
-        .. code-block:: python
-
-           import paddle
-
-           paddle.enable_static()
-
-           x = paddle.full(shape=[2, 3], fill_value=3, dtype='int64')
-           out = paddle.static.Print(x, message="The content of input layer:")
-
-           main_program = paddle.static.default_main_program()
-           exe = paddle.static.Executor(place=paddle.CPUPlace())
-           res = exe.run(main_program, fetch_list=[out])
-           # Variable: fill_constant_1.tmp_0
-           #   - message: The content of input layer:
-           #   - lod: {}
-           #   - place: CPUPlace
-           #   - shape: [2, 3]
-           #   - layout: NCHW
-           #   - dtype: long
-           #   - data: [3 3 3 3 3 3]
-    '''
-    check_variable_and_dtype(
-        input,
-        'input',
-        ['float32', 'float64', 'int32', 'int64', 'bool'],
-        'fluid.layers.Print',
-    )
-
-    helper = LayerHelper('print' + "_" + input.name, **locals())
-    output = helper.create_variable_for_type_inference(input.dtype)
-    helper.append_op(
-        type='print',
-        inputs={'In': input},
-        outputs={'Out': output},
-        attrs={
-            'first_n': first_n,
-            'summarize': summarize,
-            'message': message or "",
-            'print_tensor_name': print_tensor_name,
-            'print_tensor_type': print_tensor_type,
-            'print_tensor_shape': print_tensor_shape,
-            'print_tensor_layout': print_tensor_layout,
-            'print_tensor_lod': print_tensor_lod,
-            'print_phase': print_phase.upper(),
-        },
-    )
-    return output
-
-
 # (TODO: Mine) There exists dependency. It will be removed later.
 class BlockGuard:
     """
@@ -1512,6 +1413,7 @@ def map_fn(n1, n2, name, order):
     return nest1_out, nest2_out
 
 
+# TODO: It will be deleted later.
 class Switch:
     """
     :api_attr: Static Graph
diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py
index 6d86fa9448c17..65fe1d1e77b35 100644
--- a/python/paddle/fluid/layers/learning_rate_scheduler.py
+++ b/python/paddle/fluid/layers/learning_rate_scheduler.py
@@ -359,9 +359,11 @@ def polynomial_decay(
                     shape=[1], dtype='float32', value=1.0
                 )
 
-                with control_flow.Switch() as switch:
-                    with switch.case(global_step == zero_var):
-                        paddle.assign(one_var, output=div_res)
+                div_val = paddle.static.nn.cond(
+                    global_step == zero_var, lambda: one_var, lambda: div_res
+                )
+                paddle.assign(div_val, output=div_res)
+
                 decay_steps = decay_steps * div_res
             else:
                 decay_steps_var = paddle.tensor.fill_constant(
@@ -432,7 +434,7 @@ def piecewise_decay(boundaries, values):
                 persistable=True,
                 name="learning_rate",
             )
-
+            # TODO: fluid.layers.control_flow.Switch should be replaced by paddle.static.nn.case(or cond) if possible
             with control_flow.Switch() as switch:
                 for i in range(len(boundaries)):
                     boundary_val = paddle.tensor.fill_constant(
@@ -455,7 +457,6 @@ def piecewise_decay(boundaries, values):
                         value=float(values[len(values) - 1]),
                         out=lr,
                     )
-
             return lr
 
 
@@ -589,17 +590,19 @@ def linear_lr_warmup(learning_rate, warmup_steps, start_lr, end_lr):
             )
 
             global_step = _decay_step_counter()
-
-            with control_flow.Switch() as switch:
-                with switch.case(global_step < warmup_steps):
-                    decayed_lr = start_lr + linear_step * (
-                        global_step / float(warmup_steps)
+            if not isinstance(learning_rate, Variable):
+                learning_rate = paddle.tensor.fill_constant(
+                    shape=[1], dtype=dtype, value=float(learning_rate)
+                )
+            lr_val = paddle.static.nn.case(
+                pred_fn_pairs=[
+                    (
+                        global_step < warmup_steps,
+                        lambda: start_lr
+                        + linear_step * (global_step / float(warmup_steps)),
                     )
-                    paddle.assign(decayed_lr, lr)
-                with switch.default():
-                    if not isinstance(learning_rate, Variable):
-                        learning_rate = paddle.tensor.fill_constant(
-                            shape=[1], dtype=dtype, value=float(learning_rate)
-                        )
-                    paddle.assign(learning_rate, lr)
+                ],
+                default=lambda: learning_rate,
+            )
+            paddle.assign(lr_val, lr)
             return lr
diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py
index 1dca88d61e327..f6bd5dbd37c33 100755
--- a/python/paddle/fluid/optimizer.py
+++ b/python/paddle/fluid/optimizer.py
@@ -4373,12 +4373,12 @@ def __init__(self, decay=0.999, thres_steps=None, name=None):
                 ema = block._clone_variable(self._ema_vars[param.name])
                 paddle.assign(param, output=tmp)
                 # bias correction
-                with layers.control_flow.Switch() as switch:
-                    with switch.case(global_step > 0):
-                        paddle.assign(ema / (1.0 - decay_pow), output=param)
-                    with switch.default():
-                        paddle.assign(ema, output=param)
-
+                param_val = paddle.static.nn.cond(
+                    global_step > 0,
+                    lambda: ema / (1.0 - decay_pow),
+                    lambda: ema,
+                )
+                paddle.assign(param_val, output=param)
         self.restore_program = Program()
         block = self.restore_program.global_block()
         with program_guard(main_program=self.restore_program):
@@ -4399,13 +4399,12 @@ def _get_ema_decay(self):
 
             if self._thres_steps is not None:
                 decay_t = (self._thres_steps + 1.0) / (self._thres_steps + 10.0)
-                with layers.control_flow.Switch() as switch:
-                    with switch.case(decay_t < self._decay):
-                        paddle.assign(decay_t, decay_var)
-                    with switch.default():
-                        paddle.assign(
-                            np.array([self._decay], dtype=np.float32), decay_var
-                        )
+                decay_val = paddle.static.nn.cond(
+                    decay_t < self._decay,
+                    lambda: decay_t,
+                    lambda: np.array([self._decay], dtype=np.float32),
+                )
+                paddle.assign(decay_val, decay_var)
         return decay_var
 
     def _get_decay_pow(self, block):
@@ -7408,26 +7407,30 @@ def minimize(self, loss, startup_program=None):
             )
 
             mod = paddle.remainder(step, k)
-            with layers.control_flow.Switch() as switch:
-                with switch.case(step == one_var):
-                    for param_name in params:
-                        fast_var = main_block.var(param_name)
-                        slow_var = param_to_slow[param_name]
-                        paddle.assign(fast_var, output=slow_var)
-                with switch.case(mod == zero_var):
-                    for param_name in params:
-                        fast_var = main_block.var(param_name)
-                        slow_var = param_to_slow[param_name]
-                        tmp_var = paddle.add(
-                            paddle.multiply(fast_var, alpha),
-                            paddle.multiply(
-                                slow_var, paddle.subtract(one_var, alpha)
-                            ),
-                        )
-                        paddle.assign(tmp_var, output=slow_var)
-                        paddle.assign(tmp_var, output=fast_var)
-                with switch.default():
-                    pass
+            for param_name in params:
+                fast_var = main_block.var(param_name)
+                slow_var = param_to_slow[param_name]
+                tmp_var = paddle.add(
+                    paddle.multiply(fast_var, alpha),
+                    paddle.multiply(slow_var, paddle.subtract(one_var, alpha)),
+                )
+                slow_val = paddle.static.nn.case(
+                    [
+                        (step == one_var, lambda: fast_var),
+                        (mod == zero_var, lambda: tmp_var),
+                    ],
+                    default=lambda: slow_var,
+                )
+                paddle.assign(slow_val, slow_var)
+
+                fast_val = paddle.static.nn.case(
+                    [
+                        (mod == zero_var, lambda: tmp_var),
+                    ],
+                    default=lambda: fast_var,
+                )
+                paddle.assign(fast_val, fast_var)
+
         return mini_out
 
 
diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_heter_pipeline_ctr.py b/python/paddle/fluid/tests/unittests/dist_fleet_heter_pipeline_ctr.py
index 5dd23f1352559..a5010e275aa8a 100644
--- a/python/paddle/fluid/tests/unittests/dist_fleet_heter_pipeline_ctr.py
+++ b/python/paddle/fluid/tests/unittests/dist_fleet_heter_pipeline_ctr.py
@@ -126,7 +126,7 @@ def net(self, args, batch_size=4, lr=0.01):
                 input=predict, label=label, reduction='none', use_softmax=False
             )
             avg_cost = paddle.mean(x=cost)
-            fluid.layers.Print(avg_cost, message="avg_cost")
+            paddle.static.Print(avg_cost, message="avg_cost")
 
         self.feeds = datas
         self.train_file_path = ["fake1", "fake2"]
diff --git a/python/paddle/fluid/tests/unittests/test_switch.py b/python/paddle/fluid/tests/unittests/test_switch.py
index d5d118867b1a5..428e5537f8ba6 100644
--- a/python/paddle/fluid/tests/unittests/test_switch.py
+++ b/python/paddle/fluid/tests/unittests/test_switch.py
@@ -15,7 +15,7 @@
 import unittest
 
 import paddle
-from paddle.fluid import core, framework, layers
+from paddle.fluid import core, framework
 from paddle.fluid.executor import Executor
 from paddle.fluid.framework import default_startup_program
 
@@ -40,15 +40,15 @@ def check_switch(self, value):
             shape=[1], value=-1.0, dtype='float32', persistable=True
         )
 
-        with layers.Switch() as switch:
-            with switch.case(paddle.less_than(x, zero_var)):
-                paddle.assign(zero_var, result)
-            with switch.case(paddle.less_than(x, one_var)):
-                paddle.assign(one_var, result)
-            with switch.case(paddle.less_than(x, two_var)):
-                paddle.assign(two_var, result)
-            with switch.default():
-                paddle.assign(three_var, result)
+        res = paddle.static.nn.case(
+            pred_fn_pairs=[
+                (paddle.less_than(x, zero_var), lambda: zero_var),
+                (paddle.less_than(x, one_var), lambda: one_var),
+                (paddle.less_than(x, two_var), lambda: two_var),
+            ],
+            default=lambda: three_var,
+        )
+        paddle.assign(res, result)
 
         cpu = core.CPUPlace()
         exe = Executor(cpu)
@@ -85,17 +85,19 @@ def test_error(self):
 
             # 1. The type of 'condition' in case must be Variable.
             def test_condition_type():
-                with layers.Switch() as switch:
-                    with switch.case(1):
-                        paddle.assign(zero_var, result)
+                res = paddle.static.nn.case(
+                    [(1, lambda: zero_var)], default=lambda: result
+                )
+                paddle.assign(res, result)
 
             self.assertRaises(TypeError, test_condition_type)
 
             # 2. The dtype of 'condition' in case must be 'bool'.
             def test_condition_dtype():
-                with layers.Switch() as switch:
-                    with switch.case(cond):
-                        paddle.assign(zero_var, result)
+                res = paddle.static.nn.case(
+                    [cond, lambda: zero_var], default=lambda: result
+                )
+                paddle.assign(res, result)
 
             self.assertRaises(TypeError, test_condition_dtype)
 
diff --git a/python/paddle/jit/dy2static/convert_operators.py b/python/paddle/jit/dy2static/convert_operators.py
index ad9abcc9849ef..52d6b7cb854d9 100644
--- a/python/paddle/jit/dy2static/convert_operators.py
+++ b/python/paddle/jit/dy2static/convert_operators.py
@@ -21,7 +21,7 @@
     in_declarative_mode,
 )
 from paddle.fluid.framework import Variable, core
-from paddle.fluid.layers import Print, control_flow
+from paddle.fluid.layers import control_flow
 from paddle.fluid.layers.control_flow import while_loop
 
 from .utils import (
@@ -749,7 +749,7 @@ def convert_print(*objects, sep=' ', end='\n', file=None, flush=False):
     """
     for obj in objects:
         if isinstance(obj, Variable):
-            Print(obj)
+            paddle.static.Print(obj)
     print(*objects, sep=sep, end=end, file=file, flush=flush)
 
 
diff --git a/python/paddle/nn/layer/rnn.py b/python/paddle/nn/layer/rnn.py
index cc8ab648b8895..2a0c9157a7a4a 100644
--- a/python/paddle/nn/layer/rnn.py
+++ b/python/paddle/nn/layer/rnn.py
@@ -299,13 +299,11 @@ def _switch_grad(x, stop=False):
         pre_state = paddle.utils.map_structure(
             lambda x: paddle.tensor.array_read(x, start_i), init_array
         )
-        # pre_state = paddle.fluid.layers.Print( pre_state, message="pre")
         outputs, new_states = cell(step_in, pre_state, **kwargs)
         assert isinstance(outputs, paddle.fluid.framework.Variable)
         paddle.utils.assert_same_structure(new_states, pre_state)
         if sequence_length:
             step_mask = paddle.unsqueeze(mask[start_i], 1)
-            # paddle.fluid.layers.Print( step_mask, message="mask")
             # new_states = map_structure(
             #     partial(_maybe_copy, step_mask=step_mask),
             #     pre_state, new_states
diff --git a/python/paddle/reader/decorator.py b/python/paddle/reader/decorator.py
index e5c47ebdb3432..bd40c4553e89d 100644
--- a/python/paddle/reader/decorator.py
+++ b/python/paddle/reader/decorator.py
@@ -553,8 +553,9 @@ def _impl():
             with fluid.program_guard(fluid.Program(), fluid.Program()):
                 place = fluid.CPUPlace()
                 # the 1st 2 is batch size
+
                 image = paddle.static.data(name='image', dtype='int64', shape=[2, 1, 2])
-                fluid.layers.Print(image)
+                paddle.static.Print(image)
                 # print detailed tensor info of image variable
 
                 reader = fluid.io.PyReader(feed_list=[image], capacity=2)
diff --git a/python/paddle/static/__init__.py b/python/paddle/static/__init__.py
index f63971b966a74..084579a58e591 100644
--- a/python/paddle/static/__init__.py
+++ b/python/paddle/static/__init__.py
@@ -65,7 +65,7 @@
 from ..fluid.framework import Parameter  # noqa: F401
 from ..fluid.framework import ipu_shard_guard  # noqa: F401
 from ..fluid.framework import set_ipu_shard  # noqa: F401
-from ..fluid.layers.control_flow import Print  # noqa: F401
+from .nn.control_flow import Print  # noqa: F401
 from ..fluid.param_attr import WeightNormParamAttr  # noqa: F401
 from ..fluid.optimizer import Optimizer  # noqa: F401
 from ..fluid.optimizer import Adam  # noqa: F401
diff --git a/python/paddle/static/nn/control_flow.py b/python/paddle/static/nn/control_flow.py
index c5d5265477541..bc5f1d2d5d6f5 100644
--- a/python/paddle/static/nn/control_flow.py
+++ b/python/paddle/static/nn/control_flow.py
@@ -24,7 +24,7 @@
     convert_dtype,
 )
 from paddle.fluid import core
-from paddle.fluid.framework import Operator, Program, Variable
+from paddle.fluid.framework import Operator, Program, Variable, static_only
 
 # Temporary solution, it will be deleted later
 from paddle.fluid.layers.control_flow import ConditionalBlock, select_input
@@ -1329,3 +1329,101 @@ def map_fn(x):
     nest1_out = pack_sequence_as(nest1, list(map(map_fn, flatten(nest1))))
     nest2_out = pack_sequence_as(nest2, list(map(map_fn, flatten(nest2))))
     return nest1_out, nest2_out
+
+
+@static_only
+def Print(
+    input,
+    first_n=-1,
+    message=None,
+    summarize=20,
+    print_tensor_name=True,
+    print_tensor_type=True,
+    print_tensor_shape=True,
+    print_tensor_layout=True,
+    print_tensor_lod=True,
+    print_phase='both',
+):
+    '''
+    :api_attr: Static Graph
+
+    **Print operator**
+
+    This creates a print op that will print when a tensor is accessed.
+
+    Wraps the tensor passed in so that whenever that a tensor is accessed,
+    the message `message` is printed, along with the current value of the
+    tensor `t`.
+
+    Args:
+        input (Tensor): A Tensor to print.
+        first_n (int, optional): Only log `first_n` number of times. Default: -1.
+        message (str, optional): A string message to print as a prefix. Default: None.
+        summarize (int, optional): Number of elements in the tensor to be print. If
+                it's value is -1, then all elements in the tensor will be print.
+        print_tensor_name (bool, optional): Print the tensor name. Default: True.
+        print_tensor_type (bool, optional): Print the tensor type. Defaultt: True.
+        print_tensor_shape (bool, optional): Print the tensor shape. Default: True.
+        print_tensor_layout (bool, optional): Print the tensor layout. Default: True.
+        print_tensor_lod (bool, optional): Print the tensor lod. Default: True.
+        print_phase (str, optional): Which phase to displace, including 'forward',
+                'backward' and 'both'. Default: 'both'. If set to 'backward', will
+                only print the gradients of input tensor; If set to 'both', will
+                both print the input tensor itself and the gradients of input tensor.
+
+    Returns:
+        Tensor: Output tensor.
+
+    NOTES:
+        The input and output are two different Tensor, and in the
+        following process, you should use the output Tensor but not the input,
+        otherwise, the print layer doesn't have backward.
+
+    Examples:
+        .. code-block:: python
+
+           import paddle
+
+           paddle.enable_static()
+
+           x = paddle.full(shape=[2, 3], fill_value=3, dtype='int64')
+           out = paddle.static.Print(x, message="The content of input layer:")
+
+           main_program = paddle.static.default_main_program()
+           exe = paddle.static.Executor(place=paddle.CPUPlace())
+           res = exe.run(main_program, fetch_list=[out])
+           # Variable: fill_constant_1.tmp_0
+           #   - message: The content of input layer:
+           #   - lod: {}
+           #   - place: CPUPlace
+           #   - shape: [2, 3]
+           #   - layout: NCHW
+           #   - dtype: long
+           #   - data: [3 3 3 3 3 3]
+    '''
+    check_variable_and_dtype(
+        input,
+        'input',
+        ['float32', 'float64', 'int32', 'int64', 'bool'],
+        'paddle.static.Print',
+    )
+
+    helper = LayerHelper('print' + "_" + input.name, **locals())
+    output = helper.create_variable_for_type_inference(input.dtype)
+    helper.append_op(
+        type='print',
+        inputs={'In': input},
+        outputs={'Out': output},
+        attrs={
+            'first_n': first_n,
+            'summarize': summarize,
+            'message': message or "",
+            'print_tensor_name': print_tensor_name,
+            'print_tensor_type': print_tensor_type,
+            'print_tensor_shape': print_tensor_shape,
+            'print_tensor_layout': print_tensor_layout,
+            'print_tensor_lod': print_tensor_lod,
+            'print_phase': print_phase.upper(),
+        },
+    )
+    return output
diff --git a/test/contrib/test_multi_precision_fp16_train.py b/test/contrib/test_multi_precision_fp16_train.py
index a364d2161ebe4..218cfcd542da4 100644
--- a/test/contrib/test_multi_precision_fp16_train.py
+++ b/test/contrib/test_multi_precision_fp16_train.py
@@ -295,12 +295,10 @@ def decorate_with_data_loader(self):
                 one_var = paddle.tensor.fill_constant(
                     shape=[1], dtype='int64', value=1
                 )
-                with fluid.layers.control_flow.Switch() as switch:
-                    with switch.case(label != zero_var):
-                        paddle.assign(zero_var, output=label)
-                    with switch.default():
-                        paddle.assign(one_var, output=label)
-
+                label_val = paddle.static.nn.cond(
+                    label != zero_var, lambda: zero_var, lambda: one_var
+                )
+                paddle.assign(label_val, output=label)
                 net = resnet_cifar10(image)
                 logits = paddle.static.nn.fc(
                     x=net, size=10, activation="softmax"
diff --git a/test/ipu/test_print_op_ipu.py b/test/ipu/test_print_op_ipu.py
index 358866ee0a812..10449cd48ae83 100644
--- a/test/ipu/test_print_op_ipu.py
+++ b/test/ipu/test_print_op_ipu.py
@@ -55,7 +55,7 @@ def build_model(self):
             dtype=self.feed_dtype[0],
         )
         out = paddle.static.nn.conv2d(x, num_filters=3, filter_size=3)
-        out = paddle.fluid.layers.Print(out, **self.attrs)
+        out = paddle.static.Print(out, **self.attrs)
 
         if self.is_training:
             loss = paddle.mean(out)

From 6f684bd2ee5de97053292a7bf648419273e671c3 Mon Sep 17 00:00:00 2001
From: Shaojie WANG <wsjmessi@163.com>
Date: Tue, 25 Apr 2023 00:03:56 -0700
Subject: [PATCH 049/405] fix shared memory over usage in embedding grad kernel
 on deterministic mode (#53247)

* fix shared memory over usage in embedding grad kernel on determistic mode

* use IdT as interger dtype
---
 .../phi/kernels/gpu/embedding_grad_kernel.cu  | 32 +++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/paddle/phi/kernels/gpu/embedding_grad_kernel.cu b/paddle/phi/kernels/gpu/embedding_grad_kernel.cu
index cb34f5844b0d9..4771dd15dd296 100644
--- a/paddle/phi/kernels/gpu/embedding_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/embedding_grad_kernel.cu
@@ -18,6 +18,7 @@
 #include "glog/logging.h"
 #include "paddle/phi/backends/gpu/gpu_context.h"
 #include "paddle/phi/backends/gpu/gpu_primitives.h"
+#include "paddle/phi/common/amp_type_traits.h"
 #include "paddle/phi/common/data_type.h"
 #include "paddle/phi/common/memory_utils.h"
 #include "paddle/phi/core/kernel_registry.h"
@@ -74,16 +75,14 @@ __global__ void EmbeddingGrad(T* table,
 }
 
 template <typename T, typename IdT>
-__global__ void EmbeddingGradDeterministic(T* table,
-                                           const T* output,
-                                           const IdT* ids,
-                                           const int64_t K,
-                                           const int64_t D) {
+__global__ void EmbeddingGradDeterministic(
+    T* table, const T* output, const IdT* ids, const IdT K, const IdT D) {
+  using MT = typename dtype::MPTypeTrait<T>::Type;
   extern __shared__ char buf[];
-  T* smem = reinterpret_cast<T*>(buf);
-  T* my_s = smem + WARP_SIZE * threadIdx.y;
-  int64_t* indices_batch =
-      reinterpret_cast<int64_t*>(buf + sizeof(T) * WARP_SIZE * BLOCKDIMY);
+  MT* smem = reinterpret_cast<MT*>(buf);
+  MT* my_s = smem + WARP_SIZE * threadIdx.y;
+  IdT* indices_batch =
+      reinterpret_cast<IdT*>(buf + sizeof(MT) * WARP_SIZE * BLOCKDIMY);
 
   const int stride = static_cast<int>(D);
 
@@ -97,10 +96,10 @@ __global__ void EmbeddingGradDeterministic(T* table,
        batch_start += WARP_SIZE * BLOCKDIMY) {
     int tid = threadIdx.x + threadIdx.y * WARP_SIZE;
     if (batch_start + tid < K)
-      indices_batch[tid] = static_cast<int64_t>(ids[batch_start + tid]);
+      indices_batch[tid] = static_cast<IdT>(ids[batch_start + tid]);
 
     int batch_end =
-        min(static_cast<int64_t>(batch_start + WARP_SIZE * BLOCKDIMY), K);
+        min(static_cast<IdT>(batch_start + WARP_SIZE * BLOCKDIMY), K);
 
     // Loop over the batch of <= 1024 loaded indices in chunks of BLOCKDIMY
     for (int chunk_start = batch_start; chunk_start < batch_end;
@@ -112,10 +111,10 @@ __global__ void EmbeddingGradDeterministic(T* table,
 
       int n_this_chunk = min(batch_end - chunk_start, BLOCKDIMY);
 
-      int64_t src_row = static_cast<int64_t>(chunk_start + threadIdx.y);
-      int64_t dst_row = indices_batch[src_row - batch_start];
+      IdT src_row = static_cast<IdT>(chunk_start + threadIdx.y);
+      IdT dst_row = indices_batch[src_row - batch_start];
       if (src_row < K && feature < stride)
-        my_s[threadIdx.x] = static_cast<T>(output[src_row * D + feature]);
+        my_s[threadIdx.x] = static_cast<MT>(output[src_row * D + feature]);
 
       __syncthreads();
 
@@ -202,11 +201,12 @@ struct EmbeddingGradCUDAFunctor {
       if (FLAGS_embedding_deterministic) {
         dim3 threads(WARP_SIZE, BLOCKDIMY);
         dim3 grids(static_cast<int>((D + WARP_SIZE - 1) / WARP_SIZE));
+        using MT = typename dtype::MPTypeTrait<T>::Type;
         EmbeddingGradDeterministic<T, IdT>
             <<<grids,
                threads,
-               sizeof(T) * WARP_SIZE * BLOCKDIMY +
-                   sizeof(int) * WARP_SIZE * BLOCKDIMY,
+               sizeof(MT) * WARP_SIZE * BLOCKDIMY +
+                   sizeof(IdT) * WARP_SIZE * BLOCKDIMY,
                dev_ctx_.stream()>>>(d_table, d_output, ids, K, D);
       } else {
         const int gridx = 2 * dev_ctx_.GetSMCount();

From d7a5e900fab07060a542b32fad887c3f0422f493 Mon Sep 17 00:00:00 2001
From: cyberslack_lee <luhputu0815@gmail.com>
Date: Tue, 25 Apr 2023 15:05:48 +0800
Subject: [PATCH 050/405] =?UTF-8?q?=20=E3=80=90Hackathon=20No.61=E3=80=91m?=
 =?UTF-8?q?in=20=E7=AE=97=E5=AD=90FP16/BF16=E5=8D=95=E6=B5=8B=E5=AE=8C?=
 =?UTF-8?q?=E5=96=84=20(#52887)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../phi/kernels/gpu/reduce_min_grad_kernel.cu | 61 ++++++++++++++++++-
 paddle/phi/kernels/kps/reduce_min_kernel.cu   | 12 +++-
 paddle/phi/kernels/reduce_min_kernel.cc       | 15 ++++-
 .../fluid/tests/unittests/test_reduce_op.py   | 45 ++++++++++++++
 python/paddle/tensor/math.py                  |  5 +-
 5 files changed, 132 insertions(+), 6 deletions(-)

diff --git a/paddle/phi/kernels/gpu/reduce_min_grad_kernel.cu b/paddle/phi/kernels/gpu/reduce_min_grad_kernel.cu
index ea1d377c45976..86cccc5e03b1c 100644
--- a/paddle/phi/kernels/gpu/reduce_min_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/reduce_min_grad_kernel.cu
@@ -16,8 +16,63 @@
 
 #include "paddle/phi/backends/gpu/gpu_context.h"
 #include "paddle/phi/core/kernel_registry.h"
-#include "paddle/phi/kernels/impl/reduce_min_grad_kernel_impl.h"
+#include "paddle/phi/kernels/funcs/broadcast_function.h"
+#include "paddle/phi/kernels/funcs/compare_functors.h"
+#include "paddle/phi/kernels/funcs/elementwise_functor.h"
+#include "paddle/phi/kernels/funcs/reduce_function.h"
 
+namespace phi {
+
+template <typename T, typename Context>
+void ReduceMinGradKernel(const Context& dev_ctx,
+                         const DenseTensor& x,
+                         const DenseTensor& out,
+                         const DenseTensor& out_grad,
+                         const IntArray& dims,
+                         bool keep_dim,
+                         bool reduce_all,
+                         DenseTensor* x_grad) {
+  dev_ctx.Alloc(x_grad, x.dtype());
+  reduce_all = recompute_reduce_all(x, dims, reduce_all);
+
+  // get reduce_dim
+  int dim_size = x.dims().size();
+  auto reduce_dims =
+      funcs::details::GetReduceDim(dims.GetData(), dim_size, reduce_all);
+  auto update_dims = vectorize(x.dims());
+  for (auto i : reduce_dims) {
+    update_dims[i] = 1;
+  }
+
+  // make new tensor of out and out_grad
+  phi::DenseTensor new_out(out.type());
+  new_out.ShareDataWith(out);
+  new_out.Resize(phi::make_ddim(update_dims));
+
+  phi::DenseTensor new_out_grad(out_grad.type());
+  new_out_grad.ShareDataWith(out_grad);
+  new_out_grad.Resize(phi::make_ddim(update_dims));
+
+  // make equal_out
+  phi::DenseTensor* equal_out = new phi::DenseTensor();
+  equal_out->Resize(x.dims());
+  dev_ctx.template Alloc<T>(equal_out);
+
+  // compute
+  // 1. equal_out = Equal(x, y)
+  std::vector<const phi::DenseTensor*> equal_inputs = {&new_out, &x};
+  std::vector<phi::DenseTensor*> equal_outputs = {equal_out};
+  funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
+      dev_ctx, equal_inputs, &equal_outputs, 0, funcs::EqualFunctor<T>());
+
+  // 2. dx = dout * 1
+  std::vector<const phi::DenseTensor*> mul_inputs = {&new_out_grad, equal_out};
+  std::vector<phi::DenseTensor*> mul_outputs = {x_grad};
+  funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
+      dev_ctx, mul_inputs, &mul_outputs, 0, funcs::MultiplyFunctor<T>());
+  delete equal_out;
+}
+}  // namespace phi
 PD_REGISTER_KERNEL(min_grad,
                    GPU,
                    ALL_LAYOUT,
@@ -25,4 +80,6 @@ PD_REGISTER_KERNEL(min_grad,
                    float,
                    double,
                    int,
-                   int64_t) {}
+                   int64_t,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/paddle/phi/kernels/kps/reduce_min_kernel.cu b/paddle/phi/kernels/kps/reduce_min_kernel.cu
index 450fee16b4ca9..8ed9ec30c1920 100644
--- a/paddle/phi/kernels/kps/reduce_min_kernel.cu
+++ b/paddle/phi/kernels/kps/reduce_min_kernel.cu
@@ -36,6 +36,14 @@ void MinRawKernel(const Context& dev_ctx,
 #ifdef PADDLE_WITH_XPU_KP
 PD_REGISTER_KERNEL(min_raw, KPS, ALL_LAYOUT, phi::MinRawKernel, float) {}
 #else
-PD_REGISTER_KERNEL(
-    min_raw, KPS, ALL_LAYOUT, phi::MinRawKernel, float, double, int, int64_t) {}
+PD_REGISTER_KERNEL(min_raw,
+                   KPS,
+                   ALL_LAYOUT,
+                   phi::MinRawKernel,
+                   float,
+                   double,
+                   int,
+                   int64_t,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
 #endif
diff --git a/paddle/phi/kernels/reduce_min_kernel.cc b/paddle/phi/kernels/reduce_min_kernel.cc
index c4c58c8342e60..ff50e9d1077b0 100644
--- a/paddle/phi/kernels/reduce_min_kernel.cc
+++ b/paddle/phi/kernels/reduce_min_kernel.cc
@@ -39,7 +39,20 @@ void MinKernel(const Context& dev_ctx,
 PD_REGISTER_KERNEL(
     min, CPU, ALL_LAYOUT, phi::MinKernel, float, double, int, int64_t) {}
 
-#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+#if defined(PADDLE_WITH_CUDA)
+PD_REGISTER_KERNEL(min,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::MinKernel,
+                   float,
+                   double,
+                   int,
+                   int64_t,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
+#endif
+
+#if defined(PADDLE_WITH_HIP)
 PD_REGISTER_KERNEL(
     min, GPU, ALL_LAYOUT, phi::MinKernel, float, double, int, int64_t) {}
 #endif
diff --git a/python/paddle/fluid/tests/unittests/test_reduce_op.py b/python/paddle/fluid/tests/unittests/test_reduce_op.py
index 050879369244d..631b760a7b8da 100644
--- a/python/paddle/fluid/tests/unittests/test_reduce_op.py
+++ b/python/paddle/fluid/tests/unittests/test_reduce_op.py
@@ -418,6 +418,51 @@ def test_check_output(self):
         self.check_output()
 
 
+@skip_check_grad_ci(
+    reason="reduce_min is discontinuous non-derivable function,"
+    " its gradient check is not supported by unittest framework."
+)
+class TestMinFP16Op(OpTest):
+    """Remove Min with subgradient from gradient check to confirm the success of CI."""
+
+    def setUp(self):
+        self.op_type = "reduce_min"
+        self.python_api = paddle.min
+        self.public_python_api = paddle.min
+        self.init_dtype()
+        if self.dtype == np.uint16:
+            x = np.random.random((5, 6, 10)).astype(np.float32)
+            self.inputs = {'X': convert_float_to_uint16(x)}
+        else:
+            x = np.random.random((5, 6, 10)).astype(self.dtype)
+            self.inputs = {'X': x}
+        self.attrs = {'dim': [2], 'keep_dim': True}
+        out = x.min(axis=tuple(self.attrs['dim']), keepdims=True)
+        if self.dtype == np.uint16:
+            self.outputs = {'Out': convert_float_to_uint16(out)}
+        else:
+            self.outputs = {'Out': out}
+
+    def init_dtype(self):
+        self.dtype = np.float16
+
+    def test_check_output(self):
+        self.check_output()
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support the bfloat16",
+)
+class TestMinBF16Op(TestMinFP16Op):
+    def init_dtype(self):
+        self.dtype = np.uint16
+
+    def test_check_output(self):
+        self.check_output_with_place(core.CUDAPlace(0))
+
+
 def raw_reduce_prod(x, dim=[0], keep_dim=False):
     return paddle.prod(x, dim, keep_dim)
 
diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py
index 6b07f57e33a17..9f5212abf9147 100644
--- a/python/paddle/tensor/math.py
+++ b/python/paddle/tensor/math.py
@@ -2456,7 +2456,10 @@ def min(x, axis=None, keepdim=False, name=None):
         reduce_all, axis = _get_reduce_axis_with_tensor(axis, x)
         helper = LayerHelper('min', **locals())
         check_variable_and_dtype(
-            x, 'x', ['float32', 'float64', 'int32', 'int64'], 'min'
+            x,
+            'x',
+            ['float16', 'uint16', 'float32', 'float64', 'int32', 'int64'],
+            'min',
         )
 
         out = helper.create_variable_for_type_inference(dtype=x.dtype)

From eb677102725e0a653e1f3d291f35162fd9b18f1d Mon Sep 17 00:00:00 2001
From: Difer <707065510@qq.com>
Date: Tue, 25 Apr 2023 15:19:26 +0800
Subject: [PATCH 051/405] =?UTF-8?q?=E3=80=90Hackathon=20No57=E3=80=91add?=
 =?UTF-8?q?=5Fbf16=5Ffp16=20unittest=20for=20conv3d=20&=20conv3d=5Ftranspo?=
 =?UTF-8?q?se=20(#52195)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add test+conv3d_transpose_part2

* fix some merge error

* fix codestyle

* fix typo

* fix codestyle

* fix some error

* add redef float2uint

* fix conv3d and conv3d_transpose
---
 .../fluid/tests/unittests/test_conv3d_op.py   | 128 +++++++++++++--
 .../unittests/test_conv3d_transpose_op.py     | 147 +++++++++++++++++-
 .../test_conv3d_transpose_part2_op.py         |  22 ++-
 3 files changed, 277 insertions(+), 20 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_op.py
index 0b843663827c1..d2b6545a81a4c 100644
--- a/python/paddle/fluid/tests/unittests/test_conv3d_op.py
+++ b/python/paddle/fluid/tests/unittests/test_conv3d_op.py
@@ -15,10 +15,16 @@
 import unittest
 
 import numpy as np
-from eager_op_test import OpTest, paddle_static_guard
+from eager_op_test import (
+    OpTest,
+    convert_float_to_uint16,
+    get_numeric_gradient,
+    paddle_static_guard,
+)
 
 import paddle
 from paddle.fluid import core
+from paddle.fluid.tests.unittests.testsuite import create_op
 
 
 def conv3d_forward_naive(
@@ -179,6 +185,77 @@ def init_kernel_type(self):
     globals()[cls_name] = TestCUDNNCase
 
 
+def create_test_cudnn_bf16_class(parent):
+    @unittest.skipIf(
+        not core.is_compiled_with_cuda()
+        or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+        "core is not compiled with CUDA and do not support bfloat16",
+    )
+    class TestConv3DCUDNNBF16(parent):
+        def get_numeric_grad(self, place, check_name):
+            scope = core.Scope()
+            self._check_grad_helper()
+            op = create_op(
+                scope, self.op_type, self.inputs, self.outputs, self.attrs
+            )
+            return get_numeric_gradient(
+                place, scope, op, self.inputs_fp32, check_name, ['Output']
+            )
+
+        def init_kernel_type(self):
+            self.use_cudnn = True
+            self.dtype = np.uint16
+
+        def test_check_output(self):
+            place = core.CUDAPlace(0)
+            self.check_output_with_place(
+                place, check_dygraph=(not self.use_mkldnn)
+            )
+
+        def test_check_grad_no_filter(self):
+            place = core.CUDAPlace(0)
+            numeric_grads = self.get_numeric_grad(place, 'Input')
+
+            self.check_grad_with_place(
+                place,
+                ['Input'],
+                'Output',
+                no_grad_set={'Filter'},
+                check_dygraph=(not self.use_mkldnn),
+                user_defined_grads=[numeric_grads],
+            )
+
+        def test_check_grad_no_input(self):
+            place = core.CUDAPlace(0)
+            numeric_grads = self.get_numeric_grad(place, 'Filter')
+
+            self.check_grad_with_place(
+                place,
+                ['Filter'],
+                'Output',
+                no_grad_set={'Input'},
+                check_dygraph=(not self.use_mkldnn),
+                user_defined_grads=[numeric_grads],
+            )
+
+        def test_check_grad(self):
+            place = core.CUDAPlace(0)
+            numeric_input_grads = self.get_numeric_grad(place, 'Input')
+            numeric_fliter_grads = self.get_numeric_grad(place, 'Filter')
+
+            self.check_grad_with_place(
+                place,
+                {'Input', 'Filter'},
+                'Output',
+                user_defined_grads=[numeric_input_grads, numeric_fliter_grads],
+                check_dygraph=(not self.use_mkldnn),
+            )
+
+    cls_name = "{}_{}".format(parent.__name__, "CUDNNBF16OP")
+    TestConv3DCUDNNBF16.__name__ = cls_name
+    globals()[cls_name] = TestConv3DCUDNNBF16
+
+
 def create_test_padding_SAME_class(parent):
     class TestPaddingSMAECase(parent):
         def init_paddings(self):
@@ -323,19 +400,37 @@ def setUp(self):
             'dilations': self.dilations,
         }
 
-        input = np.random.random(self.input_size).astype(self.dtype)
-        filter = np.random.random(self.filter_size).astype(self.dtype)
+        if self.is_bfloat16_op():
+            input = np.random.random(self.input_size).astype(np.float32)
+            filter = np.random.random(self.filter_size).astype(np.float32)
+        else:
+            input = np.random.random(self.input_size).astype(self.dtype)
+            filter = np.random.random(self.filter_size).astype(self.dtype)
+
         output = conv3d_forward_naive(
             input,
             filter,
             self.groups,
             conv3d_param,
-        ).astype(self.dtype)
+        )
+
+        if self.is_bfloat16_op():
+            output = convert_float_to_uint16(output)
+            self.inputs = {
+                'Input': convert_float_to_uint16(input),
+                'Filter': convert_float_to_uint16(filter),
+            }
+            self.inputs_fp32 = {
+                'Input': OpTest.np_dtype_to_fluid_dtype(input),
+                'Filter': OpTest.np_dtype_to_fluid_dtype(filter),
+            }
+        else:
+            output = output.astype(self.dtype)
+            self.inputs = {
+                'Input': OpTest.np_dtype_to_fluid_dtype(input),
+                'Filter': OpTest.np_dtype_to_fluid_dtype(filter),
+            }
 
-        self.inputs = {
-            'Input': OpTest.np_dtype_to_fluid_dtype(input),
-            'Filter': OpTest.np_dtype_to_fluid_dtype(filter),
-        }
         self.attrs = {
             'strides': self.stride,
             'paddings': self.pad,
@@ -358,8 +453,6 @@ def test_check_output(self):
         )
 
     def test_check_grad(self):
-        if self.dtype == np.float16:
-            return
         place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace()
         # TODO(wangzhongpu): support mkldnn op in dygraph mode
         self.check_grad_with_place(
@@ -371,8 +464,7 @@ def test_check_grad(self):
         )
 
     def test_check_grad_no_filter(self):
-        if self.dtype == np.float16:
-            return
+
         place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace()
         # TODO(wangzhongpu): support mkldnn op in dygraph mode
         self.check_grad_with_place(
@@ -385,8 +477,7 @@ def test_check_grad_no_filter(self):
         )
 
     def test_check_grad_no_input(self):
-        if self.dtype == np.float16:
-            return
+
         place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace()
         # TODO(wangzhongpu): support mkldnn op in dygraph mode
         self.check_grad_with_place(
@@ -617,6 +708,14 @@ def init_kernel_type(self):
         self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64
 
 
+# ----------------Conv3DCUDNN bf16----------------
+create_test_cudnn_bf16_class(TestConv3DOp)
+create_test_cudnn_bf16_class(TestWithGroup1)
+create_test_cudnn_bf16_class(TestWithGroup2)
+create_test_cudnn_bf16_class(TestWith1x1)
+create_test_cudnn_bf16_class(TestWithInput1x1Filter1x1)
+
+
 # ---- test asymmetric padding ----
 
 
@@ -1114,4 +1213,5 @@ def run_8():
 
 
 if __name__ == '__main__':
+
     unittest.main()
diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py
index 22e22d9b2f66a..c0814754cc231 100644
--- a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py
+++ b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py
@@ -19,11 +19,25 @@
 import paddle
 
 paddle.enable_static()
-from eager_op_test import OpTest
+from eager_op_test import OpTest, copy_bits_from_float_to_uint16
 
 from paddle.fluid import core
 
 
+def convert_float_to_uint16(float_list, data_format="NCHW"):
+    if data_format == "NHWC":
+        float_list = np.transpose(float_list, [0, 4, 1, 2, 3])
+
+    new_output = []
+    for x in np.nditer(float_list):
+        new_output.append(np.uint16(copy_bits_from_float_to_uint16(x)))
+    new_output = np.reshape(new_output, float_list.shape).view(np.uint16)
+
+    if data_format == "NHWC":
+        new_output = np.transpose(new_output, [0, 2, 3, 4, 1])
+    return new_output
+
+
 def conv3dtranspose_forward_naive(input_, filter_, attrs):
     padding_algorithm = attrs['padding_algorithm']
     if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]:
@@ -134,6 +148,86 @@ def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride):
     return out
 
 
+def create_test_cudnn_fp16_class(parent, grad_check=True):
+    @unittest.skipIf(
+        not core.is_compiled_with_cuda(), "core is not compiled with CUDA"
+    )
+    class TestConv3DTransposeCUDNNFP16(parent):
+        def init_kernel_type(self):
+            self.use_cudnn = True
+            self.dtype = np.float16
+
+        def test_check_output(self):
+            if core.is_compiled_with_cuda():
+                place = core.CUDAPlace(0)
+                if core.is_float16_supported(place):
+                    self.check_output_with_place(place, atol=2e-2)
+
+        def test_check_grad_no_filter(self):
+            place = core.CUDAPlace(0)
+            if core.is_float16_supported(place) and grad_check:
+                self.check_grad_with_place(
+                    place, ['Input'], 'Output', no_grad_set={'Filter'}
+                )
+
+        def test_check_grad_no_input(self):
+            place = core.CUDAPlace(0)
+            if core.is_float16_supported(place) and grad_check:
+                self.check_grad_with_place(
+                    place, ['Filter'], 'Output', no_grad_set={'Input'}
+                )
+
+    cls_name = "{}_{}".format(parent.__name__, "CUDNNFP16OP")
+    TestConv3DTransposeCUDNNFP16.__name__ = cls_name
+    globals()[cls_name] = TestConv3DTransposeCUDNNFP16
+
+
+def create_test_cudnn_bf16_class(parent):
+    @unittest.skipIf(
+        not core.is_compiled_with_cuda()
+        or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+        "core is not compiled with CUDA and do not support bfloat16",
+    )
+    class TestConv3DTransposeCUDNNBF16(parent):
+        def init_kernel_type(self):
+            self.use_cudnn = True
+            self.dtype = np.uint16
+
+        def test_check_output(self):
+            place = core.CUDAPlace(0)
+            self.check_output_with_place(place)
+
+        def test_check_grad(self):
+            place = core.CUDAPlace(0)
+            self.check_grad_with_place(
+                place,
+                {'Input', 'Filter'},
+                'Output',
+            )
+
+        def test_check_grad_no_filter(self):
+            place = core.CUDAPlace(0)
+            self.check_grad_with_place(
+                place,
+                ['Input'],
+                'Output',
+                no_grad_set={'Filter'},
+            )
+
+        def test_check_grad_no_input(self):
+            place = core.CUDAPlace(0)
+            self.check_grad_with_place(
+                place,
+                ['Filter'],
+                'Output',
+                no_grad_set={'Input'},
+            )
+
+    cls_name = "{}_{}".format(parent.__name__, "CUDNNBF16OP")
+    TestConv3DTransposeCUDNNBF16.__name__ = cls_name
+    globals()[cls_name] = TestConv3DTransposeCUDNNBF16
+
+
 def conv3d_transpose_wrapper(
     x,
     weight,
@@ -172,12 +266,16 @@ def setUp(self):
         self.pad = [0, 0, 0]
         self.padding_algorithm = "EXPLICIT"
         self.init_op_type()
+        self.init_kernel_type()
         self.init_test_case()
 
-        input_ = np.random.random(self.input_size).astype("float32")
-        filter_ = np.random.random(self.filter_size).astype("float32")
+        if self.is_bfloat16_op():
+            input = np.random.random(self.input_size).astype(np.float32)
+            filter = np.random.random(self.filter_size).astype(np.float32)
+        else:
+            input = np.random.random(self.input_size).astype(self.dtype)
+            filter = np.random.random(self.filter_size).astype(self.dtype)
 
-        self.inputs = {'Input': input_, 'Filter': filter_}
         self.attrs = {
             'strides': self.stride,
             'paddings': self.pad,
@@ -189,9 +287,21 @@ def setUp(self):
         }
 
         output = conv3dtranspose_forward_naive(
-            input_, filter_, self.attrs
+            input, filter, self.attrs
         ).astype("float32")
 
+        if self.is_bfloat16_op():
+            self.inputs = {
+                'Input': convert_float_to_uint16(input),
+                'Filter': convert_float_to_uint16(filter),
+            }
+        else:
+            self.inputs = {
+                'Input': input,
+                'Filter': filter,
+            }
+            output = output.astype(self.dtype)
+
         self.outputs = {'Output': output}
 
     def test_check_output(self):
@@ -264,6 +374,9 @@ def init_op_type(self):
         self.op_type = "conv3d_transpose"
         self.python_api = conv3d_transpose_wrapper
 
+    def init_kernel_type(self):
+        self.dtype = np.float32
+
 
 class TestWithSymmetricPad(TestConv3DTransposeOp):
     def init_test_case(self):
@@ -596,6 +709,30 @@ def init_op_type(self):
         self.python_api = conv3d_transpose_wrapper
 
 
+# ----------------Conv3DTransposeCUDNN fp16----------------
+create_test_cudnn_fp16_class(TestConv3DTransposeOp)
+create_test_cudnn_fp16_class(TestWithSymmetricPad)
+create_test_cudnn_fp16_class(TestWithAsymmetricPad)
+create_test_cudnn_fp16_class(TestWithSAMEPad)
+create_test_cudnn_fp16_class(TestWithVALIDPad)
+create_test_cudnn_fp16_class(TestWithStride)
+create_test_cudnn_fp16_class(TestWithGroups)
+create_test_cudnn_fp16_class(TestWithDilation)
+create_test_cudnn_fp16_class(Test_NHWC)
+
+
+# ----------------Conv3DTransposeCUDNN bf16----------------
+create_test_cudnn_bf16_class(TestConv3DTransposeOp)
+create_test_cudnn_bf16_class(TestWithSymmetricPad)
+create_test_cudnn_bf16_class(TestWithAsymmetricPad)
+create_test_cudnn_bf16_class(TestWithSAMEPad)
+create_test_cudnn_bf16_class(TestWithVALIDPad)
+create_test_cudnn_bf16_class(TestWithStride)
+create_test_cudnn_bf16_class(TestWithGroups)
+create_test_cudnn_bf16_class(TestWithDilation)
+create_test_cudnn_bf16_class(Test_NHWC)
+
+
 class TestConv3dTranspose(unittest.TestCase):
     def error_weight_input(self):
         array = np.array([1], dtype=np.float32)
diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_part2_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_part2_op.py
index b51cdd9b1087a..54d31a7bbd22b 100644
--- a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_part2_op.py
+++ b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_part2_op.py
@@ -15,7 +15,11 @@
 import unittest
 
 import numpy as np
-from test_conv3d_transpose_op import TestConv3DTransposeOp
+from test_conv3d_transpose_op import (
+    TestConv3DTransposeOp,
+    create_test_cudnn_bf16_class,
+    create_test_cudnn_fp16_class,
+)
 
 import paddle
 from paddle import fluid
@@ -84,6 +88,22 @@ def init_test_case(self):
         self.data_format = 'NHWC'
 
 
+# ----------------Conv3DTransposeCUDNN fp16----------------
+create_test_cudnn_fp16_class(TestWithSymmetricPad_NHWC)
+create_test_cudnn_fp16_class(TestWithAsymmetricPad_NHWC)
+create_test_cudnn_fp16_class(TestWithGroups_NHWC)
+create_test_cudnn_fp16_class(TestWithStride_NHWC)
+create_test_cudnn_fp16_class(TestWithDilation_NHWC)
+
+
+# ----------------Conv3DTransposeCUDNN bf16----------------
+create_test_cudnn_bf16_class(TestWithSymmetricPad_NHWC)
+create_test_cudnn_bf16_class(TestWithAsymmetricPad_NHWC)
+create_test_cudnn_bf16_class(TestWithGroups_NHWC)
+create_test_cudnn_bf16_class(TestWithStride_NHWC)
+create_test_cudnn_bf16_class(TestWithDilation_NHWC)
+
+
 class TestConv3DTransposeAPI(unittest.TestCase):
     def test_case1(self):
         data1 = paddle.static.data(

From 8d4b64e8f537e4438eadb601a3a01835999d26ed Mon Sep 17 00:00:00 2001
From: Chitsing KUI <kuizhiqing@msn.com>
Date: Tue, 25 Apr 2023 15:34:04 +0800
Subject: [PATCH 052/405] [DEBUG] print modifed flags (#53243)

* print modifed flags

* fix ref, opt print

* fix default getter

* fix ut
---
 .../pybind/global_value_getter_setter.cc      | 89 ++++++++++++++-----
 python/paddle/distributed/parallel.py         | 31 ++++++-
 2 files changed, 98 insertions(+), 22 deletions(-)

diff --git a/paddle/fluid/pybind/global_value_getter_setter.cc b/paddle/fluid/pybind/global_value_getter_setter.cc
index e6935b577d737..94e3ca1ba41bd 100644
--- a/paddle/fluid/pybind/global_value_getter_setter.cc
+++ b/paddle/fluid/pybind/global_value_getter_setter.cc
@@ -64,6 +64,11 @@ class PYBIND11_HIDDEN GlobalVarGetterSetterRegistry {
     return [&]() -> py::object { return py::cast(var); };
   }
 
+  template <typename T>
+  static Getter CreateDefaultValueGetter(const T &var) {
+    return [=]() -> py::object { return py::cast(var); };
+  }
+
   template <typename T>
   static Setter CreateSetter(T *var) {
     return [var](const py::object &obj) { *var = py::cast<T>(obj); };
@@ -71,14 +76,23 @@ class PYBIND11_HIDDEN GlobalVarGetterSetterRegistry {
 
  private:
   struct VarInfo {
-    VarInfo(bool is_public, const Getter &getter)
-        : is_public(is_public), getter(getter) {}
-
-    VarInfo(bool is_public, const Getter &getter, const Setter &setter)
-        : is_public(is_public), getter(getter), setter(setter) {}
+    VarInfo(bool is_public, const Getter &getter, const Getter &default_getter)
+        : is_public(is_public),
+          getter(getter),
+          default_getter(default_getter) {}
+
+    VarInfo(bool is_public,
+            const Getter &getter,
+            const Getter &default_getter,
+            const Setter &setter)
+        : is_public(is_public),
+          getter(getter),
+          default_getter(default_getter),
+          setter(setter) {}
 
     const bool is_public;
     const Getter getter;
+    const Getter default_getter;
     const Setter setter;
   };
 
@@ -87,7 +101,10 @@ class PYBIND11_HIDDEN GlobalVarGetterSetterRegistry {
 
   static GlobalVarGetterSetterRegistry *MutableInstance() { return &instance_; }
 
-  void Register(const std::string &name, bool is_public, const Getter &getter) {
+  void Register(const std::string &name,
+                bool is_public,
+                const Getter &getter,
+                const Getter &default_getter) {
     PADDLE_ENFORCE_EQ(
         HasGetterMethod(name),
         false,
@@ -96,12 +113,13 @@ class PYBIND11_HIDDEN GlobalVarGetterSetterRegistry {
     PADDLE_ENFORCE_NOT_NULL(getter,
                             platform::errors::InvalidArgument(
                                 "Getter of %s should not be null", name));
-    var_infos_.insert({name, VarInfo(is_public, getter)});
+    var_infos_.insert({name, VarInfo(is_public, getter, default_getter)});
   }
 
   void Register(const std::string &name,
                 bool is_public,
                 const Getter &getter,
+                const Getter &default_getter,
                 const Setter &setter) {
     PADDLE_ENFORCE_EQ(
         HasGetterMethod(name),
@@ -122,7 +140,8 @@ class PYBIND11_HIDDEN GlobalVarGetterSetterRegistry {
     PADDLE_ENFORCE_NOT_NULL(setter,
                             platform::errors::InvalidArgument(
                                 "Setter of %s should not be null", name));
-    var_infos_.insert({name, VarInfo(is_public, getter, setter)});
+    var_infos_.insert(
+        {name, VarInfo(is_public, getter, default_getter, setter)});
   }
 
   const Getter &GetterMethod(const std::string &name) const {
@@ -133,6 +152,14 @@ class PYBIND11_HIDDEN GlobalVarGetterSetterRegistry {
     return var_infos_.at(name).getter;
   }
 
+  const Getter &DefaultGetterMethod(const std::string &name) const {
+    PADDLE_ENFORCE_EQ(
+        HasGetterMethod(name),
+        true,
+        platform::errors::NotFound("Cannot find global variable %s", name));
+    return var_infos_.at(name).default_getter;
+  }
+
   py::object GetOrReturnDefaultValue(const std::string &name,
                                      const py::object &default_value) const {
     if (HasGetterMethod(name)) {
@@ -142,6 +169,14 @@ class PYBIND11_HIDDEN GlobalVarGetterSetterRegistry {
     }
   }
 
+  py::object GetDefaultValue(const std::string &name) const {
+    if (HasGetterMethod(name)) {
+      return DefaultGetterMethod(name)();
+    } else {
+      return py::cast(Py_None);
+    }
+  }
+
   py::object Get(const std::string &name) const { return GetterMethod(name)(); }
 
   const Setter &SetterMethod(const std::string &name) const {
@@ -198,6 +233,9 @@ void BindGlobalValueGetterSetter(pybind11::module *module) {
       .def("__contains__", &GlobalVarGetterSetterRegistry::HasGetterMethod)
       .def("keys", &GlobalVarGetterSetterRegistry::Keys)
       .def("is_public", &GlobalVarGetterSetterRegistry::IsPublic)
+      .def("get_default",
+           &GlobalVarGetterSetterRegistry::GetDefaultValue,
+           py::arg("key"))
       .def("get",
            &GlobalVarGetterSetterRegistry::GetOrReturnDefaultValue,
            py::arg("key"),
@@ -209,13 +247,15 @@ void BindGlobalValueGetterSetter(pybind11::module *module) {
 }
 
 /* Public vars are designed to be writable. */
-#define REGISTER_PUBLIC_GLOBAL_VAR(var)                                    \
-  do {                                                                     \
-    auto *instance = GlobalVarGetterSetterRegistry::MutableInstance();     \
-    instance->Register(#var,                                               \
-                       /*is_public=*/true,                                 \
-                       GlobalVarGetterSetterRegistry::CreateGetter(var),   \
-                       GlobalVarGetterSetterRegistry::CreateSetter(&var)); \
+#define REGISTER_PUBLIC_GLOBAL_VAR(var)                                \
+  do {                                                                 \
+    auto *instance = GlobalVarGetterSetterRegistry::MutableInstance(); \
+    instance->Register(                                                \
+        #var,                                                          \
+        /*is_public=*/true,                                            \
+        GlobalVarGetterSetterRegistry::CreateGetter(var),              \
+        GlobalVarGetterSetterRegistry::CreateDefaultValueGetter(var),  \
+        GlobalVarGetterSetterRegistry::CreateSetter(&var));            \
   } while (0)
 
 struct RegisterGetterSetterVisitor {
@@ -225,18 +265,25 @@ struct RegisterGetterSetterVisitor {
       : name_(name), is_writable_(is_writable), value_ptr_(value_ptr) {}
 
   template <typename T>
-  void operator()(const T &) const {
+  void operator()(const T &default_value) const {
     auto &value = *static_cast<T *>(value_ptr_);
     auto *instance = GlobalVarGetterSetterRegistry::MutableInstance();
     bool is_public = is_writable_;  // currently, all writable vars are public
     if (is_writable_) {
-      instance->Register(name_,
-                         is_public,
-                         GlobalVarGetterSetterRegistry::CreateGetter(value),
-                         GlobalVarGetterSetterRegistry::CreateSetter(&value));
+      instance->Register(
+          name_,
+          is_public,
+          GlobalVarGetterSetterRegistry::CreateGetter(value),
+          GlobalVarGetterSetterRegistry::CreateDefaultValueGetter(
+              default_value),
+          GlobalVarGetterSetterRegistry::CreateSetter(&value));
     } else {
       instance->Register(
-          name_, is_public, GlobalVarGetterSetterRegistry::CreateGetter(value));
+          name_,
+          is_public,
+          GlobalVarGetterSetterRegistry::CreateGetter(value),
+          GlobalVarGetterSetterRegistry::CreateDefaultValueGetter(
+              default_value));
     }
   }
 
diff --git a/python/paddle/distributed/parallel.py b/python/paddle/distributed/parallel.py
index c016f9d743c7f..3fdf7cdcdd954 100644
--- a/python/paddle/distributed/parallel.py
+++ b/python/paddle/distributed/parallel.py
@@ -14,9 +14,10 @@
 
 import itertools
 import os
+import sys
 import time
 import warnings
-from collections import OrderedDict
+from collections import OrderedDict, namedtuple
 from contextlib import contextmanager
 from multiprocessing import Manager  # noqa: F401
 from multiprocessing import Process  # noqa: F401
@@ -905,6 +906,31 @@ def _check_var_exists(var_name):
         )
 
 
+def _get_modified_flags():
+    ret = []
+    FLAGS = namedtuple('FLAGS', ['name', 'current_value', 'default_value'])
+    global_flags = core.globals()
+    for key in global_flags.keys():
+        value = global_flags.get(key)
+        default_value = global_flags.get_default(key)
+        if not value == default_value:
+            ret.append(FLAGS(key, value, default_value))
+    return ret
+
+
+def _print_modified_flags(modified_flags):
+    if len(modified_flags) > 0:
+        sys.stderr.write(
+            "======================= Modified FLAGS detected =======================\n"
+        )
+        for flag in modified_flags:
+            sys.stderr.write(str(flag))
+            sys.stderr.write("\n")
+        sys.stderr.write(
+            "=======================================================================\n"
+        )
+
+
 def init_parallel_env():
     """
 
@@ -967,6 +993,9 @@ def train():
 
     """
 
+    modified_flags = _get_modified_flags()
+    _print_modified_flags(modified_flags)
+
     # 0. get env & check world size
     global _global_parallel_env
     # when call init_parallel_env, need update `_global_parallel_env`

From 1ddf93961f57be866b1136fa21793f783a559243 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Tue, 25 Apr 2023 09:33:38 +0800
Subject: [PATCH 053/405] Simplify codes and fix backward.

---
 .../forwards/fused_gate_attention_fwd_func.cc |  14 +-
 .../nodes/fused_gate_attention_node.cc        |  12 ++
 .../api/manual/fluid_manual/nodes/nodes.h     |  10 +
 .../operators/fused/fused_gate_attention.h    | 186 +++++++-----------
 .../fused/fused_gate_attention_op.cc          |   1 -
 .../fused/fused_gate_attention_op.cu          |  15 +-
 6 files changed, 114 insertions(+), 124 deletions(-)

diff --git a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gate_attention_fwd_func.cc b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gate_attention_fwd_func.cc
index b0585bc7acd54..546b60438fedc 100644
--- a/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gate_attention_fwd_func.cc
+++ b/paddle/fluid/eager/api/manual/fluid_manual/forwards/fused_gate_attention_fwd_func.cc
@@ -314,10 +314,10 @@ fused_gate_attention_dygraph_function(
         has_gating = PADDLE_GET_CONST(bool, attrs.at("has_gating"));
       }
 
-      // bool use_flash_attn = false;
-      // if (attrs.count("use_flash_attn")) {
-      //   use_flash_attn = PADDLE_GET_CONST(bool, attrs.at("use_flash_attn"));
-      // }
+      bool use_flash_attn = false;
+      if (attrs.count("use_flash_attn")) {
+        use_flash_attn = PADDLE_GET_CONST(bool, attrs.at("use_flash_attn"));
+      }
 
       // Set Attributes
       grad_node->SetAttrMap(std::move(attrs));
@@ -365,6 +365,12 @@ fused_gate_attention_dygraph_function(
         grad_node->SetGradOutMeta(NonbatchedBias, 6);
       }
 
+      if (use_flash_attn) {
+        grad_node->SetTensorWrapperSoftmaxLse(SoftmaxLse);
+        grad_node->SetTensorWrapperSrcMask(SrcMask);
+        grad_node->SetGradOutMeta(SrcMask, 7);
+      }
+
       egr::EagerUtils::SetOutRankWithSlot(p_autograd_QueryTransposeOut, 0);
       grad_node->SetGradInMeta(QueryTransposeOut, 0);
       egr::EagerUtils::SetOutRankWithSlot(p_autograd_KeyTransposeOut, 1);
diff --git a/paddle/fluid/eager/api/manual/fluid_manual/nodes/fused_gate_attention_node.cc b/paddle/fluid/eager/api/manual/fluid_manual/nodes/fused_gate_attention_node.cc
index 8c427eba8cd0a..3692a20faed98 100644
--- a/paddle/fluid/eager/api/manual/fluid_manual/nodes/fused_gate_attention_node.cc
+++ b/paddle/fluid/eager/api/manual/fluid_manual/nodes/fused_gate_attention_node.cc
@@ -45,6 +45,11 @@ fused_gate_attentionGradNodeCompat::operator()(
     has_gating = PADDLE_GET_CONST(bool, attr_map_.at("has_gating"));
   }
 
+  bool use_flash_attn = false;
+  if (attr_map_.count("use_flash_attn")) {
+    use_flash_attn = PADDLE_GET_CONST(bool, attr_map_.at("use_flash_attn"));
+  }
+
   std::map<std::string, std::vector<std::shared_ptr<egr::EagerVariable>>> ins0 =
       {{"FMHAOut",
         egr::EagerUtils::TrySyncToVars(
@@ -168,6 +173,13 @@ fused_gate_attentionGradNodeCompat::operator()(
           egr::Controller::Instance().GenerateUniqueName())};
   }
 
+  if (use_flash_attn) {
+    auto SrcMask = egr::EagerUtils::RecoverTensorWrapper(&this->SrcMask_);
+    ins0["SrcMask"] = egr::EagerUtils::TrySyncToVars(SrcMask);
+    auto SoftmaxLse = egr::EagerUtils::RecoverTensorWrapper(&this->SoftmaxLse_);
+    ins0["SoftmaxLse"] = egr::EagerUtils::TrySyncToVars(SoftmaxLse);
+  }
+
   auto& attrs_map0 = this->attr_map_;
   // Pass the entire attribute map to TraceOp
   // The underlying kernel will pickup whatever attribute they need at runtime
diff --git a/paddle/fluid/eager/api/manual/fluid_manual/nodes/nodes.h b/paddle/fluid/eager/api/manual/fluid_manual/nodes/nodes.h
index b0576672ae18e..212f9d9f1da19 100644
--- a/paddle/fluid/eager/api/manual/fluid_manual/nodes/nodes.h
+++ b/paddle/fluid/eager/api/manual/fluid_manual/nodes/nodes.h
@@ -61,12 +61,14 @@ class fused_gate_attentionGradNodeCompat : public egr::GradNodeBase {
     GateOut_.clear();
     GateWeight_.clear();
     NonbatchedBias_.clear();
+    SrcMask_.clear();
     OutLinearBias_.clear();
     OutLinearWeight_.clear();
     QKVTransposeOut_.clear();
     QKVWeight_.clear();
     Query_.clear();
     SoftmaxOut_.clear();
+    SoftmaxLse_.clear();
     Key_.clear();
     QueryWeight_.clear();
     KeyWeight_.clear();
@@ -103,6 +105,9 @@ class fused_gate_attentionGradNodeCompat : public egr::GradNodeBase {
   void SetTensorWrapperNonbatchedBias(const paddle::Tensor& NonbatchedBias) {
     NonbatchedBias_ = egr::TensorWrapper(NonbatchedBias, false);
   }
+  void SetTensorWrapperSrcMask(const paddle::Tensor& SrcMask) {
+    SrcMask_ = egr::TensorWrapper(SrcMask, false);
+  }
   void SetTensorWrapperOutLinearBias(const paddle::Tensor& OutLinearBias) {
     OutLinearBias_ = egr::TensorWrapper(OutLinearBias, false);
   }
@@ -121,6 +126,9 @@ class fused_gate_attentionGradNodeCompat : public egr::GradNodeBase {
   void SetTensorWrapperSoftmaxOut(const paddle::Tensor& SoftmaxOut) {
     SoftmaxOut_ = egr::TensorWrapper(SoftmaxOut, false);
   }
+  void SetTensorWrapperSoftmaxLse(const paddle::Tensor& SoftmaxLse) {
+    SoftmaxLse_ = egr::TensorWrapper(SoftmaxLse, false);
+  }
   void SetTensorWrapperKey(const paddle::Tensor& Key) {
     Key_ = egr::TensorWrapper(Key, false);
   }
@@ -160,12 +168,14 @@ class fused_gate_attentionGradNodeCompat : public egr::GradNodeBase {
   egr::TensorWrapper GateOut_;
   egr::TensorWrapper GateWeight_;
   egr::TensorWrapper NonbatchedBias_;
+  egr::TensorWrapper SrcMask_;
   egr::TensorWrapper OutLinearBias_;
   egr::TensorWrapper OutLinearWeight_;
   egr::TensorWrapper QKVTransposeOut_;
   egr::TensorWrapper QKVWeight_;
   egr::TensorWrapper Query_;
   egr::TensorWrapper SoftmaxOut_;
+  egr::TensorWrapper SoftmaxLse_;
 
   egr::TensorWrapper Key_;
   egr::TensorWrapper QueryWeight_;
diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 07b4b5973cac8..3579caf1bc99d 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -951,10 +951,6 @@ class FlashAttnWithGating {
                       phi::DenseTensor* fmha_out,
                       phi::DenseTensor* gate_out,
                       GateAttentionConfig<T>* config) {
-    T* q_ptr = nullptr;
-    T* k_ptr = nullptr;
-    T* v_ptr = nullptr;
-
     bool is_bf16 =
         qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
 
@@ -992,29 +988,14 @@ class FlashAttnWithGating {
 
     // q_size == k_size
     int64_t q_size = config->GetQuerySize();
-    q_ptr = qkv_transpose_out->data<T>();
-    k_ptr = q_ptr + q_size;
-    v_ptr = k_ptr + q_size;
+    T* q_ptr = qkv_transpose_out->data<T>();
+    T* k_ptr = q_ptr + q_size;
+    T* v_ptr = k_ptr + q_size;
 
     // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
     phi::DenseTensor cu_seq_q, cu_seq_k;
-    int64_t end_size = seq_batch_size + 1;
-    int64_t seq_size = 0;
-    int64_t start = 0;
-    int64_t end = end_size;
-    int64_t step = static_cast<int>(config->seq_len_r);
-    phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
-    cu_seq_q.Resize({end_size});
-    cu_seq_k.Resize({end_size});
-    AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_q", &cu_seq_q);
-    AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_k", &cu_seq_k);
-    int64_t block = std::min(seq_size, static_cast<int64_t>(256));
-    int64_t grid = (seq_size + block - 1) / block;
-    FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
-        start, step, end, cu_seq_q.data<int32_t>(), cu_seq_k.data<int32_t>());
-    VLOG(4) << "[Flash_attn] cu_seq_len : start = " << start
-            << ", step = " << step << ", end = " << end;
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_) << "2: Init cu_seq_q and cu_seq_k";
+    int64_t step = static_cast<int64_t>(config->seq_len_r);
+    AllocAndInitSeqQK(seq_batch_size, step, &cu_seq_q, &cu_seq_k);
 
     // 3. Dealing with mask and bias for flash_attn.
     phi::DenseTensor temp_mask, temp_bias;
@@ -1071,9 +1052,7 @@ class FlashAttnWithGating {
               << "]";
 
     // 6. construct random seed
-    auto gen = dev_ctx_.GetGenerator();
-    uint64_t inc = batch_size_ * num_heads_ * 32;
-    auto seed_offset_pair = gen->IncrementOffset(inc);
+    auto seed_offset_pair = GenerateSeedOffsetPair(batch_size_, num_heads_);
     uint64_t seed = seed_offset_pair.first;
     uint64_t offset = seed_offset_pair.second;
 
@@ -1124,13 +1103,7 @@ class FlashAttnWithGating {
     LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
               << "7: Get workspace_size=" << workspace_size;
 
-    phi::DenseTensor workspace;
-    if (workspace_size > 0) {
-      workspace = phi::Empty<float, phi::GPUContext>(
-          dev_ctx_, {int64_t(workspace_size / sizeof(float))});
-      DBGPTR(workspace.data(), "workspace");
-    }
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_) << "Allocate workspace";
+    phi::DenseTensor workspace = CreateWorkspace(workspace_size);
 
     LOG(INFO) << "qkv_transpose_out: " << TensorDebugString(qkv_transpose_out);
     LOG(INFO) << "src_mask: " << TensorDebugString(src_mask);
@@ -1179,27 +1152,15 @@ class FlashAttnWithGating {
     }
   }
 
-  void ComputeBackward(const phi::DenseTensor* q_transpose_out,
-                       const phi::DenseTensor* k_transpose_out,
-                       const phi::DenseTensor* v_transpose_out,
-                       const phi::DenseTensor* qkv_transpose_out,
+  void ComputeBackward(const phi::DenseTensor* qkv_transpose_out,
+                       const phi::DenseTensor* src_mask,
+                       const phi::DenseTensor* nonbatched_bias,
+                       const phi::DenseTensor* softmax_lse,
+                       const phi::DenseTensor* fmha_out,
                        const phi::DenseTensor* fmha_out_grad,
                        phi::DenseTensor* src_mask_grad,
                        phi::DenseTensor* nonbatched_bias_grad,
-                       GateAttentionGradConfig<T>* config,
-                       const phi::DenseTensor* fmha_out = nullptr,
-                       const phi::DenseTensor* softmax_lse = nullptr,
-                       const phi::DenseTensor* nonbatched_bias = nullptr,
-                       const phi::DenseTensor* src_mask = nullptr) {
-    T* q_grad_ptr = nullptr;
-    T* k_grad_ptr = nullptr;
-    T* v_grad_ptr = nullptr;
-
-    phi::DenseTensor q_transpose_out_grad;
-    phi::DenseTensor k_transpose_out_grad;
-    phi::DenseTensor v_transpose_out_grad;
-    phi::DenseTensor qkv_transpose_out_grad;
-
+                       GateAttentionGradConfig<T>* config) {
     bool is_bf16 =
         qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
 
@@ -1215,38 +1176,29 @@ class FlashAttnWithGating {
         qkv_transpose_out,
         platform::errors::NotFound("The input qkv_transpose_out can not be"
                                    "nullptr when merge_qkv is true."));
+
     int64_t q_size = config->GetQuerySize();
     const T* q_ptr = qkv_transpose_out->data<T>();
     const T* k_ptr = q_ptr + q_size;
     const T* v_ptr = k_ptr + q_size;
 
+    phi::DenseTensor qkv_transpose_out_grad;
     qkv_transpose_out_grad.Resize(config->qkv_transpose_out_dims);
     AllocWithDebugInfo<T>(
         dev_ctx_, "qkv_transpose_out_grad", &qkv_transpose_out_grad);
 
+    T* q_grad_ptr = qkv_transpose_out_grad.data<T>();
+    T* k_grad_ptr = q_grad_ptr + q_size;
+    T* v_grad_ptr = k_grad_ptr + q_size;
+
     int seq_batch_size = static_cast<int>(config->batch_size) *
                          static_cast<int>(config->seq_len_m);
     LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
-    // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
+    // 2. Init with cu_seq_q and cu_seq_k for flash_attn.
     phi::DenseTensor cu_seq_q, cu_seq_k;
-    int64_t start = 0;
-    int64_t step = static_cast<int>(config->seq_len_r);
-    int64_t end_size = (seq_batch_size + 1);
-    int64_t end = end_size;
-    int64_t seq_size = 0;
-    phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
-    cu_seq_q.Resize({end_size});
-    cu_seq_k.Resize({end_size});
-    AllocWithDebugInfo<int32_t>(dev_ctx_, "Grad: cu_seq_q", &cu_seq_q);
-    AllocWithDebugInfo<int32_t>(dev_ctx_, "Grad: cu_seq_k", &cu_seq_k);
-    int64_t block = std::min(seq_size, static_cast<int64_t>(256));
-    int64_t grid = (seq_size + block - 1) / block;
-    FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
-        start, step, end, cu_seq_q.data<int32_t>(), cu_seq_k.data<int32_t>());
-    VLOG(4) << "[Flash_attn] cu_seq_len : start = " << start
-            << ", step = " << step << ", end = " << end;
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+    int64_t step = static_cast<int64_t>(config->seq_len_r);
+    AllocAndInitSeqQK(seq_batch_size, step, &cu_seq_q, &cu_seq_k);
 
     // 3. Dealing with mask and bias for flash_attn.
     phi::DenseTensor temp_mask, temp_bias;
@@ -1308,27 +1260,17 @@ class FlashAttnWithGating {
     }
     LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
-    q_ptr = q_transpose_out->data<T>();
-    k_ptr = k_transpose_out->data<T>();
-    v_ptr = v_transpose_out->data<T>();
-    q_transpose_out_grad.Resize(config->q_transpose_out_dims);
-    k_transpose_out_grad.Resize(config->kv_transpose_out_dims);
-    v_transpose_out_grad.Resize(config->kv_transpose_out_dims);
-
-    q_grad_ptr = dev_ctx_.Alloc<T>(&q_transpose_out_grad,
-                                   q_transpose_out_grad.numel() * sizeof(T));
-    k_grad_ptr = dev_ctx_.Alloc<T>(&k_transpose_out_grad,
-                                   k_transpose_out_grad.numel() * sizeof(T));
-    v_grad_ptr = dev_ctx_.Alloc<T>(&v_transpose_out_grad,
-                                   v_transpose_out_grad.numel() * sizeof(T));
-
     // 6. construct random seed
-    auto gen = dev_ctx_.GetGenerator();
-    uint64_t inc = batch_size_ * num_heads_ * 32;
-    auto seed_offset_pair = gen->IncrementOffset(inc);
+    auto seed_offset_pair = GenerateSeedOffsetPair(batch_size_, num_heads_);
     uint64_t seed = seed_offset_pair.first;
     uint64_t offset = seed_offset_pair.second;
 
+    LOG(INFO) << "fmha_out: " << TensorDebugString(fmha_out);
+    LOG(INFO) << "fmha_out_grad: " << TensorDebugString(fmha_out_grad);
+    LOG(INFO) << "softmax_lse: " << TensorDebugString(softmax_lse);
+    LOG(INFO) << "softmax_d: " << TensorDebugString(&softmax_d);
+    LOG(INFO) << "bias_d: " << TensorDebugString(&bias_d);
+
     // 7. flas_attn part one, get temp worksapce size.
     uint64_t workspace_size;
     float p_dropout = 0.f;
@@ -1376,15 +1318,7 @@ class FlashAttnWithGating {
     }
     LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
 
-    phi::DenseTensor workspace;
-    printf("workspace_size = %d\n", workspace_size);
-    if (workspace_size > 0) {
-      workspace = phi::Empty<float, phi::GPUContext>(
-          dev_ctx_, {int64_t(workspace_size / sizeof(float))});
-      DBGPTR(workspace.data(), "workspace");
-    }
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
-
+    phi::DenseTensor workspace = CreateWorkspace(workspace_size);
     succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
         static_cast<const void*>(q_ptr),
         static_cast<const void*>(k_ptr),
@@ -1447,23 +1381,55 @@ class FlashAttnWithGating {
           {0});
     }
 
-    if (merge_qkv_) {
-      phi::DenseTensor* qkv_out_grad = config->GetQKVOutGrad();
-      ComputeQKVTransposeBackward(qkv_transpose_out_grad, qkv_out_grad);
-    } else {
-      phi::DenseTensor* q_out_grad = config->GetQueryOutGrad();
-      phi::DenseTensor* k_out_grad = config->GetKeyOutGrad();
-      phi::DenseTensor* v_out_grad = config->GetValueOutGrad();
-      ComputeQKVTransposeBackward(q_transpose_out_grad,
-                                  k_transpose_out_grad,
-                                  v_transpose_out_grad,
-                                  q_out_grad,
-                                  k_out_grad,
-                                  v_out_grad);
-    }
+    phi::DenseTensor* qkv_out_grad = config->GetQKVOutGrad();
+    ComputeQKVTransposeBackward(qkv_transpose_out_grad, qkv_out_grad);
   }
 
  private:
+  void AllocAndInitSeqQK(int64_t seq_batch_size,
+                         int64_t step,
+                         phi::DenseTensor* cu_seq_q,
+                         phi::DenseTensor* cu_seq_k) {
+    int64_t start = 0;
+    int64_t end_size = seq_batch_size + 1;
+    int64_t end = end_size;
+    int64_t seq_size = 0;
+    phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
+
+    cu_seq_q->Resize({end_size});
+    cu_seq_k->Resize({end_size});
+    AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_q", cu_seq_q);
+    AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_k", cu_seq_k);
+
+    int64_t block = std::min(seq_size, static_cast<int64_t>(256));
+    int64_t grid = (seq_size + block - 1) / block;
+    FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
+        start, step, end, cu_seq_q->data<int32_t>(), cu_seq_k->data<int32_t>());
+
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
+              << "AllocAndInit cu_seq_q and cu_seq_k: start=" << start
+              << ", step=" << step << ", end=" << end;
+  }
+
+  phi::DenseTensor CreateWorkspace(uint64_t workspace_size) {
+    phi::DenseTensor workspace;
+    if (workspace_size > 0) {
+      workspace = phi::Empty<float, phi::GPUContext>(
+          dev_ctx_, {int64_t(workspace_size / sizeof(float))});
+      DBGPTR(workspace.data(), "workspace");
+    }
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
+              << "Allocate workspace: workspace_size=" << workspace_size;
+    return workspace;
+  }
+
+  std::pair<uint64_t, uint64_t> GenerateSeedOffsetPair(int64_t batch_size,
+                                                       int64_t num_heads) {
+    auto gen = dev_ctx_.GetGenerator();
+    uint64_t inc = batch_size * num_heads * 32;
+    return gen->IncrementOffset(inc);
+  }
+
   // [batch_size, seq_len_m, seq_len_r, 3, num_heads, head_dim] ->
   //         [3, batch_size, seq_len_m, seq_len_r, num_heads, head_dim]
   void ComputeQKVTransposeForwardForFlashAttn(
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cc b/paddle/fluid/operators/fused/fused_gate_attention_op.cc
index 9743bc33fd055..7175a20787bc6 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cc
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cc
@@ -314,7 +314,6 @@ class FusedGateAttentionGradOpMaker : public framework::SingleGradOpMaker<T> {
       op->SetInput("QKVTransposeOut", this->Output("QKVTransposeOut"));
 
       if (use_flash_attn) {
-        op->SetInput("NonbatchedBias", this->Input("NonbatchedBias"));
         op->SetInput("SrcMask", this->Input("SrcMask"));
         op->SetInput("SoftmaxLse", this->Output("SoftmaxLse"));
       }
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cu b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
index a0f97ce59109b..261e2e377b5f8 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cu
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
@@ -557,18 +557,15 @@ class FusedGateAttentionGradKernel : public framework::OpKernel<T> {
       const auto *softmax_lse = ctx.Input<phi::DenseTensor>("SoftmaxLse");
 
       auto fmha_compute = FlashAttnWithGating<T>(dev_ctx, merge_qkv);
-      fmha_compute.ComputeBackward(q_transpose_out,
-                                   k_transpose_out,
-                                   v_transpose_out,
-                                   qkv_transpose_out,
+      fmha_compute.ComputeBackward(qkv_transpose_out,
+                                   src_mask,
+                                   non_batched_bias,
+                                   softmax_lse,
+                                   fmha_out,
                                    &fmha_out_grad,
                                    nullptr,
                                    nonbatched_bias_grad,
-                                   &config,
-                                   fmha_out,
-                                   softmax_lse,
-                                   non_batched_bias,
-                                   src_mask);
+                                   &config);
     } else {
       const auto *softmax_out = ctx.Input<phi::DenseTensor>("SoftmaxOut");
 

From 22e96bde5bd2d9ca01c9911c28fb6faa617f0599 Mon Sep 17 00:00:00 2001
From: YuanRisheng <yuanrisheng@baidu.com>
Date: Tue, 25 Apr 2023 15:54:44 +0800
Subject: [PATCH 054/405] [PHI]Add flags macro for PHI (#52991)

* add flags for phi

* fix compile bugs

* fix ci bugs

* fix inference bugs

* fix cinn' bugs

* fix cinn bugs

* perfect code according comment

* fix ci bugs

* fix ci bugs
---
 .../collective/process_group_nccl.cc          |   3 +-
 .../fluid/distributed/collective/reducer.cc   |   3 +-
 .../distributed/fleet_executor/carrier.cc     |   2 +-
 .../ps/service/communicator/communicator.h    |   3 +-
 .../distributed/ps/service/heter_server.h     |   3 +-
 .../ps/table/common_graph_table.cc            |  12 +-
 .../distributed/ps/table/ssd_sparse_table.cc  |   4 +-
 .../distributed/ps/table/tensor_table.cc      |   3 +-
 .../fluid/distributed/ps/table/tensor_table.h |   3 +-
 .../eager_manual/forwards/add_n_fwd_func.cc   |   3 +-
 .../forwards/conv2d_fwd_function.cc           |   3 +-
 .../forwards/sync_batch_norm_fwd_func.cc      |   5 +-
 .../manual/eager_manual/nodes/add_n_node.cc   |   4 +-
 .../manual/eager_manual/nodes/conv2d_nodes.cc |   4 +-
 .../nodes/sync_batch_norm_node.cc             |   3 +-
 .../generator/eager_gen.py                    |   7 +-
 paddle/fluid/eager/nan_inf_utils.cc           |   5 +-
 paddle/fluid/framework/custom_operator.cc     |   4 +-
 paddle/fluid/framework/data_feed.cc           |   2 +-
 paddle/fluid/framework/data_feed.cu           |   9 +-
 paddle/fluid/framework/data_feed.h            |   9 +-
 paddle/fluid/framework/data_set.cc            |   6 +-
 .../framework/details/all_reduce_op_handle.cc |   3 +-
 .../details/computation_op_handle.cc          |   4 +-
 .../details/fused_all_reduce_op_handle.cc     |   3 +-
 .../grad_merge_all_reduce_op_handle.cc        |   3 +-
 .../framework/details/nan_inf_utils_detail.cc |   3 +-
 .../framework/details/nan_inf_utils_detail.cu |   3 +-
 .../framework/details/nan_inf_utils_detail.h  |   4 +-
 .../fluid/framework/details/nccl_op_handle.h  |   3 +-
 .../framework/details/reduce_op_handle.cc     |   2 +-
 .../details/scope_buffered_monitor.cc         |   3 +-
 .../details/sparse_all_reduce_op_handle.cc    |   3 +-
 .../framework/dist_multi_trainer_test.cc      |   2 +-
 paddle/fluid/framework/executor.cc            |   3 +-
 .../cudf/concurrent_unordered_map.cuh.h       |   3 +-
 .../framework/fleet/heter_ps/gpu_graph_node.h |   4 +-
 .../fleet/heter_ps/gpu_graph_utils.h          |   3 +-
 .../fleet/heter_ps/graph_gpu_ps_table.h       |   3 +-
 .../fleet/heter_ps/graph_gpu_wrapper.cu       |   5 +-
 .../framework/fleet/heter_ps/heter_comm_inl.h |  19 +-
 .../fleet/heter_ps/heter_resource.cc          |   5 +-
 .../fluid/framework/fleet/ps_gpu_wrapper.cc   |   5 +-
 paddle/fluid/framework/fleet/ps_gpu_wrapper.h |   4 +-
 paddle/fluid/framework/garbage_collector.cc   |   7 +-
 paddle/fluid/framework/hogwild_worker.cc      |   3 +-
 .../framework/ir/coalesce_grad_tensor_pass.cc |   2 +-
 paddle/fluid/framework/ir/graph.cc            |   2 +-
 paddle/fluid/framework/ir/graph_helper.cc     |   2 +-
 ...est_reference_count_pass_last_lived_ops.cc |   3 +-
 .../new_executor/executor_statistics.cc       |   2 +-
 .../garbage_collector/garbage_collector.cc    |   5 +-
 .../interpreter/dependency_builder.cc         |   2 +-
 .../interpreter/interpreter_util.cc           |   5 +-
 .../framework/new_executor/interpretercore.cc |   7 +-
 paddle/fluid/framework/op_registry.h          |   6 +-
 paddle/fluid/framework/operator.cc            |   7 +-
 paddle/fluid/framework/operator.h             |   3 +-
 .../framework/paddle2cinn/cinn_compiler.cc    |   7 +-
 .../paddle2cinn/cinn_compiler_test.cc         |   5 +-
 paddle/fluid/framework/parallel_executor.cc   |   5 +-
 paddle/fluid/framework/scope.cc               |   2 +-
 paddle/fluid/framework/unused_var_check.cc    |   3 +-
 paddle/fluid/imperative/basic_engine.cc       |   3 +-
 paddle/fluid/imperative/flags.cc              |   2 +-
 paddle/fluid/imperative/layer.cc              |   3 +-
 .../fluid/imperative/partial_grad_engine.cc   |   3 +-
 paddle/fluid/imperative/prepared_operator.cc  |   5 +-
 paddle/fluid/imperative/prepared_operator.h   |   3 +-
 paddle/fluid/imperative/profiler.cc           |   3 +-
 paddle/fluid/imperative/tracer.cc             |   7 +-
 paddle/fluid/inference/api/analysis_config.cc |   3 +-
 .../api/demo_ci/trt_mobilenet_demo.cc         |   2 -
 .../fluid/inference/api/demo_ci/vis_demo.cc   |   3 -
 paddle/fluid/inference/tensorrt/engine.h      |   3 +-
 paddle/fluid/jit/serializer.cc                |   3 +-
 paddle/fluid/memory/allocation/allocator.h    |   3 +-
 .../memory/allocation/allocator_facade.cc     |   6 +-
 .../allocator_facade_abs_flags_test.cc        |  11 +-
 .../allocator_facade_frac_flags_test.cc       |  11 +-
 .../memory/allocation/allocator_strategy.cc   |   4 +-
 .../auto_growth_best_fit_allocator.cc         |   2 +-
 ...o_growth_best_fit_allocator_facade_test.cc |   8 +-
 .../memory/allocation/buddy_allocator.cc      |   4 +-
 .../memory/allocation/buddy_allocator_test.cc |   8 +-
 .../fluid/memory/allocation/mmap_allocator.cc |   3 +-
 .../allocation/naive_best_fit_allocator.cc    |   9 +-
 .../memory/allocation/system_allocator.cc     |  11 +-
 .../allocation/system_allocator_test.cc       |   6 +-
 .../allocation/thread_local_allocator_test.cc |   5 +-
 paddle/fluid/operators/activation_op.cc       |   3 +-
 paddle/fluid/operators/batch_norm_op.cu       |   3 +-
 paddle/fluid/operators/cinn/cinn_launch_op.cc |   3 +-
 paddle/fluid/operators/cinn/cinn_launch_op.h  |   6 +-
 .../operators/cinn/cinn_launch_op_test.cc     |  10 +-
 .../controlflow/conditional_block_infer_op.cc |   3 +-
 .../controlflow/conditional_block_op.cc       |   2 +-
 .../fluid/operators/controlflow/while_op.cc   |   1 +
 .../operators/fused/cudnn_bn_add_relu_test.cc |   3 +-
 .../operators/fused/fused_bn_activation_op.cu |   3 +-
 .../fused/fused_bn_add_activation_op.cu       |   3 +-
 .../operators/fused/fused_dropout_helper.h    |   3 +-
 .../fused/fused_multi_transformer_op.cu.h     |   3 +-
 .../pscore/heter_listen_and_serv_op.cc        |   2 +-
 .../pscore/heter_listen_and_server_test.cc    |   5 +-
 paddle/fluid/operators/run_program_op.h       |   3 +-
 paddle/fluid/platform/cpu_info_test.cc        |   3 +-
 .../platform/cuda_graph_with_memory_pool.cc   |   3 +-
 paddle/fluid/platform/device/gpu/gpu_info.cc  |  12 +-
 paddle/fluid/platform/device_code.cc          |   2 +-
 paddle/fluid/platform/enforce.h               |   2 +-
 paddle/fluid/platform/flags.h                 |  77 ++++
 paddle/fluid/platform/gen_comm_id_helper.cc   |   4 +-
 paddle/fluid/platform/init.cc                 |   3 +-
 paddle/fluid/platform/place.cc                |   2 +-
 paddle/fluid/platform/profiler.cc             |   2 +-
 paddle/fluid/platform/profiler/host_tracer.cc |   2 +-
 .../fluid/platform/test_limit_gpu_memory.cu   |   5 +-
 .../utils/static/composite_grad_desc_maker.h  |   2 +-
 paddle/fluid/prim/utils/utils.cc              |   2 +-
 paddle/fluid/pybind/eager_functions.cc        |   4 +-
 paddle/fluid/pybind/eager_method.cc           |   3 +-
 paddle/fluid/pybind/parallel_executor.cc      |   3 +-
 paddle/fluid/pybind/place.cc                  |   3 +-
 paddle/fluid/pybind/pybind.cc                 |   3 +-
 paddle/fluid/pybind/reader_py.cc              |   3 +-
 paddle/fluid/pybind/tensor.cc                 |   5 +-
 paddle/phi/api/profiler/profiler.cc           |  12 +-
 paddle/phi/api/profiler/profiler.h            |   4 +-
 paddle/phi/backends/cpu/cpu_info.cc           |   6 +-
 paddle/phi/backends/dynload/dynamic_loader.cc |  86 ++--
 paddle/phi/backends/xpu/xpu_info.cc           |   2 +-
 paddle/phi/core/flags.cc                      | 387 +++++++++---------
 paddle/phi/core/flags.h                       | 145 ++++++-
 paddle/phi/kernels/funcs/jit/gen_base.cc      |   2 +-
 paddle/phi/kernels/funcs/jit/gen_base.h       |   3 +-
 paddle/utils/pybind.cc                        |   4 +-
 test/cpp/inference/test_helper.h              |   3 +-
 test/cpp/prim/test_eager_prim.cc              |   4 +-
 test/cpp/prim/test_static_prim.cc             |   3 +-
 140 files changed, 761 insertions(+), 503 deletions(-)
 create mode 100644 paddle/fluid/platform/flags.h

diff --git a/paddle/fluid/distributed/collective/process_group_nccl.cc b/paddle/fluid/distributed/collective/process_group_nccl.cc
index 4653799401bbe..ed6118287fa91 100644
--- a/paddle/fluid/distributed/collective/process_group_nccl.cc
+++ b/paddle/fluid/distributed/collective/process_group_nccl.cc
@@ -23,9 +23,10 @@
 #include "paddle/phi/core/distributed/check/nccl_dynamic_check.h"
 #include "paddle/phi/core/distributed/check/static_check.h"
 #include "paddle/phi/core/enforce.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/utils/data_type.h"
 
-DECLARE_bool(nccl_blocking_wait);
+PHI_DECLARE_bool(nccl_blocking_wait);
 DECLARE_bool(use_stream_safe_cuda_allocator);
 
 // set this flag to `true` and recompile to enable dynamic checks
diff --git a/paddle/fluid/distributed/collective/reducer.cc b/paddle/fluid/distributed/collective/reducer.cc
index defc84fbe3d9c..260da509a17cb 100644
--- a/paddle/fluid/distributed/collective/reducer.cc
+++ b/paddle/fluid/distributed/collective/reducer.cc
@@ -15,9 +15,10 @@
 #include "paddle/fluid/distributed/collective/reducer.h"
 #include "paddle/phi/backends/device_guard.h"
 #include "paddle/phi/backends/device_manager.h"
+#include "paddle/phi/core/flags.h"
 
 DECLARE_bool(use_stream_safe_cuda_allocator);
-DECLARE_string(allocator_strategy);
+PHI_DECLARE_string(allocator_strategy);
 
 namespace paddle {
 namespace distributed {
diff --git a/paddle/fluid/distributed/fleet_executor/carrier.cc b/paddle/fluid/distributed/fleet_executor/carrier.cc
index ba6ab7aa55e38..82d99a3835230 100644
--- a/paddle/fluid/distributed/fleet_executor/carrier.cc
+++ b/paddle/fluid/distributed/fleet_executor/carrier.cc
@@ -28,7 +28,7 @@
 #include "paddle/fluid/framework/scope.h"
 #include "paddle/fluid/framework/variable.h"
 #include "paddle/fluid/framework/variable_helper.h"
-
+#include "paddle/fluid/platform/flags.h"
 PADDLE_DEFINE_EXPORTED_bool(
     fleet_executor_with_standalone,
     false,
diff --git a/paddle/fluid/distributed/ps/service/communicator/communicator.h b/paddle/fluid/distributed/ps/service/communicator/communicator.h
index 5247b9a3a3554..f3aa23a77826d 100644
--- a/paddle/fluid/distributed/ps/service/communicator/communicator.h
+++ b/paddle/fluid/distributed/ps/service/communicator/communicator.h
@@ -41,6 +41,7 @@ limitations under the License. */
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/platform/place.h"
 #include "paddle/fluid/string/split.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/funcs/blas/blas.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
 #include "paddle/phi/kernels/funcs/selected_rows_functor.h"
@@ -52,7 +53,7 @@ struct CommContext;
 }  // namespace distributed
 }  // namespace paddle
 
-DECLARE_bool(communicator_is_sgd_optimizer);
+PHI_DECLARE_bool(communicator_is_sgd_optimizer);
 
 namespace paddle {
 namespace distributed {
diff --git a/paddle/fluid/distributed/ps/service/heter_server.h b/paddle/fluid/distributed/ps/service/heter_server.h
index 73e51871c964e..88d37e6022ed5 100755
--- a/paddle/fluid/distributed/ps/service/heter_server.h
+++ b/paddle/fluid/distributed/ps/service/heter_server.h
@@ -38,6 +38,7 @@ limitations under the License. */
 #include "paddle/fluid/framework/variable_helper.h"
 #include "paddle/fluid/platform/macros.h"  // for DISABLE_COPY_AND_ASSIGN
 #include "paddle/fluid/platform/profiler.h"
+#include "paddle/phi/core/flags.h"
 
 namespace google {
 namespace protobuf {
@@ -52,7 +53,7 @@ class ProgramDesc;
 class Scope;
 }  // namespace framework
 }  // namespace paddle
-DECLARE_double(eager_delete_tensor_gb);
+PHI_DECLARE_double(eager_delete_tensor_gb);
 namespace paddle {
 namespace distributed {
 
diff --git a/paddle/fluid/distributed/ps/table/common_graph_table.cc b/paddle/fluid/distributed/ps/table/common_graph_table.cc
index f2b54def608af..13b44d409fad7 100644
--- a/paddle/fluid/distributed/ps/table/common_graph_table.cc
+++ b/paddle/fluid/distributed/ps/table/common_graph_table.cc
@@ -21,7 +21,6 @@
 #include <set>
 #include <sstream>
 
-#include "gflags/gflags.h"
 #include "paddle/fluid/distributed/common/utils.h"
 #include "paddle/fluid/distributed/ps/table/graph/graph_node.h"
 #include "paddle/fluid/framework/fleet/fleet_wrapper.h"
@@ -30,13 +29,14 @@
 #include "paddle/fluid/platform/timer.h"
 #include "paddle/fluid/string/printf.h"
 #include "paddle/fluid/string/string_helper.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/generator.h"
 
-DECLARE_bool(graph_load_in_parallel);
-DECLARE_bool(graph_get_neighbor_id);
-DECLARE_int32(gpugraph_storage_mode);
-DECLARE_uint64(gpugraph_slot_feasign_max_num);
-DECLARE_bool(graph_metapath_split_opt);
+PHI_DECLARE_bool(graph_load_in_parallel);
+PHI_DECLARE_bool(graph_get_neighbor_id);
+PHI_DECLARE_int32(gpugraph_storage_mode);
+PHI_DECLARE_uint64(gpugraph_slot_feasign_max_num);
+PHI_DECLARE_bool(graph_metapath_split_opt);
 
 namespace paddle {
 namespace distributed {
diff --git a/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc b/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc
index aa626a5e49fb5..fbc6af49870d5 100644
--- a/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc
+++ b/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc
@@ -18,8 +18,8 @@
 #include "paddle/fluid/distributed/common/local_random.h"
 #include "paddle/fluid/distributed/common/topk_calculator.h"
 #include "paddle/fluid/framework/archive.h"
+#include "paddle/fluid/platform/flags.h"
 #include "paddle/utils/string/string_helper.h"
-
 DECLARE_bool(pserver_print_missed_key_num_every_push);
 DECLARE_bool(pserver_create_value_when_push);
 DECLARE_bool(pserver_enable_create_feasign_randomly);
@@ -549,7 +549,6 @@ int32_t SSDSparseTable::Shrink(const std::string& param) {
 }
 
 int32_t SSDSparseTable::UpdateTable() {
-  // TODO implement with multi-thread
   int count = 0;
   for (int i = 0; i < _real_local_shard_num; ++i) {
     auto& shard = _local_shards[i];
@@ -578,7 +577,6 @@ int64_t SSDSparseTable::LocalSize() {
   for (int i = 0; i < _real_local_shard_num; ++i) {
     local_size += _local_shards[i].size();
   }
-  // TODO rocksdb size
   return local_size;
 }
 
diff --git a/paddle/fluid/distributed/ps/table/tensor_table.cc b/paddle/fluid/distributed/ps/table/tensor_table.cc
index 187c7021d0281..adda11a78f637 100644
--- a/paddle/fluid/distributed/ps/table/tensor_table.cc
+++ b/paddle/fluid/distributed/ps/table/tensor_table.cc
@@ -13,8 +13,9 @@
 // limitations under the License.
 
 #include "paddle/fluid/distributed/ps/table/tensor_table.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_double(eager_delete_tensor_gb);
+PHI_DECLARE_double(eager_delete_tensor_gb);
 namespace paddle {
 namespace distributed {}  // namespace distributed
 }  // namespace paddle
diff --git a/paddle/fluid/distributed/ps/table/tensor_table.h b/paddle/fluid/distributed/ps/table/tensor_table.h
index 9fb29d727cf9a..0139bd2737ea9 100644
--- a/paddle/fluid/distributed/ps/table/tensor_table.h
+++ b/paddle/fluid/distributed/ps/table/tensor_table.h
@@ -28,6 +28,7 @@
 #include "paddle/fluid/framework/executor.h"
 #include "paddle/fluid/framework/program_desc.h"
 #include "paddle/fluid/platform/device_context.h"
+#include "paddle/phi/core/flags.h"
 
 namespace paddle {
 namespace framework {
@@ -37,7 +38,7 @@ struct ExecutorPrepareContext;
 }  // namespace framework
 }  // namespace paddle
 
-DECLARE_double(eager_delete_tensor_gb);
+PHI_DECLARE_double(eager_delete_tensor_gb);
 
 namespace paddle {
 namespace distributed {
diff --git a/paddle/fluid/eager/api/manual/eager_manual/forwards/add_n_fwd_func.cc b/paddle/fluid/eager/api/manual/eager_manual/forwards/add_n_fwd_func.cc
index 5804d37540957..d14832b80a1db 100644
--- a/paddle/fluid/eager/api/manual/eager_manual/forwards/add_n_fwd_func.cc
+++ b/paddle/fluid/eager/api/manual/eager_manual/forwards/add_n_fwd_func.cc
@@ -19,8 +19,9 @@
 #include "paddle/fluid/eager/eager_amp_auto_cast.h"
 #include "paddle/fluid/eager/nan_inf_utils.h"
 #include "paddle/fluid/platform/profiler/event_tracing.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(check_nan_inf);
+PHI_DECLARE_bool(check_nan_inf);
 
 paddle::Tensor add_n_ad_func(const std::vector<paddle::Tensor>& x) {
   // Dygraph Record Event
diff --git a/paddle/fluid/eager/api/manual/eager_manual/forwards/conv2d_fwd_function.cc b/paddle/fluid/eager/api/manual/eager_manual/forwards/conv2d_fwd_function.cc
index 39fa77d3ae95f..b7ca5a7c26710 100644
--- a/paddle/fluid/eager/api/manual/eager_manual/forwards/conv2d_fwd_function.cc
+++ b/paddle/fluid/eager/api/manual/eager_manual/forwards/conv2d_fwd_function.cc
@@ -20,8 +20,9 @@
 #include "paddle/fluid/eager/eager_layout_auto_tune.h"
 #include "paddle/fluid/eager/nan_inf_utils.h"
 #include "paddle/fluid/platform/profiler/event_tracing.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(check_nan_inf);
+PHI_DECLARE_bool(check_nan_inf);
 
 paddle::Tensor conv2d_ad_func(const paddle::Tensor& input,
                               const paddle::Tensor& filter,
diff --git a/paddle/fluid/eager/api/manual/eager_manual/forwards/sync_batch_norm_fwd_func.cc b/paddle/fluid/eager/api/manual/eager_manual/forwards/sync_batch_norm_fwd_func.cc
index 5bd9571dbf195..a5b0c4d70b07f 100644
--- a/paddle/fluid/eager/api/manual/eager_manual/forwards/sync_batch_norm_fwd_func.cc
+++ b/paddle/fluid/eager/api/manual/eager_manual/forwards/sync_batch_norm_fwd_func.cc
@@ -21,10 +21,11 @@
 #include "paddle/fluid/eager/nan_inf_utils.h"
 #include "paddle/fluid/platform/profiler/event_tracing.h"
 #include "paddle/phi/api/include/sparse_api.h"
+#include "paddle/phi/core/flags.h"
 
 #pragma GCC diagnostic ignored "-Wunused-variable"
-DECLARE_bool(check_nan_inf);
-DECLARE_string(tensor_operants_mode);
+PHI_DECLARE_bool(check_nan_inf);
+PHI_DECLARE_string(tensor_operants_mode);
 
 std::tuple<paddle::Tensor,
            paddle::Tensor&,
diff --git a/paddle/fluid/eager/api/manual/eager_manual/nodes/add_n_node.cc b/paddle/fluid/eager/api/manual/eager_manual/nodes/add_n_node.cc
index 896660c45f6f5..d63c9e0161f2b 100644
--- a/paddle/fluid/eager/api/manual/eager_manual/nodes/add_n_node.cc
+++ b/paddle/fluid/eager/api/manual/eager_manual/nodes/add_n_node.cc
@@ -22,7 +22,9 @@
 #include "paddle/fluid/imperative/tracer.h"
 #include "paddle/phi/api/all.h"
 #include "paddle/phi/api/lib/api_custom_impl.h"
-DECLARE_bool(check_nan_inf);
+#include "paddle/phi/core/flags.h"
+
+PHI_DECLARE_bool(check_nan_inf);
 
 paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
 AddNGradNodeFinal::operator()(
diff --git a/paddle/fluid/eager/api/manual/eager_manual/nodes/conv2d_nodes.cc b/paddle/fluid/eager/api/manual/eager_manual/nodes/conv2d_nodes.cc
index ce06b02728719..0d9862b7b2f2d 100644
--- a/paddle/fluid/eager/api/manual/eager_manual/nodes/conv2d_nodes.cc
+++ b/paddle/fluid/eager/api/manual/eager_manual/nodes/conv2d_nodes.cc
@@ -26,7 +26,9 @@
 
 #include "paddle/fluid/eager/api/manual/eager_manual/nodes/nodes.h"
 #include "paddle/phi/api/include/sparse_api.h"
-DECLARE_bool(check_nan_inf);
+#include "paddle/phi/core/flags.h"
+
+PHI_DECLARE_bool(check_nan_inf);
 
 paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
 Conv2dGradNodeFinal::operator()(
diff --git a/paddle/fluid/eager/api/manual/eager_manual/nodes/sync_batch_norm_node.cc b/paddle/fluid/eager/api/manual/eager_manual/nodes/sync_batch_norm_node.cc
index 4fc847a1ecd8b..4242f74c81c08 100644
--- a/paddle/fluid/eager/api/manual/eager_manual/nodes/sync_batch_norm_node.cc
+++ b/paddle/fluid/eager/api/manual/eager_manual/nodes/sync_batch_norm_node.cc
@@ -25,8 +25,9 @@
 #include "paddle/phi/api/backward/sparse_bw_api.h"
 #include "paddle/phi/api/include/sparse_api.h"
 #include "paddle/phi/api/lib/api_custom_impl.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(check_nan_inf);
+PHI_DECLARE_bool(check_nan_inf);
 
 paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
 SyncBatchNormGradNode::operator()(
diff --git a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
index aa4afa965b8c1..c5db4aee73de0 100644
--- a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
+++ b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
@@ -361,7 +361,8 @@ class {} : public egr::GradNodeBase {{
 #include "paddle/fluid/prim/api/composite_backward/composite_backward_api.h"
 #include "paddle/fluid/prim/api/all.h"
 #include "paddle/fluid/prim/utils/utils.h"
-DECLARE_bool(check_nan_inf);
+#include "paddle/phi/core/flags.h"
+PHI_DECLARE_bool(check_nan_inf);
 {}
 """
 
@@ -388,8 +389,10 @@ class {} : public egr::GradNodeBase {{
 #include "paddle/phi/backends/gpu/gpu_info.h"
 #include "paddle/fluid/eager/nan_inf_utils.h"
 #include "paddle/fluid/eager/api/manual/eager_manual/dygraph_forward_api.h"
+#include "paddle/phi/core/flags.h"
+
 DECLARE_bool(check_nan_inf);
-DECLARE_string(tensor_operants_mode);
+PHI_DECLARE_string(tensor_operants_mode);
 {}
 {}
 """
diff --git a/paddle/fluid/eager/nan_inf_utils.cc b/paddle/fluid/eager/nan_inf_utils.cc
index e71ae7cf11939..c183c64fa7a1d 100644
--- a/paddle/fluid/eager/nan_inf_utils.cc
+++ b/paddle/fluid/eager/nan_inf_utils.cc
@@ -17,11 +17,12 @@
 #include "paddle/fluid/framework/details/nan_inf_utils_detail.h"
 #include "paddle/phi/backends/cpu/cpu_context.h"
 #include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/compat/convert_utils.h"
 #include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/selected_rows.h"
 
-#include "paddle/phi/core/compat/convert_utils.h"
-DECLARE_int32(check_nan_inf_level);
+PHI_DECLARE_int32(check_nan_inf_level);
 namespace egr {
 
 static std::unordered_set<std::string>& nan_inf_check_op_list() {
diff --git a/paddle/fluid/framework/custom_operator.cc b/paddle/fluid/framework/custom_operator.cc
index 18ae5627633a0..ebfed9a6f73f6 100644
--- a/paddle/fluid/framework/custom_operator.cc
+++ b/paddle/fluid/framework/custom_operator.cc
@@ -44,11 +44,11 @@ limitations under the License. */
 #include "paddle/phi/backends/device_manager.h"
 #endif
 
-#include "gflags/gflags.h"
 #include "paddle/phi/api/include/operants_manager.h"
 #include "paddle/phi/api/include/tensor_operants.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_string(tensor_operants_mode);
+PHI_DECLARE_string(tensor_operants_mode);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/data_feed.cc b/paddle/fluid/framework/data_feed.cc
index 63adcd59912aa..32c4845bd0d57 100644
--- a/paddle/fluid/framework/data_feed.cc
+++ b/paddle/fluid/framework/data_feed.cc
@@ -30,7 +30,7 @@ limitations under the License. */
 #include "paddle/fluid/platform/timer.h"
 
 USE_INT_STAT(STAT_total_feasign_num_in_mem);
-DECLARE_bool(enable_ins_parser_file);
+PHI_DECLARE_bool(enable_ins_parser_file);
 namespace paddle {
 namespace framework {
 
diff --git a/paddle/fluid/framework/data_feed.cu b/paddle/fluid/framework/data_feed.cu
index f9cf10a1aabbe..b84529066fd8c 100644
--- a/paddle/fluid/framework/data_feed.cu
+++ b/paddle/fluid/framework/data_feed.cu
@@ -31,13 +31,14 @@ limitations under the License. */
 #include "paddle/fluid/framework/fleet/heter_ps/hashtable.h"
 #include "paddle/fluid/framework/fleet/ps_gpu_wrapper.h"
 #include "paddle/fluid/framework/io/fs.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/gpu/graph_reindex_funcs.h"
 #include "paddle/phi/kernels/graph_reindex_kernel.h"
 
-DECLARE_bool(enable_opt_get_features);
-DECLARE_bool(graph_metapath_split_opt);
-DECLARE_int32(gpugraph_storage_mode);
-DECLARE_double(gpugraph_hbm_table_load_factor);
+PHI_DECLARE_bool(enable_opt_get_features);
+PHI_DECLARE_bool(graph_metapath_split_opt);
+PHI_DECLARE_int32(gpugraph_storage_mode);
+PHI_DECLARE_double(gpugraph_hbm_table_load_factor);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/data_feed.h b/paddle/fluid/framework/data_feed.h
index 415508944c947..a6cf44234773b 100644
--- a/paddle/fluid/framework/data_feed.h
+++ b/paddle/fluid/framework/data_feed.h
@@ -48,11 +48,12 @@ limitations under the License. */
 #include "paddle/fluid/platform/device/gpu/gpu_info.h"
 #include "paddle/phi/core/cuda_stream.h"
 #endif
+#include "paddle/phi/core/flags.h"
 
-DECLARE_int32(record_pool_max_size);
-DECLARE_int32(slotpool_thread_num);
-DECLARE_bool(enable_slotpool_wait_release);
-DECLARE_bool(enable_slotrecord_reset_shrink);
+PHI_DECLARE_int32(record_pool_max_size);
+PHI_DECLARE_int32(slotpool_thread_num);
+PHI_DECLARE_bool(enable_slotpool_wait_release);
+PHI_DECLARE_bool(enable_slotrecord_reset_shrink);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/data_set.cc b/paddle/fluid/framework/data_set.cc
index 5ebd480084ee5..3dc440b1d1c69 100644
--- a/paddle/fluid/framework/data_set.cc
+++ b/paddle/fluid/framework/data_set.cc
@@ -14,7 +14,6 @@
 
 #include "paddle/fluid/framework/data_set.h"
 
-#include "gflags/gflags.h"
 #include "google/protobuf/text_format.h"
 #if (defined PADDLE_WITH_DISTRIBUTE) && (defined PADDLE_WITH_PSCORE)
 #include "paddle/fluid/distributed/index_dataset/index_sampler.h"
@@ -24,6 +23,7 @@
 #include "paddle/fluid/framework/io/fs.h"
 #include "paddle/fluid/platform/monitor.h"
 #include "paddle/fluid/platform/timer.h"
+#include "paddle/phi/core/flags.h"
 
 #ifdef PADDLE_WITH_PSCORE
 #include "paddle/fluid/distributed/ps/wrapper/fleet.h"
@@ -37,8 +37,8 @@
 
 USE_INT_STAT(STAT_total_feasign_num_in_mem);
 USE_INT_STAT(STAT_epoch_finish);
-DECLARE_bool(graph_get_neighbor_id);
-DECLARE_int32(gpugraph_storage_mode);
+PHI_DECLARE_bool(graph_get_neighbor_id);
+PHI_DECLARE_int32(gpugraph_storage_mode);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/details/all_reduce_op_handle.cc b/paddle/fluid/framework/details/all_reduce_op_handle.cc
index 2afeef1efec40..78b8b2e078773 100644
--- a/paddle/fluid/framework/details/all_reduce_op_handle.cc
+++ b/paddle/fluid/framework/details/all_reduce_op_handle.cc
@@ -18,9 +18,10 @@
 #include "paddle/fluid/framework/details/reduce_and_gather.h"
 #include "paddle/fluid/platform/place.h"
 #include "paddle/fluid/platform/profiler/event_tracing.h"
+#include "paddle/phi/core/flags.h"
 
 #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
-DECLARE_bool(sync_nccl_allreduce);
+PHI_DECLARE_bool(sync_nccl_allreduce);
 #endif
 
 namespace paddle {
diff --git a/paddle/fluid/framework/details/computation_op_handle.cc b/paddle/fluid/framework/details/computation_op_handle.cc
index 6d6efeedb2efa..1f807f235ec33 100644
--- a/paddle/fluid/framework/details/computation_op_handle.cc
+++ b/paddle/fluid/framework/details/computation_op_handle.cc
@@ -13,10 +13,10 @@
 // limitations under the License.
 
 #include "paddle/fluid/framework/details/computation_op_handle.h"
-
 #include <string>
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(allreduce_record_one_event);
+PHI_DECLARE_bool(allreduce_record_one_event);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/details/fused_all_reduce_op_handle.cc b/paddle/fluid/framework/details/fused_all_reduce_op_handle.cc
index de9161b1e3312..29d5697b23f0d 100644
--- a/paddle/fluid/framework/details/fused_all_reduce_op_handle.cc
+++ b/paddle/fluid/framework/details/fused_all_reduce_op_handle.cc
@@ -19,9 +19,10 @@
 #include "paddle/fluid/platform/place.h"
 #include "paddle/fluid/platform/profiler/event_tracing.h"
 #include "paddle/phi/backends/device_memory_aligment.h"
+#include "paddle/phi/core/flags.h"
 
 DEFINE_bool(skip_fused_all_reduce_check, false, "");
-DECLARE_bool(allreduce_record_one_event);
+PHI_DECLARE_bool(allreduce_record_one_event);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/details/grad_merge_all_reduce_op_handle.cc b/paddle/fluid/framework/details/grad_merge_all_reduce_op_handle.cc
index 0f79d781d1217..15648aa058f07 100644
--- a/paddle/fluid/framework/details/grad_merge_all_reduce_op_handle.cc
+++ b/paddle/fluid/framework/details/grad_merge_all_reduce_op_handle.cc
@@ -14,9 +14,10 @@
 #include "paddle/fluid/framework/details/grad_merge_all_reduce_op_handle.h"
 
 #include "paddle/fluid/platform/profiler/event_tracing.h"
+#include "paddle/phi/core/flags.h"
 
 #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
-DECLARE_bool(sync_nccl_allreduce);
+PHI_DECLARE_bool(sync_nccl_allreduce);
 #endif
 
 namespace paddle {
diff --git a/paddle/fluid/framework/details/nan_inf_utils_detail.cc b/paddle/fluid/framework/details/nan_inf_utils_detail.cc
index 7890e37e67246..5d6975df9c4d6 100644
--- a/paddle/fluid/framework/details/nan_inf_utils_detail.cc
+++ b/paddle/fluid/framework/details/nan_inf_utils_detail.cc
@@ -20,9 +20,10 @@
 #include "paddle/phi/common/amp_type_traits.h"
 
 #include "paddle/fluid/framework/convert_utils.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/funcs/eigen/extensions.h"
 
-DECLARE_int32(check_nan_inf_level);
+PHI_DECLARE_int32(check_nan_inf_level);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/details/nan_inf_utils_detail.cu b/paddle/fluid/framework/details/nan_inf_utils_detail.cu
index 3e001299e8e38..5569a6f29af90 100644
--- a/paddle/fluid/framework/details/nan_inf_utils_detail.cu
+++ b/paddle/fluid/framework/details/nan_inf_utils_detail.cu
@@ -24,9 +24,10 @@
 #include "paddle/fluid/framework/scope.h"
 #include "paddle/phi/common/amp_type_traits.h"
 #include "paddle/phi/common/memory_utils.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/funcs/math_cuda_utils.h"
 
-DECLARE_int32(check_nan_inf_level);
+PHI_DECLARE_int32(check_nan_inf_level);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/details/nan_inf_utils_detail.h b/paddle/fluid/framework/details/nan_inf_utils_detail.h
index 59865162cc365..f4ee2c20b9d2f 100644
--- a/paddle/fluid/framework/details/nan_inf_utils_detail.h
+++ b/paddle/fluid/framework/details/nan_inf_utils_detail.h
@@ -20,8 +20,8 @@
 #include "paddle/fluid/platform/complex.h"
 #include "paddle/fluid/platform/place.h"
 #include "paddle/phi/common/amp_type_traits.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/funcs/eigen/extensions.h"
-
 #ifdef _WIN32
 #include <direct.h>
 #include <io.h>
@@ -31,7 +31,7 @@
 #define MKDIR(path) mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)
 #endif
 
-DECLARE_int32(check_nan_inf_level);
+PHI_DECLARE_int32(check_nan_inf_level);
 namespace paddle {
 namespace framework {
 namespace details {
diff --git a/paddle/fluid/framework/details/nccl_op_handle.h b/paddle/fluid/framework/details/nccl_op_handle.h
index 5ed4d80203fb3..db7fc45b246c6 100644
--- a/paddle/fluid/framework/details/nccl_op_handle.h
+++ b/paddle/fluid/framework/details/nccl_op_handle.h
@@ -28,8 +28,9 @@
 #include "paddle/fluid/platform/dynload/rccl.h"
 #endif
 #include "paddle/fluid/platform/device/gpu/nccl_helper.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(sync_nccl_allreduce);
+PHI_DECLARE_bool(sync_nccl_allreduce);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/details/reduce_op_handle.cc b/paddle/fluid/framework/details/reduce_op_handle.cc
index a81269b1ecea8..e17f3cbcf6c56 100644
--- a/paddle/fluid/framework/details/reduce_op_handle.cc
+++ b/paddle/fluid/framework/details/reduce_op_handle.cc
@@ -18,9 +18,9 @@
 #include "paddle/fluid/framework/details/container_cast.h"
 #include "paddle/fluid/framework/details/reduce_and_gather.h"
 #include "paddle/fluid/framework/details/variable_visitor.h"
+#include "paddle/fluid/platform/flags.h"
 #include "paddle/fluid/platform/place.h"
 #include "paddle/fluid/platform/profiler/event_tracing.h"
-
 PADDLE_DEFINE_EXPORTED_bool(
     cpu_deterministic,
     false,
diff --git a/paddle/fluid/framework/details/scope_buffered_monitor.cc b/paddle/fluid/framework/details/scope_buffered_monitor.cc
index 88a074763e482..a84fb8eec9eb6 100644
--- a/paddle/fluid/framework/details/scope_buffered_monitor.cc
+++ b/paddle/fluid/framework/details/scope_buffered_monitor.cc
@@ -15,6 +15,7 @@
 #include "paddle/fluid/framework/details/scope_buffered_monitor.h"
 
 #include "paddle/fluid/platform/profiler/event_tracing.h"
+#include "paddle/phi/core/flags.h"
 
 namespace paddle {
 namespace framework {
@@ -22,7 +23,7 @@ class Variable;
 }  // namespace framework
 }  // namespace paddle
 
-DECLARE_double(local_exe_sub_scope_limit);
+PHI_DECLARE_double(local_exe_sub_scope_limit);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/details/sparse_all_reduce_op_handle.cc b/paddle/fluid/framework/details/sparse_all_reduce_op_handle.cc
index d4397225ac81d..d5ab538548361 100644
--- a/paddle/fluid/framework/details/sparse_all_reduce_op_handle.cc
+++ b/paddle/fluid/framework/details/sparse_all_reduce_op_handle.cc
@@ -26,8 +26,9 @@
 #include "paddle/fluid/platform/cuda_device_guard.h"
 #include "paddle/fluid/platform/device/gpu/gpu_info.h"
 #include "paddle/fluid/platform/profiler/event_tracing.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(sync_nccl_allreduce);
+PHI_DECLARE_bool(sync_nccl_allreduce);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/dist_multi_trainer_test.cc b/paddle/fluid/framework/dist_multi_trainer_test.cc
index a96a7c3a24f9b..f4c232e2b6b42 100644
--- a/paddle/fluid/framework/dist_multi_trainer_test.cc
+++ b/paddle/fluid/framework/dist_multi_trainer_test.cc
@@ -22,7 +22,7 @@
 #else
 #define _LINUX
 #endif
-DECLARE_bool(enable_exit_when_partial_worker);
+PHI_DECLARE_bool(enable_exit_when_partial_worker);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/executor.cc b/paddle/fluid/framework/executor.cc
index 573971c6d123b..e0ad2255743c4 100644
--- a/paddle/fluid/framework/executor.cc
+++ b/paddle/fluid/framework/executor.cc
@@ -29,9 +29,10 @@ limitations under the License. */
 #include "paddle/fluid/platform/mkldnn_helper.h"
 #endif
 #include "paddle/fluid/framework/executor_gc_helper.h"
+#include "paddle/phi/core/flags.h"
 
 DECLARE_bool(benchmark);
-DECLARE_bool(use_mkldnn);
+PHI_DECLARE_bool(use_mkldnn);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/fleet/heter_ps/cudf/concurrent_unordered_map.cuh.h b/paddle/fluid/framework/fleet/heter_ps/cudf/concurrent_unordered_map.cuh.h
index b4590548d70fb..2c58573f0153c 100644
--- a/paddle/fluid/framework/fleet/heter_ps/cudf/concurrent_unordered_map.cuh.h
+++ b/paddle/fluid/framework/fleet/heter_ps/cudf/concurrent_unordered_map.cuh.h
@@ -31,6 +31,7 @@
 #include "hash_functions.cuh"
 #include "managed.cuh"
 #include "managed_allocator.cuh"
+#include "paddle/phi/core/flags.h"
 
 // TODO: replace this with CUDA_TRY and propagate the error
 #ifndef CUDA_RT_CALL
@@ -51,7 +52,7 @@
   }
 #endif
 
-DECLARE_bool(gpugraph_enable_hbm_table_collision_stat);
+PHI_DECLARE_bool(gpugraph_enable_hbm_table_collision_stat);
 
 // TODO: can we do this more efficiently?
 __inline__ __device__ int8_t atomicCAS(int8_t* address,
diff --git a/paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h b/paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h
index e67698110b3a3..32c4c404388b5 100644
--- a/paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h
+++ b/paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h
@@ -22,7 +22,9 @@
 #include "paddle/fluid/memory/memory.h"
 #include "paddle/fluid/platform/cuda_device_guard.h"
 #include "paddle/phi/core/enforce.h"
-DECLARE_bool(gpugraph_load_node_list_into_hbm);
+#include "paddle/phi/core/flags.h"
+
+PHI_DECLARE_bool(gpugraph_load_node_list_into_hbm);
 namespace paddle {
 namespace framework {
 struct GpuPsNodeInfo {
diff --git a/paddle/fluid/framework/fleet/heter_ps/gpu_graph_utils.h b/paddle/fluid/framework/fleet/heter_ps/gpu_graph_utils.h
index 655e3c2a5cf0d..fbdcb181e0f28 100644
--- a/paddle/fluid/framework/fleet/heter_ps/gpu_graph_utils.h
+++ b/paddle/fluid/framework/fleet/heter_ps/gpu_graph_utils.h
@@ -22,8 +22,9 @@
 #include <random>
 #include <vector>
 #include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(gpugraph_debug_gpu_memory);
+PHI_DECLARE_bool(gpugraph_debug_gpu_memory);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h
index e32cb1fac4b3a..8bbe85b192a40 100644
--- a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h
+++ b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h
@@ -22,9 +22,10 @@
 #include "paddle/fluid/framework/fleet/heter_ps/heter_comm.h"
 #include "paddle/fluid/framework/fleet/heter_ps/heter_comm_kernel.h"
 #include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/core/flags.h"
 #ifdef PADDLE_WITH_HETERPS
 
-DECLARE_double(gpugraph_hbm_table_load_factor);
+PHI_DECLARE_double(gpugraph_hbm_table_load_factor);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.cu b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.cu
index 2ca1eba77e57a..597db9520ca43 100644
--- a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.cu
+++ b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.cu
@@ -18,8 +18,9 @@
 #include "paddle/fluid/framework/fleet/heter_ps/gpu_graph_utils.h"
 #include "paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h"
 #include "paddle/fluid/framework/fleet/heter_ps/heter_resource.h"
-DECLARE_int32(gpugraph_storage_mode);
-DECLARE_bool(graph_metapath_split_opt);
+#include "paddle/phi/core/flags.h"
+PHI_DECLARE_int32(gpugraph_storage_mode);
+PHI_DECLARE_bool(graph_metapath_split_opt);
 namespace paddle {
 namespace framework {
 #ifdef PADDLE_WITH_HETERPS
diff --git a/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h b/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h
index 61e7f6b0281b4..4c00c1d150205 100644
--- a/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h
+++ b/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h
@@ -25,15 +25,16 @@ limitations under the License. */
 #ifdef PADDLE_WITH_XPU_KP
 #include "paddle/fluid/platform/device/xpu/xpu_info.h"
 #endif
-
-DECLARE_double(gpugraph_hbm_table_load_factor);
-DECLARE_bool(gpugraph_enable_gpu_direct_access);
-DECLARE_bool(gpugraph_enable_segment_merge_grads);
-DECLARE_uint64(gpugraph_merge_grads_segment_size);
-DECLARE_int32(gpugraph_dedup_pull_push_mode);
-DECLARE_bool(enable_tracker_all2all);
-DECLARE_bool(enable_all2all_use_fp16);
-DECLARE_bool(enable_sparse_inner_gather);
+#include "paddle/phi/core/flags.h"
+
+PHI_DECLARE_double(gpugraph_hbm_table_load_factor);
+PHI_DECLARE_bool(gpugraph_enable_gpu_direct_access);
+PHI_DECLARE_bool(gpugraph_enable_segment_merge_grads);
+PHI_DECLARE_uint64(gpugraph_merge_grads_segment_size);
+PHI_DECLARE_int32(gpugraph_dedup_pull_push_mode);
+PHI_DECLARE_bool(enable_tracker_all2all);
+PHI_DECLARE_bool(enable_all2all_use_fp16);
+PHI_DECLARE_bool(enable_sparse_inner_gather);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/fleet/heter_ps/heter_resource.cc b/paddle/fluid/framework/fleet/heter_ps/heter_resource.cc
index 63e9226fcc095..866a1292bcbd8 100644
--- a/paddle/fluid/framework/fleet/heter_ps/heter_resource.cc
+++ b/paddle/fluid/framework/fleet/heter_ps/heter_resource.cc
@@ -23,10 +23,11 @@ limitations under the License. */
 #include "paddle/fluid/platform/device/xpu/enforce_xpu.h"
 #include "paddle/fluid/platform/device/xpu/xpu_info.h"
 #endif
+#include "paddle/phi/core/flags.h"
 #include "paddle/utils/string/string_helper.h"
 
-DECLARE_bool(enable_auto_detect_gpu_topo);
-DECLARE_bool(enable_auto_rdma_trans);
+PHI_DECLARE_bool(enable_auto_detect_gpu_topo);
+PHI_DECLARE_bool(enable_auto_rdma_trans);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/fleet/ps_gpu_wrapper.cc b/paddle/fluid/framework/fleet/ps_gpu_wrapper.cc
index 7ab49bc025e27..5b48d54543b74 100644
--- a/paddle/fluid/framework/fleet/ps_gpu_wrapper.cc
+++ b/paddle/fluid/framework/fleet/ps_gpu_wrapper.cc
@@ -42,9 +42,10 @@ limitations under the License. */
 #if defined(PADDLE_WITH_PSCORE)
 #include "paddle/fluid/distributed/ps/table/depends/feature_value.h"
 #endif
+#include "paddle/phi/core/flags.h"
 
-DECLARE_int32(gpugraph_dedup_pull_push_mode);
-DECLARE_int32(gpugraph_storage_mode);
+PHI_DECLARE_int32(gpugraph_dedup_pull_push_mode);
+PHI_DECLARE_int32(gpugraph_storage_mode);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/fleet/ps_gpu_wrapper.h b/paddle/fluid/framework/fleet/ps_gpu_wrapper.h
index 0d4a6c4871d6a..98c44685ea273 100644
--- a/paddle/fluid/framework/fleet/ps_gpu_wrapper.h
+++ b/paddle/fluid/framework/fleet/ps_gpu_wrapper.h
@@ -66,7 +66,9 @@ limitations under the License. */
 #include "downpour_accessor.h"  // NOLINT
 #endif
 #include "paddle/fluid/framework/fleet/heter_ps/log_patch.h"
-DECLARE_int32(gpugraph_storage_mode);
+#include "paddle/phi/core/flags.h"
+
+PHI_DECLARE_int32(gpugraph_storage_mode);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/garbage_collector.cc b/paddle/fluid/framework/garbage_collector.cc
index 40222dff3088a..3296679e1eeeb 100644
--- a/paddle/fluid/framework/garbage_collector.cc
+++ b/paddle/fluid/framework/garbage_collector.cc
@@ -19,10 +19,11 @@
 #include "gflags/gflags.h"
 #include "paddle/fluid/framework/garbage_collector.h"
 #include "paddle/fluid/platform/device/device_wrapper.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_double(eager_delete_tensor_gb);
-DECLARE_double(memory_fraction_of_eager_deletion);
-DECLARE_bool(fast_eager_deletion_mode);
+PHI_DECLARE_double(eager_delete_tensor_gb);
+PHI_DECLARE_double(memory_fraction_of_eager_deletion);
+PHI_DECLARE_bool(fast_eager_deletion_mode);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/hogwild_worker.cc b/paddle/fluid/framework/hogwild_worker.cc
index b5baeacaec835..ce50efc67008a 100644
--- a/paddle/fluid/framework/hogwild_worker.cc
+++ b/paddle/fluid/framework/hogwild_worker.cc
@@ -29,8 +29,9 @@ limitations under the License. */
 #if defined(PADDLE_WITH_GLOO)
 #include "paddle/fluid/framework/fleet/gloo_wrapper.h"
 #endif
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(enable_exit_when_partial_worker);
+PHI_DECLARE_bool(enable_exit_when_partial_worker);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/ir/coalesce_grad_tensor_pass.cc b/paddle/fluid/framework/ir/coalesce_grad_tensor_pass.cc
index 0c09ceac9955a..9f9d8b2bac258 100644
--- a/paddle/fluid/framework/ir/coalesce_grad_tensor_pass.cc
+++ b/paddle/fluid/framework/ir/coalesce_grad_tensor_pass.cc
@@ -19,7 +19,7 @@
 
 #include "paddle/fluid/framework/details/multi_devices_helper.h"
 #include "paddle/fluid/framework/ir/graph_helper.h"
-
+#include "paddle/fluid/platform/flags.h"
 namespace paddle {
 namespace framework {
 class ProgramDesc;
diff --git a/paddle/fluid/framework/ir/graph.cc b/paddle/fluid/framework/ir/graph.cc
index 5143ccfe4531c..caaba441d1ff3 100644
--- a/paddle/fluid/framework/ir/graph.cc
+++ b/paddle/fluid/framework/ir/graph.cc
@@ -17,7 +17,7 @@ limitations under the License. */
 #include <memory>
 
 #include "paddle/fluid/framework/operator.h"
-
+#include "paddle/fluid/platform/flags.h"
 PADDLE_DEFINE_EXPORTED_bool(convert_all_blocks,
                             true,
                             "Convert all blocks in program into SSAgraphs");
diff --git a/paddle/fluid/framework/ir/graph_helper.cc b/paddle/fluid/framework/ir/graph_helper.cc
index 7e6ef668fb398..04ff8c0190837 100644
--- a/paddle/fluid/framework/ir/graph_helper.cc
+++ b/paddle/fluid/framework/ir/graph_helper.cc
@@ -28,7 +28,7 @@ limitations under the License. */
 #include "paddle/fluid/framework/details/nccl_op_handle.h"
 #include "paddle/fluid/platform/collective_helper.h"
 #endif
-
+#include "paddle/fluid/platform/flags.h"
 DECLARE_bool(convert_all_blocks);
 PADDLE_DEFINE_EXPORTED_string(print_sub_graph_dir,
                               "",
diff --git a/paddle/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc b/paddle/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc
index 02934922f821e..a0f1d9eed0038 100644
--- a/paddle/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc
+++ b/paddle/fluid/framework/ir/memory_optimize_pass/test_reference_count_pass_last_lived_ops.cc
@@ -20,6 +20,7 @@
 #include "paddle/fluid/framework/ir/memory_optimize_pass/reference_count_pass_helper.h"
 #include "paddle/fluid/framework/parallel_executor.h"
 #include "paddle/fluid/framework/program_desc.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/kernel_registry.h"
 
 USE_OP_ITSELF(scale);
@@ -29,7 +30,7 @@ USE_OP_ITSELF(elementwise_add_grad);
 
 PD_DECLARE_KERNEL(scale, CPU, ALL_LAYOUT);
 
-DECLARE_double(eager_delete_tensor_gb);
+PHI_DECLARE_double(eager_delete_tensor_gb);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/new_executor/executor_statistics.cc b/paddle/fluid/framework/new_executor/executor_statistics.cc
index f773a386af473..c1ba3b193f1de 100644
--- a/paddle/fluid/framework/new_executor/executor_statistics.cc
+++ b/paddle/fluid/framework/new_executor/executor_statistics.cc
@@ -24,9 +24,9 @@
 #include <vector>
 
 #include "glog/logging.h"
+#include "paddle/fluid/platform/flags.h"
 #include "paddle/fluid/platform/os_info.h"
 #include "paddle/fluid/platform/profiler/utils.h"
-#include "paddle/phi/core/flags.h"
 
 DECLARE_bool(use_stream_safe_cuda_allocator);
 PADDLE_DEFINE_EXPORTED_string(static_executor_perfstat_filepath,
diff --git a/paddle/fluid/framework/new_executor/garbage_collector/garbage_collector.cc b/paddle/fluid/framework/new_executor/garbage_collector/garbage_collector.cc
index 8ff8b9528322f..ed2d48f4b1d85 100644
--- a/paddle/fluid/framework/new_executor/garbage_collector/garbage_collector.cc
+++ b/paddle/fluid/framework/new_executor/garbage_collector/garbage_collector.cc
@@ -17,9 +17,10 @@
 #include "paddle/fluid/framework/new_executor/garbage_collector/event_garbage_collector.h"
 #include "paddle/fluid/framework/new_executor/garbage_collector/fast_garbage_collector.h"
 #include "paddle/fluid/framework/new_executor/garbage_collector/no_event_garbage_collector.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(fast_eager_deletion_mode);
-DECLARE_bool(new_executor_use_cuda_graph);
+PHI_DECLARE_bool(fast_eager_deletion_mode);
+PHI_DECLARE_bool(new_executor_use_cuda_graph);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/new_executor/interpreter/dependency_builder.cc b/paddle/fluid/framework/new_executor/interpreter/dependency_builder.cc
index 9d9a5e126a9ce..3f0cce3c37830 100644
--- a/paddle/fluid/framework/new_executor/interpreter/dependency_builder.cc
+++ b/paddle/fluid/framework/new_executor/interpreter/dependency_builder.cc
@@ -16,7 +16,7 @@
 
 #include <queue>
 #include "paddle/fluid/framework/new_executor/interpreter/interpreter_util.h"
-
+#include "paddle/fluid/platform/flags.h"
 PADDLE_DEFINE_EXPORTED_bool(
     add_dependency_for_communication_op,
     true,
diff --git a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
index 090e973155334..3e630c32278ed 100644
--- a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
+++ b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
@@ -27,6 +27,7 @@
 #include "paddle/fluid/operators/controlflow/recurrent_op_helper.h"
 #include "paddle/fluid/operators/controlflow/while_op_helper.h"
 #include "paddle/fluid/operators/ops_extra_info.h"
+#include "paddle/fluid/platform/flags.h"
 #include "paddle/phi/core/distributed/comm_context_manager.h"
 #include "paddle/phi/core/kernel_context.h"
 #include "paddle/phi/core/kernel_factory.h"
@@ -40,8 +41,8 @@ PADDLE_DEFINE_EXPORTED_bool(
     false,
     "Log memory stats after each op runs, just used for debug.");
 
-DECLARE_bool(use_mkldnn);
-DECLARE_bool(check_nan_inf);
+PHI_DECLARE_bool(use_mkldnn);
+PHI_DECLARE_bool(check_nan_inf);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/new_executor/interpretercore.cc b/paddle/fluid/framework/new_executor/interpretercore.cc
index d61a0a472d873..494a5afa7ed56 100644
--- a/paddle/fluid/framework/new_executor/interpretercore.cc
+++ b/paddle/fluid/framework/new_executor/interpretercore.cc
@@ -33,6 +33,7 @@
 #include "paddle/fluid/platform/mkldnn_helper.h"
 #endif
 #include "paddle/fluid/platform/cuda_graph_with_memory_pool.h"
+#include "paddle/fluid/platform/flags.h"
 #include "paddle/phi/backends/device_manager.h"
 
 PADDLE_DEFINE_EXPORTED_bool(
@@ -51,11 +52,11 @@ PADDLE_DEFINE_EXPORTED_bool(new_executor_use_local_scope,
                             "Use local_scope in new executor(especially used "
                             "in UT), can turn off for better performance");
 
-DECLARE_bool(check_nan_inf);
+PHI_DECLARE_bool(check_nan_inf);
 DECLARE_bool(benchmark);
-DECLARE_bool(new_executor_use_cuda_graph);
+PHI_DECLARE_bool(new_executor_use_cuda_graph);
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-DECLARE_bool(sync_nccl_allreduce);
+PHI_DECLARE_bool(sync_nccl_allreduce);
 #endif
 
 constexpr const char* kExceptionCaught = "ExceptionCaught";
diff --git a/paddle/fluid/framework/op_registry.h b/paddle/fluid/framework/op_registry.h
index c0c62a6fd2aa8..13ef07ab9855e 100644
--- a/paddle/fluid/framework/op_registry.h
+++ b/paddle/fluid/framework/op_registry.h
@@ -25,8 +25,7 @@ limitations under the License. */
 #include <unordered_set>
 
 #define GLOG_NO_ABBREVIATED_SEVERITIES  // msvc conflict logging with windows.h
-#include "gflags/gflags.h"
-#include "glog/logging.h"  // For VLOG()
+#include "glog/logging.h"               // For VLOG()
 #include "paddle/fluid/framework/attribute.h"
 #include "paddle/fluid/framework/details/op_registry.h"
 #include "paddle/fluid/framework/grad_op_desc_maker.h"
@@ -34,6 +33,7 @@ limitations under the License. */
 #include "paddle/fluid/framework/operator.h"
 #include "paddle/fluid/framework/scope.h"
 #include "paddle/fluid/framework/shape_inference.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/core/macros.h"
 
@@ -70,7 +70,7 @@ class Version;
 }  // namespace framework
 }  // namespace paddle
 
-DECLARE_bool(check_kernel_launch);
+PHI_DECLARE_bool(check_kernel_launch);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc
index c6da5403dc5fc..65d31b87ed355 100644
--- a/paddle/fluid/framework/operator.cc
+++ b/paddle/fluid/framework/operator.cc
@@ -39,6 +39,7 @@ limitations under the License. */
 #include "paddle/phi/common/scalar.h"
 #include "paddle/phi/core/compat/get_kerneltype_forvar_utils.h"
 #include "paddle/phi/core/ddim.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/kernel_context.h"
 #include "paddle/phi/core/kernel_factory.h"
 #include "paddle/phi/ops/compat/signatures.h"
@@ -62,10 +63,10 @@ class DenseTensor;
 #endif
 
 DECLARE_bool(benchmark);
-DECLARE_bool(check_nan_inf);
+PHI_DECLARE_bool(check_nan_inf);
 DECLARE_bool(enable_unused_var_check);
-DECLARE_bool(run_kp_kernel);
-DECLARE_bool(enable_host_event_recorder_hook);
+PHI_DECLARE_bool(run_kp_kernel);
+PHI_DECLARE_bool(enable_host_event_recorder_hook);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/operator.h b/paddle/fluid/framework/operator.h
index f055213f7bbc3..23aac65dd1fdb 100644
--- a/paddle/fluid/framework/operator.h
+++ b/paddle/fluid/framework/operator.h
@@ -42,6 +42,7 @@ limitations under the License. */
 
 #include "paddle/phi/core/compat/arg_map_context.h"
 #include "paddle/phi/core/compat/op_utils.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/kernel_context.h"
 #include "paddle/phi/core/kernel_factory.h"
 #include "paddle/phi/core/macros.h"
@@ -59,7 +60,7 @@ namespace phi {
 class KernelContext;
 }
 
-DECLARE_int32(inner_op_parallelism);
+PHI_DECLARE_int32(inner_op_parallelism);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/paddle2cinn/cinn_compiler.cc b/paddle/fluid/framework/paddle2cinn/cinn_compiler.cc
index 4c1538a28fedb..c0013cfb99eec 100644
--- a/paddle/fluid/framework/paddle2cinn/cinn_compiler.cc
+++ b/paddle/fluid/framework/paddle2cinn/cinn_compiler.cc
@@ -47,10 +47,11 @@
 #include "paddle/fluid/operators/cinn/cinn_launch_context.h"
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/string/string_helper.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(enable_pe_launch_cinn);
-DECLARE_bool(enable_cinn_auto_tune);
-DECLARE_string(cinn_subgraph_graphviz_dir);
+PHI_DECLARE_bool(enable_pe_launch_cinn);
+PHI_DECLARE_bool(enable_cinn_auto_tune);
+PHI_DECLARE_string(cinn_subgraph_graphviz_dir);
 namespace paddle {
 namespace framework {
 namespace paddle2cinn {
diff --git a/paddle/fluid/framework/paddle2cinn/cinn_compiler_test.cc b/paddle/fluid/framework/paddle2cinn/cinn_compiler_test.cc
index 4e9257c0613ef..61f74b22f76b7 100644
--- a/paddle/fluid/framework/paddle2cinn/cinn_compiler_test.cc
+++ b/paddle/fluid/framework/paddle2cinn/cinn_compiler_test.cc
@@ -37,9 +37,10 @@
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/platform/place.h"
 #include "paddle/phi/core/ddim.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_string(allow_cinn_ops);
-DECLARE_string(deny_cinn_ops);
+PHI_DECLARE_string(allow_cinn_ops);
+PHI_DECLARE_string(deny_cinn_ops);
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index 675ec593d9366..3820667116bc6 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -44,11 +44,12 @@ limitations under the License. */
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 #include "paddle/fluid/platform/cuda_device_guard.h"
 #endif
+#include "paddle/fluid/platform/flags.h"
 
-DECLARE_double(eager_delete_tensor_gb);
+PHI_DECLARE_double(eager_delete_tensor_gb);
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-DECLARE_bool(sync_nccl_allreduce);
+PHI_DECLARE_bool(sync_nccl_allreduce);
 #endif
 
 #ifdef WITH_GPERFTOOLS
diff --git a/paddle/fluid/framework/scope.cc b/paddle/fluid/framework/scope.cc
index a54110add67a8..31346652da2b4 100644
--- a/paddle/fluid/framework/scope.cc
+++ b/paddle/fluid/framework/scope.cc
@@ -16,7 +16,7 @@ limitations under the License. */
 
 #include "glog/logging.h"
 #include "paddle/fluid/framework/threadpool.h"
-
+#include "paddle/fluid/platform/flags.h"
 DECLARE_bool(benchmark);
 
 PADDLE_DEFINE_EXPORTED_bool(
diff --git a/paddle/fluid/framework/unused_var_check.cc b/paddle/fluid/framework/unused_var_check.cc
index 1350dba8296ec..ad21fdf45698b 100644
--- a/paddle/fluid/framework/unused_var_check.cc
+++ b/paddle/fluid/framework/unused_var_check.cc
@@ -18,12 +18,11 @@ limitations under the License. */
 
 #include <string>
 
-#include "gflags/gflags.h"
 #include "paddle/fluid/framework/no_need_buffer_vars_inference.h"
 #include "paddle/fluid/framework/op_info.h"
 #include "paddle/fluid/framework/operator.h"
 #include "paddle/fluid/platform/enforce.h"
-
+#include "paddle/fluid/platform/flags.h"
 PADDLE_DEFINE_EXPORTED_bool(
     enable_unused_var_check,
     false,
diff --git a/paddle/fluid/imperative/basic_engine.cc b/paddle/fluid/imperative/basic_engine.cc
index b5433a43ef119..a2cea4067a0ea 100644
--- a/paddle/fluid/imperative/basic_engine.cc
+++ b/paddle/fluid/imperative/basic_engine.cc
@@ -30,10 +30,11 @@
 #include "paddle/fluid/imperative/op_base.h"
 #include "paddle/fluid/imperative/tracer.h"
 #include "paddle/fluid/platform/profiler.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/autotune/switch_autotune.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
 
-DECLARE_bool(sort_sum_gradient);
+PHI_DECLARE_bool(sort_sum_gradient);
 
 namespace paddle {
 namespace imperative {
diff --git a/paddle/fluid/imperative/flags.cc b/paddle/fluid/imperative/flags.cc
index 06c2719ebfa63..f66aacc89ec50 100644
--- a/paddle/fluid/imperative/flags.cc
+++ b/paddle/fluid/imperative/flags.cc
@@ -14,7 +14,7 @@
 
 #include "paddle/fluid/imperative/flags.h"
 
-#include "paddle/phi/core/flags.h"
+#include "paddle/fluid/platform/flags.h"
 
 PADDLE_DEFINE_EXPORTED_uint64(dygraph_debug,
                               0,
diff --git a/paddle/fluid/imperative/layer.cc b/paddle/fluid/imperative/layer.cc
index 2ac43c39d72ab..6538042a37e96 100644
--- a/paddle/fluid/imperative/layer.cc
+++ b/paddle/fluid/imperative/layer.cc
@@ -24,12 +24,13 @@
 #include "paddle/fluid/platform/device_context.h"
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/platform/profiler.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
 #ifdef PADDLE_WITH_MKLDNN
 #include "paddle/fluid/platform/mkldnn_helper.h"
 #endif
 
-DECLARE_bool(use_mkldnn);
+PHI_DECLARE_bool(use_mkldnn);
 namespace paddle {
 namespace imperative {
 
diff --git a/paddle/fluid/imperative/partial_grad_engine.cc b/paddle/fluid/imperative/partial_grad_engine.cc
index 471627e4ca524..0aff56ff88a04 100644
--- a/paddle/fluid/imperative/partial_grad_engine.cc
+++ b/paddle/fluid/imperative/partial_grad_engine.cc
@@ -33,9 +33,10 @@
 #include "paddle/fluid/platform/device_context.h"
 #include "paddle/fluid/platform/profiler/event_tracing.h"
 #include "paddle/fluid/string/string_helper.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
 
-DECLARE_bool(sort_sum_gradient);
+PHI_DECLARE_bool(sort_sum_gradient);
 
 namespace paddle {
 namespace imperative {
diff --git a/paddle/fluid/imperative/prepared_operator.cc b/paddle/fluid/imperative/prepared_operator.cc
index 31a9c28fff68d..cda2fad5d7436 100644
--- a/paddle/fluid/imperative/prepared_operator.cc
+++ b/paddle/fluid/imperative/prepared_operator.cc
@@ -33,10 +33,11 @@
 #include "paddle/fluid/platform/place.h"
 #include "paddle/fluid/platform/profiler/event_tracing.h"
 #include "paddle/fluid/platform/profiler/supplement_tracing.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(check_nan_inf);
+PHI_DECLARE_bool(check_nan_inf);
 DECLARE_bool(benchmark);
-DECLARE_bool(run_kp_kernel);
+PHI_DECLARE_bool(run_kp_kernel);
 
 namespace paddle {
 namespace imperative {
diff --git a/paddle/fluid/imperative/prepared_operator.h b/paddle/fluid/imperative/prepared_operator.h
index bd0f7a060d844..c956230b1447c 100644
--- a/paddle/fluid/imperative/prepared_operator.h
+++ b/paddle/fluid/imperative/prepared_operator.h
@@ -31,10 +31,11 @@
 #include "paddle/fluid/imperative/var_helper.h"
 #include "paddle/phi/common/place.h"
 #include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/kernel_context.h"
 #include "paddle/phi/core/selected_rows.h"
 
-DECLARE_bool(use_mkldnn);
+PHI_DECLARE_bool(use_mkldnn);
 
 namespace paddle {
 namespace imperative {
diff --git a/paddle/fluid/imperative/profiler.cc b/paddle/fluid/imperative/profiler.cc
index fc3066051d853..57f835fe5f6a8 100644
--- a/paddle/fluid/imperative/profiler.cc
+++ b/paddle/fluid/imperative/profiler.cc
@@ -20,8 +20,7 @@
 #include <glog/logging.h>
 
 #include <mutex>  // NOLINT
-
-#include "paddle/phi/core/flags.h"
+#include "paddle/fluid/platform/flags.h"
 
 PADDLE_DEFINE_EXPORTED_string(
     tracer_profile_fname,
diff --git a/paddle/fluid/imperative/tracer.cc b/paddle/fluid/imperative/tracer.cc
index 6207fc54f4d16..2d4e6622c05b7 100644
--- a/paddle/fluid/imperative/tracer.cc
+++ b/paddle/fluid/imperative/tracer.cc
@@ -30,10 +30,11 @@
 #include "paddle/fluid/platform/profiler/event_tracing.h"
 #include "paddle/fluid/string/string_helper.h"
 #include "paddle/phi/common/place.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(use_mkldnn);
-DECLARE_string(tracer_mkldnn_ops_on);
-DECLARE_string(tracer_mkldnn_ops_off);
+PHI_DECLARE_bool(use_mkldnn);
+PHI_DECLARE_string(tracer_mkldnn_ops_on);
+PHI_DECLARE_string(tracer_mkldnn_ops_off);
 
 namespace paddle {
 namespace imperative {
diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc
index b0f53c1f639ac..afb2dcd981fa8 100644
--- a/paddle/fluid/inference/api/analysis_config.cc
+++ b/paddle/fluid/inference/api/analysis_config.cc
@@ -24,6 +24,7 @@
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/platform/errors.h"
 #include "paddle/phi/backends/cpu/cpu_info.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/utils/string/split.h"
 
 #ifdef PADDLE_WITH_TENSORRT
@@ -31,7 +32,7 @@
 #endif
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-DECLARE_uint64(initial_gpu_memory_in_mb);
+PHI_DECLARE_uint64(initial_gpu_memory_in_mb);
 #endif
 
 namespace paddle {
diff --git a/paddle/fluid/inference/api/demo_ci/trt_mobilenet_demo.cc b/paddle/fluid/inference/api/demo_ci/trt_mobilenet_demo.cc
index 44bcc3bf900cb..b6b20a901b2bb 100644
--- a/paddle/fluid/inference/api/demo_ci/trt_mobilenet_demo.cc
+++ b/paddle/fluid/inference/api/demo_ci/trt_mobilenet_demo.cc
@@ -17,11 +17,9 @@ limitations under the License. */
  */
 
 #include <glog/logging.h>  // use glog instead of CHECK to avoid importing other paddle header files.
-
 #include "gflags/gflags.h"
 #include "utils.h"  // NOLINT
 
-DECLARE_double(fraction_of_gpu_memory_to_use);
 DEFINE_string(modeldir, "", "Directory of the inference model.");
 DEFINE_string(refer, "", "path to reference result for comparison.");
 DEFINE_string(data,
diff --git a/paddle/fluid/inference/api/demo_ci/vis_demo.cc b/paddle/fluid/inference/api/demo_ci/vis_demo.cc
index fad5c30e9ff22..7850b4edb1098 100644
--- a/paddle/fluid/inference/api/demo_ci/vis_demo.cc
+++ b/paddle/fluid/inference/api/demo_ci/vis_demo.cc
@@ -21,9 +21,6 @@ limitations under the License. */
 #include "gflags/gflags.h"
 #include "utils.h"  // NOLINT
 
-#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-DECLARE_double(fraction_of_gpu_memory_to_use);
-#endif
 DEFINE_string(modeldir, "", "Directory of the inference model.");
 DEFINE_string(refer, "", "path to reference result for comparison.");
 DEFINE_string(data,
diff --git a/paddle/fluid/inference/tensorrt/engine.h b/paddle/fluid/inference/tensorrt/engine.h
index 05746ea5123c8..8e1531352137f 100644
--- a/paddle/fluid/inference/tensorrt/engine.h
+++ b/paddle/fluid/inference/tensorrt/engine.h
@@ -39,10 +39,11 @@ limitations under the License. */
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/phi/common/data_type.h"
 #include "paddle/phi/common/place.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/stream.h"
 #include "paddle/utils/any.h"
 
-DECLARE_bool(trt_ibuilder_cache);
+PHI_DECLARE_bool(trt_ibuilder_cache);
 
 namespace paddle {
 namespace inference {
diff --git a/paddle/fluid/jit/serializer.cc b/paddle/fluid/jit/serializer.cc
index 21a187ad67100..57f457dbd8122 100644
--- a/paddle/fluid/jit/serializer.cc
+++ b/paddle/fluid/jit/serializer.cc
@@ -25,8 +25,9 @@
 #include "paddle/fluid/jit/layer.h"
 #include "paddle/fluid/jit/property.h"
 #include "paddle/fluid/jit/serializer_utils.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_string(jit_engine_type);
+PHI_DECLARE_string(jit_engine_type);
 
 namespace paddle {
 namespace jit {
diff --git a/paddle/fluid/memory/allocation/allocator.h b/paddle/fluid/memory/allocation/allocator.h
index a7cf653980d96..dea6855cf5abb 100644
--- a/paddle/fluid/memory/allocation/allocator.h
+++ b/paddle/fluid/memory/allocation/allocator.h
@@ -23,8 +23,9 @@
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/platform/place.h"
 #include "paddle/phi/core/allocator.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_string(allocator_strategy);
+PHI_DECLARE_string(allocator_strategy);
 
 namespace paddle {
 namespace memory {
diff --git a/paddle/fluid/memory/allocation/allocator_facade.cc b/paddle/fluid/memory/allocation/allocator_facade.cc
index 1d427008c7e4c..527e843d05bb8 100644
--- a/paddle/fluid/memory/allocation/allocator_facade.cc
+++ b/paddle/fluid/memory/allocation/allocator_facade.cc
@@ -14,7 +14,6 @@
 
 #include "paddle/fluid/memory/allocation/allocator_facade.h"
 
-#include "gflags/gflags.h"
 #include "paddle/fluid/memory/allocation/aligned_allocator.h"
 #include "paddle/fluid/memory/allocation/allocator.h"
 #include "paddle/fluid/memory/allocation/allocator_strategy.h"
@@ -62,6 +61,7 @@
 #include "paddle/fluid/memory/allocation/custom_allocator.h"
 #include "paddle/fluid/platform/device/device_wrapper.h"
 #endif
+#include "paddle/fluid/platform/flags.h"
 
 PADDLE_DEFINE_EXPORTED_int64(
     gpu_allocator_retry_time,
@@ -92,8 +92,8 @@ PADDLE_DEFINE_EXPORTED_bool(use_cuda_managed_memory,
                             "managed memory, only available for auto_growth "
                             "strategy");
 
-DECLARE_string(allocator_strategy);
-DECLARE_uint64(auto_growth_chunk_size_in_mb);
+PHI_DECLARE_string(allocator_strategy);
+PHI_DECLARE_uint64(auto_growth_chunk_size_in_mb);
 
 namespace paddle {
 namespace memory {
diff --git a/paddle/fluid/memory/allocation/allocator_facade_abs_flags_test.cc b/paddle/fluid/memory/allocation/allocator_facade_abs_flags_test.cc
index d3f16ec628660..1e09c43c4f12f 100644
--- a/paddle/fluid/memory/allocation/allocator_facade_abs_flags_test.cc
+++ b/paddle/fluid/memory/allocation/allocator_facade_abs_flags_test.cc
@@ -15,15 +15,16 @@
 #include <gtest/gtest.h>
 
 #include "paddle/fluid/memory/allocation/allocator_facade.h"
+#include "paddle/phi/core/flags.h"
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-DECLARE_double(fraction_of_gpu_memory_to_use);
-DECLARE_double(fraction_of_cuda_pinned_memory_to_use);
-DECLARE_uint64(initial_gpu_memory_in_mb);
-DECLARE_uint64(reallocate_gpu_memory_in_mb);
+PHI_DECLARE_double(fraction_of_gpu_memory_to_use);
+PHI_DECLARE_double(fraction_of_cuda_pinned_memory_to_use);
+PHI_DECLARE_uint64(initial_gpu_memory_in_mb);
+PHI_DECLARE_uint64(reallocate_gpu_memory_in_mb);
 DECLARE_int64(gpu_allocator_retry_time);
 #endif
-DECLARE_string(allocator_strategy);
+PHI_DECLARE_string(allocator_strategy);
 
 namespace paddle {
 namespace memory {
diff --git a/paddle/fluid/memory/allocation/allocator_facade_frac_flags_test.cc b/paddle/fluid/memory/allocation/allocator_facade_frac_flags_test.cc
index 85cd851a2140a..63e3eab3256c9 100644
--- a/paddle/fluid/memory/allocation/allocator_facade_frac_flags_test.cc
+++ b/paddle/fluid/memory/allocation/allocator_facade_frac_flags_test.cc
@@ -15,15 +15,16 @@
 #include <gtest/gtest.h>
 
 #include "paddle/fluid/memory/allocation/allocator_facade.h"
+#include "paddle/phi/core/flags.h"
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-DECLARE_double(fraction_of_gpu_memory_to_use);
-DECLARE_double(fraction_of_cuda_pinned_memory_to_use);
-DECLARE_uint64(initial_gpu_memory_in_mb);
-DECLARE_uint64(reallocate_gpu_memory_in_mb);
+PHI_DECLARE_double(fraction_of_gpu_memory_to_use);
+PHI_DECLARE_double(fraction_of_cuda_pinned_memory_to_use);
+PHI_DECLARE_uint64(initial_gpu_memory_in_mb);
+PHI_DECLARE_uint64(reallocate_gpu_memory_in_mb);
 DECLARE_int64(gpu_allocator_retry_time);
 #endif
-DECLARE_string(allocator_strategy);
+PHI_DECLARE_string(allocator_strategy);
 
 namespace paddle {
 namespace memory {
diff --git a/paddle/fluid/memory/allocation/allocator_strategy.cc b/paddle/fluid/memory/allocation/allocator_strategy.cc
index 518b31e943048..a133db577ee7c 100644
--- a/paddle/fluid/memory/allocation/allocator_strategy.cc
+++ b/paddle/fluid/memory/allocation/allocator_strategy.cc
@@ -14,10 +14,10 @@
 
 #include "paddle/fluid/memory/allocation/allocator_strategy.h"
 
-#include "gflags/gflags.h"
 #include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_string(allocator_strategy);
+PHI_DECLARE_string(allocator_strategy);
 
 namespace paddle {
 namespace memory {
diff --git a/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator.cc b/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator.cc
index cf3bb15fdd8a1..ccbaca8ed7188 100644
--- a/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator.cc
+++ b/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator.cc
@@ -18,8 +18,8 @@
 #include <mutex>  // NOLINT
 
 #include "paddle/fluid/memory/allocation/aligned_allocator.h"
+#include "paddle/fluid/platform/flags.h"
 #include "paddle/fluid/platform/profiler/event_tracing.h"
-#include "paddle/phi/core/flags.h"
 
 PADDLE_DEFINE_EXPORTED_READONLY_bool(
     free_idle_chunk,
diff --git a/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator_facade_test.cc b/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator_facade_test.cc
index 70c43145cc85d..bfd05b6b323fe 100644
--- a/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator_facade_test.cc
+++ b/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator_facade_test.cc
@@ -19,17 +19,17 @@
 #include <random>
 #include <thread>  // NOLINT
 
-#include "gflags/gflags.h"
 #include "paddle/fluid/memory/allocation/allocator_facade.h"
 #include "paddle/fluid/platform/device/gpu/gpu_info.h"
+#include "paddle/phi/core/flags.h"
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-DECLARE_double(fraction_of_gpu_memory_to_use);
-DECLARE_double(fraction_of_cuda_pinned_memory_to_use);
+PHI_DECLARE_double(fraction_of_gpu_memory_to_use);
+PHI_DECLARE_double(fraction_of_cuda_pinned_memory_to_use);
 DECLARE_int64(gpu_allocator_retry_time);
 #endif
 
-DECLARE_string(allocator_strategy);
+PHI_DECLARE_string(allocator_strategy);
 
 namespace paddle {
 namespace memory {
diff --git a/paddle/fluid/memory/allocation/buddy_allocator.cc b/paddle/fluid/memory/allocation/buddy_allocator.cc
index 17edc412271c5..c90c506cc0c4b 100644
--- a/paddle/fluid/memory/allocation/buddy_allocator.cc
+++ b/paddle/fluid/memory/allocation/buddy_allocator.cc
@@ -16,12 +16,12 @@ limitations under the License. */
 
 #include <algorithm>
 
-#include "gflags/gflags.h"
 #include "glog/logging.h"
+#include "paddle/phi/core/flags.h"
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 #define USE_DEVICE
-DECLARE_uint64(reallocate_gpu_memory_in_mb);
+PHI_DECLARE_uint64(reallocate_gpu_memory_in_mb);
 #endif
 
 #include "paddle/fluid/platform/device/device_wrapper.h"
diff --git a/paddle/fluid/memory/allocation/buddy_allocator_test.cc b/paddle/fluid/memory/allocation/buddy_allocator_test.cc
index 79f2e17102622..1aeb1722d0ec8 100644
--- a/paddle/fluid/memory/allocation/buddy_allocator_test.cc
+++ b/paddle/fluid/memory/allocation/buddy_allocator_test.cc
@@ -22,14 +22,14 @@ limitations under the License. */
 #include <fstream>
 #include <string>
 
-#include "gflags/gflags.h"
 #include "gtest/gtest.h"
 #include "paddle/fluid/platform/device/gpu/gpu_info.h"
+#include "paddle/phi/core/flags.h"
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-DECLARE_double(fraction_of_gpu_memory_to_use);
-DECLARE_uint64(initial_gpu_memory_in_mb);
-DECLARE_uint64(reallocate_gpu_memory_in_mb);
+PHI_DECLARE_double(fraction_of_gpu_memory_to_use);
+PHI_DECLARE_uint64(initial_gpu_memory_in_mb);
+PHI_DECLARE_uint64(reallocate_gpu_memory_in_mb);
 #endif
 
 namespace paddle {
diff --git a/paddle/fluid/memory/allocation/mmap_allocator.cc b/paddle/fluid/memory/allocation/mmap_allocator.cc
index 2e091fa1b2f60..1467d4b81081d 100644
--- a/paddle/fluid/memory/allocation/mmap_allocator.cc
+++ b/paddle/fluid/memory/allocation/mmap_allocator.cc
@@ -26,8 +26,9 @@
 
 #include "glog/logging.h"
 #include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(use_shm_cache);
+PHI_DECLARE_bool(use_shm_cache);
 
 namespace paddle {
 namespace memory {
diff --git a/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc b/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc
index 10dfe27391626..8c9eb889add6c 100644
--- a/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc
+++ b/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc
@@ -16,7 +16,6 @@
 
 #include <mutex>
 
-#include "gflags/gflags.h"
 #include "glog/logging.h"
 #include "paddle/fluid/memory/allocation/buddy_allocator.h"
 #include "paddle/fluid/memory/allocation/system_allocator.h"
@@ -30,7 +29,7 @@
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 #include "paddle/fluid/platform/cuda_device_guard.h"
 #endif
-
+#include "paddle/fluid/platform/flags.h"
 PADDLE_DEFINE_EXPORTED_bool(
     init_allocated_mem,
     false,
@@ -39,9 +38,9 @@ PADDLE_DEFINE_EXPORTED_bool(
     "To find this error in time, we use init_allocated_mem to indicate "
     "that initializing the allocated memory with a small value "
     "during unit testing.");
-DECLARE_double(fraction_of_gpu_memory_to_use);
-DECLARE_uint64(initial_gpu_memory_in_mb);
-DECLARE_uint64(reallocate_gpu_memory_in_mb);
+PHI_DECLARE_double(fraction_of_gpu_memory_to_use);
+PHI_DECLARE_uint64(initial_gpu_memory_in_mb);
+PHI_DECLARE_uint64(reallocate_gpu_memory_in_mb);
 DECLARE_bool(benchmark);
 
 namespace paddle {
diff --git a/paddle/fluid/memory/allocation/system_allocator.cc b/paddle/fluid/memory/allocation/system_allocator.cc
index f5069d655f8a7..210be01669775 100644
--- a/paddle/fluid/memory/allocation/system_allocator.cc
+++ b/paddle/fluid/memory/allocation/system_allocator.cc
@@ -26,11 +26,12 @@ limitations under the License. */
 #else
 #include <sys/mman.h>  // for mlock and munlock
 #endif
-#include "gflags/gflags.h"
+
 #include "paddle/fluid/memory/allocation/allocator.h"
 #include "paddle/fluid/platform/device/gpu/gpu_info.h"
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/phi/backends/cpu/cpu_info.h"
+#include "paddle/phi/core/flags.h"
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 #include "paddle/fluid/platform/cuda_device_guard.h"
@@ -39,10 +40,10 @@ limitations under the License. */
 #include "paddle/fluid/platform/device/device_wrapper.h"
 #include "paddle/fluid/platform/profiler/mem_tracing.h"
 
-DECLARE_bool(use_pinned_memory);
-DECLARE_double(fraction_of_gpu_memory_to_use);
-DECLARE_uint64(initial_gpu_memory_in_mb);
-DECLARE_uint64(reallocate_gpu_memory_in_mb);
+PHI_DECLARE_bool(use_pinned_memory);
+PHI_DECLARE_double(fraction_of_gpu_memory_to_use);
+PHI_DECLARE_uint64(initial_gpu_memory_in_mb);
+PHI_DECLARE_uint64(reallocate_gpu_memory_in_mb);
 
 namespace paddle {
 namespace memory {
diff --git a/paddle/fluid/memory/allocation/system_allocator_test.cc b/paddle/fluid/memory/allocation/system_allocator_test.cc
index dbb0165aa6f9f..e04d14f0adfde 100644
--- a/paddle/fluid/memory/allocation/system_allocator_test.cc
+++ b/paddle/fluid/memory/allocation/system_allocator_test.cc
@@ -14,14 +14,14 @@ limitations under the License. */
 
 #include "paddle/fluid/memory/allocation/system_allocator.h"
 
+#include <gtest/gtest.h>
 #include <memory>
 
-#include "gflags/gflags.h"
-#include "gtest/gtest.h"
 #include "paddle/fluid/memory/allocation/allocator.h"
 #include "paddle/fluid/platform/device/device_wrapper.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(use_pinned_memory);
+PHI_DECLARE_bool(use_pinned_memory);
 
 void TestAllocator(paddle::memory::detail::SystemAllocator* a, size_t size) {
   bool freed = false;
diff --git a/paddle/fluid/memory/allocation/thread_local_allocator_test.cc b/paddle/fluid/memory/allocation/thread_local_allocator_test.cc
index c3deef8da6d51..9627d4a2d6808 100644
--- a/paddle/fluid/memory/allocation/thread_local_allocator_test.cc
+++ b/paddle/fluid/memory/allocation/thread_local_allocator_test.cc
@@ -19,9 +19,10 @@
 
 #include "gtest/gtest.h"
 #include "paddle/fluid/memory/malloc.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_double(fraction_of_gpu_memory_to_use);
-DECLARE_string(allocator_strategy);
+PHI_DECLARE_double(fraction_of_gpu_memory_to_use);
+PHI_DECLARE_string(allocator_strategy);
 
 namespace paddle {
 namespace memory {
diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc
index 23f1bb7bdd340..0edcb3726d059 100644
--- a/paddle/fluid/operators/activation_op.cc
+++ b/paddle/fluid/operators/activation_op.cc
@@ -29,7 +29,8 @@ limitations under the License. */
 #include "paddle/phi/backends/dynload/port.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/infermeta/backward.h"
-DECLARE_bool(use_mkldnn);
+
+PHI_DECLARE_bool(use_mkldnn);
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/batch_norm_op.cu b/paddle/fluid/operators/batch_norm_op.cu
index f93cb32a850ef..012edde57294a 100644
--- a/paddle/fluid/operators/batch_norm_op.cu
+++ b/paddle/fluid/operators/batch_norm_op.cu
@@ -27,9 +27,10 @@ namespace cub = hipcub;
 #include "paddle/fluid/operators/batch_norm_op.h"
 #include "paddle/fluid/platform/float16.h"
 #include "paddle/phi/backends/gpu/gpu_dnn.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
 
-DECLARE_bool(cudnn_batchnorm_spatial_persistent);
+PHI_DECLARE_bool(cudnn_batchnorm_spatial_persistent);
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/cinn/cinn_launch_op.cc b/paddle/fluid/operators/cinn/cinn_launch_op.cc
index c6a4228300720..ad74d88f70e1d 100644
--- a/paddle/fluid/operators/cinn/cinn_launch_op.cc
+++ b/paddle/fluid/operators/cinn/cinn_launch_op.cc
@@ -21,9 +21,10 @@
 #include "cinn/runtime/cinn_runtime.h"
 #include "cinn/runtime/flags.h"
 #include "paddle/fluid/string/string_helper.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/generator.h"
 
-DECLARE_bool(cudnn_deterministic);
+PHI_DECLARE_bool(cudnn_deterministic);
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/cinn/cinn_launch_op.h b/paddle/fluid/operators/cinn/cinn_launch_op.h
index d5befbce611c2..59970412ea6a3 100644
--- a/paddle/fluid/operators/cinn/cinn_launch_op.h
+++ b/paddle/fluid/operators/cinn/cinn_launch_op.h
@@ -21,7 +21,6 @@
 #include <unordered_set>
 
 #include "cinn/common/target.h"
-#include "gflags/gflags.h"
 #include "paddle/fluid/framework/data_type.h"
 #include "paddle/fluid/framework/op_registry.h"
 #include "paddle/fluid/framework/operator.h"
@@ -29,9 +28,10 @@
 #include "paddle/fluid/operators/cinn/cinn_launch_context.h"
 #include "paddle/fluid/operators/cinn/cinn_op_helper.h"
 #include "paddle/fluid/platform/profiler.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(enable_pe_launch_cinn);
-DECLARE_bool(enable_interpretercore_launch_cinn);
+PHI_DECLARE_bool(enable_pe_launch_cinn);
+PHI_DECLARE_bool(enable_interpretercore_launch_cinn);
 namespace paddle {
 namespace operators {
 
diff --git a/paddle/fluid/operators/cinn/cinn_launch_op_test.cc b/paddle/fluid/operators/cinn/cinn_launch_op_test.cc
index 29ae8bfe3cd9c..9764c31200f9e 100644
--- a/paddle/fluid/operators/cinn/cinn_launch_op_test.cc
+++ b/paddle/fluid/operators/cinn/cinn_launch_op_test.cc
@@ -20,7 +20,6 @@ limitations under the License. */
 #include <random>
 #include <string>
 
-#include "gflags/gflags.h"
 #include "gtest/gtest.h"
 #include "paddle/fluid/framework/op_registry.h"
 #include "paddle/fluid/framework/paddle2cinn/cinn_compiler.h"
@@ -29,15 +28,16 @@ limitations under the License. */
 #include "paddle/fluid/platform/cpu_helper.h"
 #include "paddle/fluid/platform/init.h"
 #include "paddle/phi/core/ddim.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/kernel_registry.h"
 
 USE_OP_ITSELF(cinn_launch);
 USE_OP_ITSELF(cinn_instruction_run);
 USE_OP_ITSELF(elementwise_add);
-DECLARE_double(eager_delete_tensor_gb);
-DECLARE_bool(enable_pe_launch_cinn);
-DECLARE_bool(enable_interpretercore_launch_cinn);
-DECLARE_bool(enable_cinn_auto_tune);
+PHI_DECLARE_double(eager_delete_tensor_gb);
+PHI_DECLARE_bool(enable_pe_launch_cinn);
+PHI_DECLARE_bool(enable_interpretercore_launch_cinn);
+PHI_DECLARE_bool(enable_cinn_auto_tune);
 
 PD_DECLARE_KERNEL(cinn_launch, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(cinn_instruction_run, CPU, ALL_LAYOUT);
diff --git a/paddle/fluid/operators/controlflow/conditional_block_infer_op.cc b/paddle/fluid/operators/controlflow/conditional_block_infer_op.cc
index 44996e7e4b496..a22df97f1cd57 100644
--- a/paddle/fluid/operators/controlflow/conditional_block_infer_op.cc
+++ b/paddle/fluid/operators/controlflow/conditional_block_infer_op.cc
@@ -17,8 +17,9 @@ limitations under the License. */
 #ifdef PADDLE_WITH_MKLDNN
 #include "paddle/fluid/platform/mkldnn_helper.h"
 #endif
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(use_mkldnn);
+PHI_DECLARE_bool(use_mkldnn);
 namespace paddle {
 namespace framework {
 class OpDesc;
diff --git a/paddle/fluid/operators/controlflow/conditional_block_op.cc b/paddle/fluid/operators/controlflow/conditional_block_op.cc
index ee8ec2e276b61..156ba70467380 100644
--- a/paddle/fluid/operators/controlflow/conditional_block_op.cc
+++ b/paddle/fluid/operators/controlflow/conditional_block_op.cc
@@ -24,7 +24,7 @@ limitations under the License. */
 #include "paddle/fluid/platform/mkldnn_helper.h"
 #endif
 
-DECLARE_bool(use_mkldnn);
+PHI_DECLARE_bool(use_mkldnn);
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/controlflow/while_op.cc b/paddle/fluid/operators/controlflow/while_op.cc
index 4c7578c010473..cb472fc6948ca 100644
--- a/paddle/fluid/operators/controlflow/while_op.cc
+++ b/paddle/fluid/operators/controlflow/while_op.cc
@@ -22,6 +22,7 @@
 #ifdef PADDLE_WITH_MKLDNN
 #include "paddle/fluid/platform/mkldnn_helper.h"
 #endif
+#include "paddle/fluid/platform/flags.h"
 
 PADDLE_DEFINE_EXPORTED_bool(
     cache_inference_while_scope,
diff --git a/paddle/fluid/operators/fused/cudnn_bn_add_relu_test.cc b/paddle/fluid/operators/fused/cudnn_bn_add_relu_test.cc
index 678cb180e79dc..0d88d44defffc 100644
--- a/paddle/fluid/operators/fused/cudnn_bn_add_relu_test.cc
+++ b/paddle/fluid/operators/fused/cudnn_bn_add_relu_test.cc
@@ -23,10 +23,11 @@ limitations under the License. */
 #include "paddle/fluid/operators/fused/cudnn_bn_stats_finalize.cu.h"
 #include "paddle/fluid/operators/fused/cudnn_scale_bias_add_relu.cu.h"
 #include "paddle/fluid/platform/float16.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
 
-DECLARE_bool(cudnn_batchnorm_spatial_persistent);
+PHI_DECLARE_bool(cudnn_batchnorm_spatial_persistent);
 
 namespace framework = paddle::framework;
 namespace platform = paddle::platform;
diff --git a/paddle/fluid/operators/fused/fused_bn_activation_op.cu b/paddle/fluid/operators/fused/fused_bn_activation_op.cu
index ac2a84e473b0f..a93938fcfd043 100644
--- a/paddle/fluid/operators/fused/fused_bn_activation_op.cu
+++ b/paddle/fluid/operators/fused/fused_bn_activation_op.cu
@@ -23,10 +23,11 @@
 #include "paddle/fluid/operators/fused/fused_bn_activation_op.h"
 #include "paddle/fluid/platform/device/gpu/gpu_dnn.h"
 #include "paddle/fluid/platform/float16.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
 #include "paddle/phi/kernels/funcs/norm_utils.h"
 
-DECLARE_bool(cudnn_batchnorm_spatial_persistent);
+PHI_DECLARE_bool(cudnn_batchnorm_spatial_persistent);
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/fused/fused_bn_add_activation_op.cu b/paddle/fluid/operators/fused/fused_bn_add_activation_op.cu
index 4aae200e550da..1fa7ff1826b07 100644
--- a/paddle/fluid/operators/fused/fused_bn_add_activation_op.cu
+++ b/paddle/fluid/operators/fused/fused_bn_add_activation_op.cu
@@ -23,10 +23,11 @@
 #include "paddle/fluid/platform/device/gpu/gpu_dnn.h"
 #include "paddle/fluid/platform/float16.h"
 #include "paddle/phi/common/data_type.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
 #include "paddle/phi/kernels/funcs/norm_utils.h"
 
-DECLARE_bool(cudnn_batchnorm_spatial_persistent);
+PHI_DECLARE_bool(cudnn_batchnorm_spatial_persistent);
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/fused/fused_dropout_helper.h b/paddle/fluid/operators/fused/fused_dropout_helper.h
index 8025ba97ac004..17b6f8c445b1e 100644
--- a/paddle/fluid/operators/fused/fused_dropout_helper.h
+++ b/paddle/fluid/operators/fused/fused_dropout_helper.h
@@ -17,12 +17,13 @@ limitations under the License. */
 #include "paddle/fluid/operators/fused/fused_dropout_act_bias.h"
 #include "paddle/fluid/operators/fused/fused_layernorm_residual_dropout_bias.h"
 #include "paddle/fluid/operators/fused/fused_residual_dropout_bias.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/generator.h"
 #include "paddle/phi/kernels/funcs/dropout_impl_util.h"
 #include "paddle/phi/kernels/funcs/functors.h"
 #include "paddle/phi/kernels/layer_norm_kernel.h"
 
-DECLARE_bool(use_fast_math);
+PHI_DECLARE_bool(use_fast_math);
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/fused/fused_multi_transformer_op.cu.h b/paddle/fluid/operators/fused/fused_multi_transformer_op.cu.h
index 4769433317f0f..459a1f0f7146f 100644
--- a/paddle/fluid/operators/fused/fused_multi_transformer_op.cu.h
+++ b/paddle/fluid/operators/fused/fused_multi_transformer_op.cu.h
@@ -34,6 +34,7 @@ limitations under the License. */
 #include "paddle/fluid/platform/dynload/cublasLt.h"
 #include "paddle/phi/api/include/tensor.h"
 #include "paddle/phi/backends/gpu/gpu_device_function.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/funcs/fused_gemm_epilogue.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
 
@@ -43,7 +44,7 @@ limitations under the License. */
 #include "paddle/fluid/platform/device/gpu/nccl_helper.h"
 #endif
 
-DECLARE_bool(gemm_use_half_precision_compute_type);
+PHI_DECLARE_bool(gemm_use_half_precision_compute_type);
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/pscore/heter_listen_and_serv_op.cc b/paddle/fluid/operators/pscore/heter_listen_and_serv_op.cc
index 279277e18f702..5a39de9fa4d65 100644
--- a/paddle/fluid/operators/pscore/heter_listen_and_serv_op.cc
+++ b/paddle/fluid/operators/pscore/heter_listen_and_serv_op.cc
@@ -15,7 +15,7 @@ limitations under the License. */
 #include "paddle/fluid/operators/pscore/heter_listen_and_serv_op.h"
 
 #include "paddle/fluid/framework/op_registry.h"
-
+#include "paddle/fluid/platform/flags.h"
 PADDLE_DEFINE_EXPORTED_int32(rpc_send_thread_num,
                              12,
                              "number of threads for rpc send");
diff --git a/paddle/fluid/operators/pscore/heter_listen_and_server_test.cc b/paddle/fluid/operators/pscore/heter_listen_and_server_test.cc
index f87e66d69da4d..e29844d8f1b70 100644
--- a/paddle/fluid/operators/pscore/heter_listen_and_server_test.cc
+++ b/paddle/fluid/operators/pscore/heter_listen_and_server_test.cc
@@ -15,18 +15,19 @@ limitations under the License. */
 #include <stdlib.h>
 #include <unistd.h>
 
+#include <gtest/gtest.h>
 #include <random>
 #include <sstream>
 #include <string>
 #include <thread>  // NOLINT
 
-#include "gtest/gtest.h"
 #include "paddle/fluid/distributed/ps/service/heter_client.h"
 #include "paddle/fluid/distributed/ps/service/heter_server.h"
 #include "paddle/fluid/framework/block_desc.h"
 #include "paddle/fluid/framework/executor.h"
 #include "paddle/fluid/framework/op_registry.h"
 #include "paddle/fluid/framework/program_desc.h"
+#include "paddle/phi/core/flags.h"
 
 namespace framework = paddle::framework;
 namespace platform = paddle::platform;
@@ -34,7 +35,7 @@ namespace distributed = paddle::distributed;
 
 using MultiVarMsg = ::paddle::distributed::MultiVariableMessage;
 using VarMsg = ::paddle::distributed::VariableMessage;
-DECLARE_double(eager_delete_tensor_gb);
+PHI_DECLARE_double(eager_delete_tensor_gb);
 
 USE_OP_ITSELF(scale);
 USE_NO_KERNEL_OP(heter_listen_and_serv);
diff --git a/paddle/fluid/operators/run_program_op.h b/paddle/fluid/operators/run_program_op.h
index 513ae0d2785f9..ff6f6ceac54c8 100644
--- a/paddle/fluid/operators/run_program_op.h
+++ b/paddle/fluid/operators/run_program_op.h
@@ -37,8 +37,9 @@ limitations under the License. */
 #ifdef PADDLE_WITH_CUDA
 #include "paddle/fluid/operators/cuda_graph_with_in_out.h"
 #endif
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(use_mkldnn);
+PHI_DECLARE_bool(use_mkldnn);
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/platform/cpu_info_test.cc b/paddle/fluid/platform/cpu_info_test.cc
index e9e45c0292baf..6f7ccd03cff2f 100644
--- a/paddle/fluid/platform/cpu_info_test.cc
+++ b/paddle/fluid/platform/cpu_info_test.cc
@@ -18,8 +18,9 @@
 #include "gflags/gflags.h"
 #include "gtest/gtest.h"
 #include "paddle/fluid/string/printf.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_double(fraction_of_cpu_memory_to_use);
+PHI_DECLARE_double(fraction_of_cpu_memory_to_use);
 
 TEST(CpuMemoryUsage, Print) {
   std::stringstream ss;
diff --git a/paddle/fluid/platform/cuda_graph_with_memory_pool.cc b/paddle/fluid/platform/cuda_graph_with_memory_pool.cc
index 095a94cd32e2f..607dcf7622e47 100644
--- a/paddle/fluid/platform/cuda_graph_with_memory_pool.cc
+++ b/paddle/fluid/platform/cuda_graph_with_memory_pool.cc
@@ -17,9 +17,10 @@
 #include "paddle/fluid/memory/allocation/allocator_facade.h"
 #include "paddle/fluid/platform/device_event.h"
 #include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/core/flags.h"
 
 DECLARE_bool(use_stream_safe_cuda_allocator);
-DECLARE_bool(new_executor_use_cuda_graph);
+PHI_DECLARE_bool(new_executor_use_cuda_graph);
 
 namespace paddle {
 namespace platform {
diff --git a/paddle/fluid/platform/device/gpu/gpu_info.cc b/paddle/fluid/platform/device/gpu/gpu_info.cc
index 3373ef51836ea..7f1f2c76bd630 100644
--- a/paddle/fluid/platform/device/gpu/gpu_info.cc
+++ b/paddle/fluid/platform/device/gpu/gpu_info.cc
@@ -24,6 +24,7 @@ limitations under the License. */
 #include "paddle/fluid/memory/memory.h"
 #include "paddle/fluid/platform/cuda_device_guard.h"
 #include "paddle/fluid/platform/enforce.h"
+#include "paddle/fluid/platform/flags.h"
 #include "paddle/fluid/platform/lock_guard_ptr.h"
 #include "paddle/fluid/platform/macros.h"
 #include "paddle/fluid/platform/monitor.h"
@@ -31,7 +32,6 @@ limitations under the License. */
 #include "paddle/fluid/platform/profiler/mem_tracing.h"
 #include "paddle/fluid/string/split.h"
 #include "paddle/phi/backends/gpu/gpu_info.h"
-#include "paddle/phi/core/flags.h"
 
 #ifdef PADDLE_WITH_HIP
 #include "paddle/fluid/platform/dynload/miopen.h"
@@ -46,11 +46,11 @@ limitations under the License. */
 #endif
 #endif
 
-DECLARE_double(fraction_of_gpu_memory_to_use);
-DECLARE_uint64(initial_gpu_memory_in_mb);
-DECLARE_uint64(reallocate_gpu_memory_in_mb);
-DECLARE_bool(enable_cublas_tensor_op_math);
-DECLARE_uint64(gpu_memory_limit_mb);
+PHI_DECLARE_double(fraction_of_gpu_memory_to_use);
+PHI_DECLARE_uint64(initial_gpu_memory_in_mb);
+PHI_DECLARE_uint64(reallocate_gpu_memory_in_mb);
+PHI_DECLARE_bool(enable_cublas_tensor_op_math);
+PHI_DECLARE_uint64(gpu_memory_limit_mb);
 
 PADDLE_DEFINE_EXPORTED_bool(enable_gpu_memory_usage_log,
                             false,
diff --git a/paddle/fluid/platform/device_code.cc b/paddle/fluid/platform/device_code.cc
index f91b420be0d8a..2a410182b48a4 100644
--- a/paddle/fluid/platform/device_code.cc
+++ b/paddle/fluid/platform/device_code.cc
@@ -22,7 +22,7 @@ limitations under the License. */
 
 #include "paddle/fluid/platform/enforce.h"
 
-DECLARE_string(cuda_dir);
+PHI_DECLARE_string(cuda_dir);
 
 namespace paddle {
 namespace platform {
diff --git a/paddle/fluid/platform/enforce.h b/paddle/fluid/platform/enforce.h
index 312febb21fce2..425d4939b565f 100644
--- a/paddle/fluid/platform/enforce.h
+++ b/paddle/fluid/platform/enforce.h
@@ -107,7 +107,7 @@ namespace phi {
 class ErrorSummary;
 }  // namespace phi
 
-DECLARE_int32(call_stack_level);
+PHI_DECLARE_int32(call_stack_level);
 
 namespace paddle {
 namespace platform {
diff --git a/paddle/fluid/platform/flags.h b/paddle/fluid/platform/flags.h
new file mode 100644
index 0000000000000..32ce438deef44
--- /dev/null
+++ b/paddle/fluid/platform/flags.h
@@ -0,0 +1,77 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <string>
+#include <type_traits>
+
+#include "paddle/phi/core/flags.h"
+
+#define __PADDLE_DEFINE_EXPORTED_FLAG(                                        \
+    __name, __is_writable, __cpp_type, __gflag_type, __default_value, __doc)  \
+  DEFINE_##__gflag_type(__name, __default_value, __doc);                      \
+  struct __PaddleRegisterFlag_##__name {                                      \
+    __PaddleRegisterFlag_##__name() {                                         \
+      using FlagDeclaredType =                                                \
+          typename std::remove_reference<decltype(FLAGS_##__name)>::type;     \
+      static_assert(std::is_same<FlagDeclaredType, ::std::string>::value ||   \
+                        std::is_arithmetic<FlagDeclaredType>::value,          \
+                    "FLAGS should be std::string or arithmetic type");        \
+      auto *instance = ::phi::GetMutableExportedFlagInfoMap();                \
+      auto &info = (*instance)[#__name];                                      \
+      info.name = #__name;                                                    \
+      info.value_ptr = &(FLAGS_##__name);                                     \
+      info.default_value = static_cast<__cpp_type>(__default_value);          \
+      info.doc = __doc;                                                       \
+      info.is_writable = __is_writable;                                       \
+    }                                                                         \
+    int Touch() const { return 0; }                                           \
+  };                                                                          \
+  static __PaddleRegisterFlag_##__name __PaddleRegisterFlag_instance##__name; \
+  int TouchPaddleFlagRegister_##__name() {                                    \
+    return __PaddleRegisterFlag_instance##__name.Touch();                     \
+  }                                                                           \
+  static_assert(std::is_same<__PaddleRegisterFlag_##__name,                   \
+                             ::__PaddleRegisterFlag_##__name>::value,         \
+                "FLAGS should define in global namespace")
+
+#define PADDLE_FORCE_LINK_FLAG(__name)           \
+  extern int TouchPaddleFlagRegister_##__name(); \
+  UNUSED static int __paddle_use_flag_##__name = \
+      TouchPaddleFlagRegister_##__name()
+
+#define PADDLE_DEFINE_EXPORTED_bool(name, default_value, doc) \
+  __PADDLE_DEFINE_EXPORTED_FLAG(name, true, bool, bool, default_value, doc)
+#define PADDLE_DEFINE_EXPORTED_READONLY_bool(name, default_value, doc) \
+  __PADDLE_DEFINE_EXPORTED_FLAG(name, false, bool, bool, default_value, doc)
+
+#define PADDLE_DEFINE_EXPORTED_int32(name, default_value, doc) \
+  __PADDLE_DEFINE_EXPORTED_FLAG(name, true, int32_t, int32, default_value, doc)
+
+#define PADDLE_DEFINE_EXPORTED_int64(name, default_value, doc) \
+  __PADDLE_DEFINE_EXPORTED_FLAG(name, true, int64_t, int64, default_value, doc)
+
+#define PADDLE_DEFINE_EXPORTED_uint64(name, default_value, doc) \
+  __PADDLE_DEFINE_EXPORTED_FLAG(                                \
+      name, true, uint64_t, uint64, default_value, doc)
+
+#define PADDLE_DEFINE_EXPORTED_double(name, default_value, doc) \
+  __PADDLE_DEFINE_EXPORTED_FLAG(name, true, double, double, default_value, doc)
+
+#define PADDLE_DEFINE_EXPORTED_string(name, default_value, doc) \
+  __PADDLE_DEFINE_EXPORTED_FLAG(                                \
+      name, true, ::std::string, string, default_value, doc)
diff --git a/paddle/fluid/platform/gen_comm_id_helper.cc b/paddle/fluid/platform/gen_comm_id_helper.cc
index 308a05be51126..ca9f9d7c4f806 100644
--- a/paddle/fluid/platform/gen_comm_id_helper.cc
+++ b/paddle/fluid/platform/gen_comm_id_helper.cc
@@ -29,12 +29,12 @@ limitations under the License. */
 #include "glog/logging.h"
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/string/split.h"
-
+#include "paddle/phi/core/flags.h"
 #if defined(PADDLE_WITH_XPU_BKCL)
 #include "xpu/bkcl.h"
 #endif
 
-DECLARE_int32(get_host_by_name_time);
+PHI_DECLARE_int32(get_host_by_name_time);
 
 namespace paddle {
 namespace platform {
diff --git a/paddle/fluid/platform/init.cc b/paddle/fluid/platform/init.cc
index 4cb92cb7804d7..3fa02f82a6b95 100644
--- a/paddle/fluid/platform/init.cc
+++ b/paddle/fluid/platform/init.cc
@@ -53,10 +53,11 @@ limitations under the License. */
 
 #include "paddle/fluid/memory/allocation/allocator_facade.h"
 #include "paddle/fluid/memory/memory.h"
+#include "paddle/fluid/platform/flags.h"
 #include "paddle/phi/common/memory_utils.h"
 #include "paddle/phi/core/custom_kernel.h"
 
-DECLARE_int32(paddle_num_threads);
+PHI_DECLARE_int32(paddle_num_threads);
 PADDLE_DEFINE_EXPORTED_int32(
     multiple_of_cupti_buffer_size,
     1,
diff --git a/paddle/fluid/platform/place.cc b/paddle/fluid/platform/place.cc
index 354259ca91b74..fdaa67e3dd17d 100644
--- a/paddle/fluid/platform/place.cc
+++ b/paddle/fluid/platform/place.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "paddle/fluid/platform/place.h"
-
+#include "paddle/fluid/platform/flags.h"
 PADDLE_DEFINE_EXPORTED_bool(
     benchmark,
     false,
diff --git a/paddle/fluid/platform/profiler.cc b/paddle/fluid/platform/profiler.cc
index 6e06e8aa41ca4..cd8e8ea350f0e 100644
--- a/paddle/fluid/platform/profiler.cc
+++ b/paddle/fluid/platform/profiler.cc
@@ -32,8 +32,8 @@ limitations under the License. */
 #endif
 #include "paddle/fluid/framework/op_proto_maker.h"
 #include "paddle/fluid/framework/operator.h"
+#include "paddle/fluid/platform/flags.h"
 #include "paddle/fluid/platform/os_info.h"
-
 PADDLE_DEFINE_EXPORTED_bool(enable_rpc_profiler,
                             false,
                             "Enable rpc profiler or not.");
diff --git a/paddle/fluid/platform/profiler/host_tracer.cc b/paddle/fluid/platform/profiler/host_tracer.cc
index ba681c0360618..68ad4b4b34571 100644
--- a/paddle/fluid/platform/profiler/host_tracer.cc
+++ b/paddle/fluid/platform/profiler/host_tracer.cc
@@ -17,9 +17,9 @@
 
 #include "glog/logging.h"
 #include "paddle/fluid/framework/op_proto_maker.h"
+#include "paddle/fluid/platform/flags.h"
 #include "paddle/fluid/platform/profiler/common_event.h"
 #include "paddle/fluid/platform/profiler/host_event_recorder.h"
-#include "paddle/phi/core/flags.h"
 
 // Used to filter events, works like glog VLOG(level).
 // RecordEvent will works if host_trace_level >= level.
diff --git a/paddle/fluid/platform/test_limit_gpu_memory.cu b/paddle/fluid/platform/test_limit_gpu_memory.cu
index 6bc84a8be9e81..7fdf49ab11bad 100644
--- a/paddle/fluid/platform/test_limit_gpu_memory.cu
+++ b/paddle/fluid/platform/test_limit_gpu_memory.cu
@@ -12,12 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "gflags/gflags.h"
 #include "gtest/gtest.h"
 #include "paddle/fluid/platform/cuda_device_guard.h"
 #include "paddle/fluid/platform/device/gpu/gpu_info.h"
-
-DECLARE_uint64(gpu_memory_limit_mb);
+#include "paddle/phi/core/flags.h"
+PHI_DECLARE_uint64(gpu_memory_limit_mb);
 
 namespace paddle {
 namespace platform {
diff --git a/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h b/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h
index b1b24af231f68..eb13902be2068 100644
--- a/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h
+++ b/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h
@@ -32,7 +32,7 @@
 #include "paddle/phi/core/enforce.h"
 #include "paddle/phi/core/flags.h"
 
-DECLARE_string(tensor_operants_mode);
+PHI_DECLARE_string(tensor_operants_mode);
 
 namespace paddle {
 namespace prim {
diff --git a/paddle/fluid/prim/utils/utils.cc b/paddle/fluid/prim/utils/utils.cc
index aa5255f532adc..0cc160887e78d 100644
--- a/paddle/fluid/prim/utils/utils.cc
+++ b/paddle/fluid/prim/utils/utils.cc
@@ -13,8 +13,8 @@
 // limitations under the License.
 
 #include "paddle/fluid/prim/utils/utils.h"
+#include "paddle/fluid/platform/flags.h"
 #include "paddle/fluid/prim/utils/static/static_global_utils.h"
-#include "paddle/phi/core/flags.h"
 
 PADDLE_DEFINE_EXPORTED_bool(prim_enabled, false, "enable_prim or not");
 namespace paddle {
diff --git a/paddle/fluid/pybind/eager_functions.cc b/paddle/fluid/pybind/eager_functions.cc
index 3996ac81ef84c..f438acc5fff93 100644
--- a/paddle/fluid/pybind/eager_functions.cc
+++ b/paddle/fluid/pybind/eager_functions.cc
@@ -62,11 +62,11 @@ typedef SSIZE_T ssize_t;
 #include "paddle/fluid/pybind/cuda_streams_py.h"
 #endif
 
-#include "gflags/gflags.h"
 #include "paddle/phi/api/include/operants_manager.h"
 #include "paddle/phi/api/include/tensor_operants.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_string(tensor_operants_mode);
+PHI_DECLARE_string(tensor_operants_mode);
 
 namespace paddle {
 namespace pybind {
diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc
index 11dca753092fd..1da7bd774531b 100644
--- a/paddle/fluid/pybind/eager_method.cc
+++ b/paddle/fluid/pybind/eager_method.cc
@@ -59,10 +59,11 @@ typedef SSIZE_T ssize_t;
 #include "paddle/fluid/memory/allocation/mmap_allocator.h"
 #include "paddle/fluid/pybind/tensor_py.h"
 #include "paddle/phi/core/ddim.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/tensor_utils.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
 
-DECLARE_bool(set_to_1d);
+PHI_DECLARE_bool(set_to_1d);
 
 namespace paddle {
 namespace pybind {
diff --git a/paddle/fluid/pybind/parallel_executor.cc b/paddle/fluid/pybind/parallel_executor.cc
index 0c8898b524fae..553701906f59e 100644
--- a/paddle/fluid/pybind/parallel_executor.cc
+++ b/paddle/fluid/pybind/parallel_executor.cc
@@ -169,11 +169,12 @@ limitations under the License. */
 #include "paddle/fluid/pybind/eager_utils.h"
 #include "paddle/fluid/pybind/parallel_executor.h"
 #include "paddle/phi/api/ext/op_meta_info.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/autotune/cache.h"
 #include "paddle/phi/kernels/autotune/switch_autotune.h"
 #include "pybind11/stl.h"
 
-DECLARE_bool(use_mkldnn);
+PHI_DECLARE_bool(use_mkldnn);
 
 // disable auto conversion to list in Python
 PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);
diff --git a/paddle/fluid/pybind/place.cc b/paddle/fluid/pybind/place.cc
index 1c27d70d1bea7..59ed791c4522c 100644
--- a/paddle/fluid/pybind/place.cc
+++ b/paddle/fluid/pybind/place.cc
@@ -169,11 +169,12 @@ limitations under the License. */
 #include "paddle/fluid/pybind/eager_utils.h"
 #include "paddle/fluid/pybind/place.h"
 #include "paddle/phi/api/ext/op_meta_info.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/autotune/cache.h"
 #include "paddle/phi/kernels/autotune/switch_autotune.h"
 #include "pybind11/stl.h"
 
-DECLARE_bool(use_mkldnn);
+PHI_DECLARE_bool(use_mkldnn);
 
 // disable auto conversion to list in Python
 PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);
diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc
index 359c2266f8ea3..f4dfb133c1c36 100644
--- a/paddle/fluid/pybind/pybind.cc
+++ b/paddle/fluid/pybind/pybind.cc
@@ -193,11 +193,12 @@ limitations under the License. */
 #include "paddle/phi/api/ext/op_meta_info.h"
 #include "paddle/phi/api/include/operants_manager.h"
 #include "paddle/phi/api/include/tensor_operants.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/autotune/cache.h"
 #include "paddle/phi/kernels/autotune/switch_autotune.h"
 #include "pybind11/stl.h"
 
-DECLARE_bool(use_mkldnn);
+PHI_DECLARE_bool(use_mkldnn);
 
 // disable auto conversion to list in Python
 PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);
diff --git a/paddle/fluid/pybind/reader_py.cc b/paddle/fluid/pybind/reader_py.cc
index 8b8c29b8b1234..0640a070cbcec 100644
--- a/paddle/fluid/pybind/reader_py.cc
+++ b/paddle/fluid/pybind/reader_py.cc
@@ -32,9 +32,10 @@
 #include "paddle/fluid/operators/reader/py_reader.h"
 #include "paddle/fluid/platform/place.h"
 #include "paddle/phi/core/ddim.h"
+#include "paddle/phi/core/flags.h"
 #include "pybind11/stl.h"
 
-DECLARE_bool(reader_queue_speed_test_mode);
+PHI_DECLARE_bool(reader_queue_speed_test_mode);
 
 // disable auto conversion to list in Python
 PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);
diff --git a/paddle/fluid/pybind/tensor.cc b/paddle/fluid/pybind/tensor.cc
index cf10befbafaf4..799e48b1594de 100644
--- a/paddle/fluid/pybind/tensor.cc
+++ b/paddle/fluid/pybind/tensor.cc
@@ -170,12 +170,13 @@ limitations under the License. */
 #include "paddle/fluid/pybind/eager_utils.h"
 #include "paddle/fluid/pybind/tensor.h"
 #include "paddle/phi/api/ext/op_meta_info.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/autotune/cache.h"
 #include "paddle/phi/kernels/autotune/switch_autotune.h"
 #include "pybind11/stl.h"
 
-DECLARE_bool(use_mkldnn);
-DECLARE_bool(use_shm_cache);
+PHI_DECLARE_bool(use_mkldnn);
+PHI_DECLARE_bool(use_shm_cache);
 
 // disable auto conversion to list in Python
 PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);
diff --git a/paddle/phi/api/profiler/profiler.cc b/paddle/phi/api/profiler/profiler.cc
index 906ca12c7c581..f355872e881ee 100644
--- a/paddle/phi/api/profiler/profiler.cc
+++ b/paddle/phi/api/profiler/profiler.cc
@@ -33,13 +33,13 @@ limitations under the License. */
 #include "paddle/phi/backends/dynload/nvtx.h"
 #endif
 
-DEFINE_bool(enable_host_event_recorder_hook,
-            false,
-            "enable HostEventRecorder, hook Profiler");
+PHI_DEFINE_bool(enable_host_event_recorder_hook,
+                false,
+                "enable HostEventRecorder, hook Profiler");
 
-DEFINE_bool(enable_record_op_info,
-            false,
-            "enable operator supplement info recorder");
+PHI_DEFINE_bool(enable_record_op_info,
+                false,
+                "enable operator supplement info recorder");
 
 namespace phi {
 
diff --git a/paddle/phi/api/profiler/profiler.h b/paddle/phi/api/profiler/profiler.h
index 6863df8c635e7..fe5e3ec070a2c 100644
--- a/paddle/phi/api/profiler/profiler.h
+++ b/paddle/phi/api/profiler/profiler.h
@@ -23,12 +23,12 @@ limitations under the License. */
 #include <unordered_set>
 #include <utility>
 #include <vector>
-#include "gflags/gflags.h"
 
 #include "paddle/phi/api/profiler/event_tracing.h"
 #include "paddle/phi/api/profiler/supplement_tracing.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(enable_host_event_recorder_hook);
+PHI_DECLARE_bool(enable_host_event_recorder_hook);
 
 namespace phi {
 
diff --git a/paddle/phi/backends/cpu/cpu_info.cc b/paddle/phi/backends/cpu/cpu_info.cc
index 9d6b4e46df07d..0e91898fc3b93 100644
--- a/paddle/phi/backends/cpu/cpu_info.cc
+++ b/paddle/phi/backends/cpu/cpu_info.cc
@@ -43,9 +43,9 @@ DECLARE_double(fraction_of_cuda_pinned_memory_to_use);
 // between host and device.  Allocates too much would reduce the amount
 // of memory available to the system for paging.  So, by default, we
 // should set false to use_pinned_memory.
-PADDLE_DEFINE_EXPORTED_bool(use_pinned_memory,
-                            true,
-                            "If set, allocate cpu pinned memory.");
+PHI_DEFINE_EXPORTED_bool(use_pinned_memory,
+                         true,
+                         "If set, allocate cpu pinned memory.");
 
 namespace phi {
 namespace backends {
diff --git a/paddle/phi/backends/dynload/dynamic_loader.cc b/paddle/phi/backends/dynload/dynamic_loader.cc
index e3ec61576b2ac..85248360361ea 100644
--- a/paddle/phi/backends/dynload/dynamic_loader.cc
+++ b/paddle/phi/backends/dynload/dynamic_loader.cc
@@ -26,69 +26,73 @@ limitations under the License. */
 
 // TODO(wilber): The phi computing library requires a component to manage flags
 // (maybe not use gflags).
-#include "gflags/gflags.h"
 #include "glog/logging.h"
+#include "paddle/phi/core/flags.h"
 
-DEFINE_string(cudnn_dir,
-              "",
-              "Specify path for loading libcudnn.so. For instance, "
-              "/usr/local/cudnn/lib. If empty [default], dlopen "
-              "will search cudnn from LD_LIBRARY_PATH");
+PHI_DEFINE_string(cudnn_dir,
+                  "",
+                  "Specify path for loading libcudnn.so. For instance, "
+                  "/usr/local/cudnn/lib. If empty [default], dlopen "
+                  "will search cudnn from LD_LIBRARY_PATH");
 
-DEFINE_string(
+PHI_DEFINE_string(
     cuda_dir,
     "",
     "Specify path for loading cuda library, such as libcublas, libcublasLt "
     "libcurand, libcusolver. For instance, /usr/local/cuda/lib64. "
     "If default, dlopen will search cuda from LD_LIBRARY_PATH");
 
-DEFINE_string(nccl_dir,
-              "",
-              "Specify path for loading nccl library, such as libnccl.so. "
-              "For instance, /usr/local/cuda/lib64. If default, "
-              "dlopen will search cuda from LD_LIBRARY_PATH");
+PHI_DEFINE_string(nccl_dir,
+                  "",
+                  "Specify path for loading nccl library, such as libnccl.so. "
+                  "For instance, /usr/local/cuda/lib64. If default, "
+                  "dlopen will search cuda from LD_LIBRARY_PATH");
 
-DEFINE_string(cupti_dir, "", "Specify path for loading cupti.so.");
+PHI_DEFINE_string(cupti_dir, "", "Specify path for loading cupti.so.");
 
-DEFINE_string(
+PHI_DEFINE_string(
     tensorrt_dir,
     "",
     "Specify path for loading tensorrt library, such as libnvinfer.so.");
 
-DEFINE_string(mklml_dir, "", "Specify path for loading libmklml_intel.so.");
+PHI_DEFINE_string(mklml_dir, "", "Specify path for loading libmklml_intel.so.");
 
-DEFINE_string(lapack_dir, "", "Specify path for loading liblapack.so.");
+PHI_DEFINE_string(lapack_dir, "", "Specify path for loading liblapack.so.");
 
-DEFINE_string(mkl_dir,
-              "",
-              "Specify path for loading libmkl_rt.so. "
-              "For insrance, /opt/intel/oneapi/mkl/latest/lib/intel64/."
-              "If default, "
-              "dlopen will search mkl from LD_LIBRARY_PATH");
+PHI_DEFINE_string(mkl_dir,
+                  "",
+                  "Specify path for loading libmkl_rt.so. "
+                  "For insrance, /opt/intel/oneapi/mkl/latest/lib/intel64/."
+                  "If default, "
+                  "dlopen will search mkl from LD_LIBRARY_PATH");
 
-DEFINE_string(op_dir, "", "Specify path for loading user-defined op library.");
+PHI_DEFINE_string(op_dir,
+                  "",
+                  "Specify path for loading user-defined op library.");
 
-DEFINE_string(cusparselt_dir, "", "Specify path for loading libcusparseLt.so.");
+PHI_DEFINE_string(cusparselt_dir,
+                  "",
+                  "Specify path for loading libcusparseLt.so.");
 
 #ifdef PADDLE_WITH_HIP
 
-DEFINE_string(miopen_dir,
-              "",
-              "Specify path for loading libMIOpen.so. For instance, "
-              "/opt/rocm/miopen/lib. If empty [default], dlopen "
-              "will search miopen from LD_LIBRARY_PATH");
-
-DEFINE_string(rocm_dir,
-              "",
-              "Specify path for loading rocm library, such as librocblas, "
-              "libmiopen, libhipsparse. For instance, /opt/rocm/lib. "
-              "If default, dlopen will search rocm from LD_LIBRARY_PATH");
-
-DEFINE_string(rccl_dir,
-              "",
-              "Specify path for loading rccl library, such as librccl.so. "
-              "For instance, /opt/rocm/rccl/lib. If default, "
-              "dlopen will search rccl from LD_LIBRARY_PATH");
+PHI_DEFINE_string(miopen_dir,
+                  "",
+                  "Specify path for loading libMIOpen.so. For instance, "
+                  "/opt/rocm/miopen/lib. If empty [default], dlopen "
+                  "will search miopen from LD_LIBRARY_PATH");
+
+PHI_DEFINE_string(rocm_dir,
+                  "",
+                  "Specify path for loading rocm library, such as librocblas, "
+                  "libmiopen, libhipsparse. For instance, /opt/rocm/lib. "
+                  "If default, dlopen will search rocm from LD_LIBRARY_PATH");
+
+PHI_DEFINE_string(rccl_dir,
+                  "",
+                  "Specify path for loading rccl library, such as librccl.so. "
+                  "For instance, /opt/rocm/rccl/lib. If default, "
+                  "dlopen will search rccl from LD_LIBRARY_PATH");
 #endif
 
 namespace phi {
diff --git a/paddle/phi/backends/xpu/xpu_info.cc b/paddle/phi/backends/xpu/xpu_info.cc
index a8fbf7e7e6b7b..121c05a606917 100644
--- a/paddle/phi/backends/xpu/xpu_info.cc
+++ b/paddle/phi/backends/xpu/xpu_info.cc
@@ -25,7 +25,7 @@ limitations under the License. */
 // flags.
 #include "paddle/phi/core/flags.h"
 
-PADDLE_DEFINE_EXPORTED_string(
+PHI_DEFINE_EXPORTED_string(
     selected_xpus,
     "",
     "A list of device ids separated by comma, like: 0,1,2,3. "
diff --git a/paddle/phi/core/flags.cc b/paddle/phi/core/flags.cc
index 0b248a968785a..058ab7159e5ee 100644
--- a/paddle/phi/core/flags.cc
+++ b/paddle/phi/core/flags.cc
@@ -31,9 +31,9 @@ ExportedFlagInfoMap *GetMutableExportedFlagInfoMap() {
 
 }  // namespace phi
 
-PADDLE_DEFINE_EXPORTED_int32(inner_op_parallelism,
-                             0,
-                             "number of threads for inner op");
+PHI_DEFINE_EXPORTED_int32(inner_op_parallelism,
+                          0,
+                          "number of threads for inner op");
 
 /**
  * NOTE(paddle-dev): This file is designed to define all public FLAGS.
@@ -48,9 +48,9 @@ PADDLE_DEFINE_EXPORTED_int32(inner_op_parallelism,
  * instance to 2
  * Note:
  */
-PADDLE_DEFINE_EXPORTED_int32(paddle_num_threads,
-                             1,
-                             "Number of threads for each paddle instance.");
+PHI_DEFINE_EXPORTED_int32(paddle_num_threads,
+                          1,
+                          "Number of threads for each paddle instance.");
 
 /**
  * Low Precision Op related FLAG
@@ -63,11 +63,11 @@ PADDLE_DEFINE_EXPORTED_int32(paddle_num_threads,
  * - 1, return the low precision op list of current module.
  * - 2, return the op list of current module.
  */
-PADDLE_DEFINE_EXPORTED_int32(low_precision_op_list,
-                             0,
-                             "Setting the level of low precision op"
-                             "list printing. It will be return the "
-                             "low precision op list of current module.");
+PHI_DEFINE_EXPORTED_int32(low_precision_op_list,
+                          0,
+                          "Setting the level of low precision op"
+                          "list printing. It will be return the "
+                          "low precision op list of current module.");
 
 /**
  * Operator related FLAG
@@ -77,7 +77,7 @@ PADDLE_DEFINE_EXPORTED_int32(low_precision_op_list,
  * Example:
  * Note: Used to debug. Checking whether operator produce NAN/INF or not.
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     check_nan_inf,
     false,
     "Checking whether operator produce NAN/INF or not. It will be "
@@ -99,7 +99,7 @@ PADDLE_DEFINE_EXPORTED_bool(
  * overflowing float16's limit.
  * - 3, print the information of all tensors.
  */
-PADDLE_DEFINE_EXPORTED_int32(
+PHI_DEFINE_EXPORTED_int32(
     check_nan_inf_level,
     0,
     "Setting the check and print level when FLAGS_check_nan_inf is set.");
@@ -112,7 +112,7 @@ PADDLE_DEFINE_EXPORTED_int32(
  * Example:
  * Note: Used to debug. Checking whether operator produce NAN/INF or not.
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     enable_opt_get_features,
     false,
     "Checking whether operator produce NAN/INF or not. It will be "
@@ -130,7 +130,7 @@ PADDLE_DEFINE_EXPORTED_bool(
  * Example:
  * Note: whether to use Tensor Core, faster but it may loss precision.
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     enable_cublas_tensor_op_math,
     false,
     "The enable_cublas_tensor_op_math indicate whether to use Tensor Core, "
@@ -150,7 +150,7 @@ PADDLE_DEFINE_EXPORTED_bool(
  * Note: whether to use fp16 compute type when the input and output is fp16,
  * faster but it may loss precision.
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     gemm_use_half_precision_compute_type,
     false,
     "Whether to use fp16 compute type when the input and output is fp16, "
@@ -166,7 +166,7 @@ PADDLE_DEFINE_EXPORTED_bool(
  * cards
  * Note: A list of device ids separated by comma, like: 0,1,2,3
  */
-PADDLE_DEFINE_EXPORTED_string(
+PHI_DEFINE_EXPORTED_string(
     selected_gpus,
     "",
     "A list of device ids separated by comma, like: 0,1,2,3. "
@@ -195,7 +195,7 @@ PADDLE_DEFINE_EXPORTED_string(
  *       layer specification Once you change the layer specifications
  *       (such as M, N and K), it will re-search again.
  */
-PADDLE_DEFINE_EXPORTED_int64(
+PHI_DEFINE_EXPORTED_int64(
     cublaslt_exhaustive_search_times,
     0,
     "The times of exhaustive search for cuBlasLt matmul with/without "
@@ -210,7 +210,7 @@ PADDLE_DEFINE_EXPORTED_int64(
  * Example: FLAGS_enable_api_kernel_fallback=true would allow kernel of current
  * backend fallback to CPU one when not found
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     enable_api_kernel_fallback,
     true,
     "Whether enable api kernel fallback to CPU one when not found");
@@ -225,7 +225,7 @@ PADDLE_DEFINE_EXPORTED_bool(
  * Note: whether to use deterministic algorithm in cudnn.
  *       If true, it will slow down some operators such as conv and pooling.
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     cudnn_deterministic,
     false,
     "Whether allow using an autotuning algorithm for convolution "
@@ -241,7 +241,7 @@ PADDLE_DEFINE_EXPORTED_bool(
  * Note: whether to use deterministic algorithm in embedding op.
  *       If true, it will use deterministic CUDA kernel in embedding op.
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     embedding_deterministic,
     false,
     "Whether allow using an deterministic algorithm for embedding "
@@ -260,10 +260,9 @@ PADDLE_DEFINE_EXPORTED_bool(
  * increased.
  *       Users need to balance memory and speed.
  */
-PADDLE_DEFINE_EXPORTED_int64(
-    conv_workspace_size_limit,
-    phi::backends::gpu::kDefaultConvWorkspaceSizeLimitMB,
-    "cuDNN convolution workspace limit in MB unit.");
+PHI_DEFINE_EXPORTED_int64(conv_workspace_size_limit,
+                          phi::backends::gpu::kDefaultConvWorkspaceSizeLimitMB,
+                          "cuDNN convolution workspace limit in MB unit.");
 
 /**
  * CUDNN related FLAG
@@ -279,7 +278,7 @@ PADDLE_DEFINE_EXPORTED_int64(
  *       layer specification. Once you change the layer specifications
  *       (such as batch size, feature map size), it will search again.
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     cudnn_exhaustive_search,
     false,
     "Whether enable exhaustive search for cuDNN convolution or "
@@ -293,10 +292,10 @@ PADDLE_DEFINE_EXPORTED_bool(
  * Example:
  * Note: only used to predict for advanced developer
  */
-PADDLE_DEFINE_EXPORTED_int64(cudnn_exhaustive_search_times,
-                             -1,
-                             "Exhaustive search times for cuDNN convolution, "
-                             "default is -1, not exhaustive search");
+PHI_DEFINE_EXPORTED_int64(cudnn_exhaustive_search_times,
+                          -1,
+                          "Exhaustive search times for cuDNN convolution, "
+                          "default is -1, not exhaustive search");
 
 /**
  * CUDNN related FLAG
@@ -314,7 +313,7 @@ PADDLE_DEFINE_EXPORTED_int64(cudnn_exhaustive_search_times,
  * certain
  *       input data range.
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     cudnn_batchnorm_spatial_persistent,
     false,
     "Whether enable CUDNN_BATCHNORM_SPATIAL_PERSISTENT mode for cudnn "
@@ -333,7 +332,7 @@ PADDLE_DEFINE_EXPORTED_bool(
  *       https://github.com/PaddlePaddle/Paddle/issues/15049
  *       If you want to change this default value, why?(gongwb)
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     sync_nccl_allreduce,
     true,
     "If set true, will call `cudaStreamSynchronize(nccl_stream)`"
@@ -352,10 +351,10 @@ PADDLE_DEFINE_EXPORTED_bool(
  *       into the queue, and then the communicator takes the gradients out
  *       of the queue and sends them after merging.
  */
-PADDLE_DEFINE_EXPORTED_int32(communicator_max_merge_var_num,
-                             20,
-                             "max var num to merge and send");
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_int32(communicator_max_merge_var_num,
+                          20,
+                          "max var num to merge and send");
+PHI_DEFINE_EXPORTED_bool(
     communicator_is_sgd_optimizer,
     true,
     "gradient sent to the server is the sum of the gradients "
@@ -373,9 +372,9 @@ PADDLE_DEFINE_EXPORTED_bool(
  *       space. It is used to avoid training much faster than communication,
  *       so that too many gradients are not sent out in time.
  */
-PADDLE_DEFINE_EXPORTED_int32(communicator_send_queue_size,
-                             20,
-                             "queue size to recv gradient before send");
+PHI_DEFINE_EXPORTED_int32(communicator_send_queue_size,
+                          20,
+                          "queue size to recv gradient before send");
 #endif
 
 /**
@@ -387,10 +386,9 @@ PADDLE_DEFINE_EXPORTED_int32(communicator_send_queue_size,
  * Note: Control the number of threads used for distributed modules.
  *       If it is not set, it is set to a hard thread.
  */
-PADDLE_DEFINE_EXPORTED_int32(
-    dist_threadpool_size,
-    0,
-    "number of threads used for distributed executed.");
+PHI_DEFINE_EXPORTED_int32(dist_threadpool_size,
+                          0,
+                          "number of threads used for distributed executed.");
 
 /**
  * Garbage collector related FLAG
@@ -411,7 +409,7 @@ PADDLE_DEFINE_EXPORTED_int32(
 // Disable gc by default when inference library is built
 static const double kDefaultEagerDeleteTensorGB = 0;
 
-PADDLE_DEFINE_EXPORTED_double(
+PHI_DEFINE_EXPORTED_double(
     eager_delete_tensor_gb,
     kDefaultEagerDeleteTensorGB,
     "Memory size threshold (GB) when the garbage collector clear tensors."
@@ -429,7 +427,7 @@ PADDLE_DEFINE_EXPORTED_double(
  *       has finished, which will make the garbage collection strategy faster.
  *       Only works when garbage collection strategy is enabled.
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     fast_eager_deletion_mode,
     true,
     "Fast eager deletion mode. If enabled, memory would release "
@@ -453,7 +451,7 @@ PADDLE_DEFINE_EXPORTED_bool(
  *       largest FLAGS_memory_fraction_of_eager_deletion ratio will be released.
  *       The flag is only valid when running parallel data compilers.
  */
-PADDLE_DEFINE_EXPORTED_double(
+PHI_DEFINE_EXPORTED_double(
     memory_fraction_of_eager_deletion,
     1.0,
     "Fraction of eager deletion. If less than 1.0, all variables in "
@@ -471,7 +469,7 @@ PADDLE_DEFINE_EXPORTED_double(
  * Note: For selecting allocator policy of PaddlePaddle.
  */
 static constexpr char kDefaultAllocatorStrategy[] = "auto_growth";
-PADDLE_DEFINE_EXPORTED_string(
+PHI_DEFINE_EXPORTED_string(
     allocator_strategy,
     kDefaultAllocatorStrategy,
     "The allocation strategy, enum in [naive_best_fit, auto_growth]. "
@@ -499,10 +497,10 @@ PADDLE_DEFINE_EXPORTED_string(
  *       size as the memory block will be allocated from the CUDA pinned
  *       request util the CPU does not have enough memory.
  */
-PADDLE_DEFINE_EXPORTED_double(fraction_of_cpu_memory_to_use,
-                              1,
-                              "Default use 100% of CPU memory for PaddlePaddle,"
-                              "reserve the rest for page tables, etc");
+PHI_DEFINE_EXPORTED_double(fraction_of_cpu_memory_to_use,
+                           1,
+                           "Default use 100% of CPU memory for PaddlePaddle,"
+                           "reserve the rest for page tables, etc");
 
 /**
  * Memory related FLAG
@@ -516,10 +514,9 @@ PADDLE_DEFINE_EXPORTED_double(fraction_of_cpu_memory_to_use,
  *       FLAGS_fraction_of_cpu_memory_to_use*(total physical memory)
  *       as memory block sizes.
  */
-PADDLE_DEFINE_EXPORTED_uint64(
-    initial_cpu_memory_in_mb,
-    500ul,
-    "Initial CPU memory for PaddlePaddle, in MD unit.");
+PHI_DEFINE_EXPORTED_uint64(initial_cpu_memory_in_mb,
+                           500ul,
+                           "Initial CPU memory for PaddlePaddle, in MD unit.");
 
 /**
  * Memory related FLAG
@@ -534,7 +531,7 @@ PADDLE_DEFINE_EXPORTED_uint64(
  *       size as the memory block will be allocated from the CPU
  *       request util the CPU does not have enough memory.
  */
-PADDLE_DEFINE_EXPORTED_double(
+PHI_DEFINE_EXPORTED_double(
     fraction_of_cuda_pinned_memory_to_use,
     0.5,
     "Default use 50% of CPU memory as the pinned_memory for PaddlePaddle,"
@@ -570,7 +567,7 @@ constexpr static float fraction_of_gpu_memory_to_use = 0.92f;
 // which may lead to insufficient memory left for paddle
 constexpr static float fraction_of_gpu_memory_to_use = 0.5f;
 #endif
-PADDLE_DEFINE_EXPORTED_double(
+PHI_DEFINE_EXPORTED_double(
     fraction_of_gpu_memory_to_use,
     fraction_of_gpu_memory_to_use,
     "Allocate a trunk of gpu memory that is this fraction of the "
@@ -591,7 +588,7 @@ PADDLE_DEFINE_EXPORTED_double(
  *       FLAGS_reallocate_gpu_memory_in_mb will be requested from the GPU until
  *       the GPU has no remaining memory.
  */
-PADDLE_DEFINE_EXPORTED_uint64(
+PHI_DEFINE_EXPORTED_uint64(
     initial_gpu_memory_in_mb,
     0ul,
     "Allocate a trunk of gpu memory whose byte size is specified by "
@@ -614,14 +611,14 @@ PADDLE_DEFINE_EXPORTED_uint64(
  * Note: If the allocated GPU memory blocks are exhausted,
  *       additional GPU memory blocks are reallocated
  */
-PADDLE_DEFINE_EXPORTED_uint64(
+PHI_DEFINE_EXPORTED_uint64(
     reallocate_gpu_memory_in_mb,
     0ul,
     "If this flag is set, Paddle will reallocate the gpu memory with "
     "size specified by this flag. Else Paddle will reallocate by "
     "FLAGS_fraction_of_gpu_memory_to_use");
 
-PADDLE_DEFINE_EXPORTED_uint64(
+PHI_DEFINE_EXPORTED_uint64(
     gpu_memory_limit_mb,
     0UL,
     "The maximum gpu memory limit that the process can allocate. "
@@ -641,7 +638,7 @@ PADDLE_DEFINE_EXPORTED_uint64(
  *       The real chunk size is max(request_size,
  *       FLAGS_auto_growth_chunk_size_in_mb).
  */
-PADDLE_DEFINE_EXPORTED_uint64(
+PHI_DEFINE_EXPORTED_uint64(
     auto_growth_chunk_size_in_mb,
     0ul,
     "The minimal chunk size of GPU memory block in auto_growth allocator.  "
@@ -658,7 +655,7 @@ PADDLE_DEFINE_EXPORTED_uint64(
  * Example:
  * Note:
  */
-PADDLE_DEFINE_EXPORTED_double(
+PHI_DEFINE_EXPORTED_double(
     local_exe_sub_scope_limit,
     256.0,  // MBytes
     "The memory up limit of sub-scopes of local execution scope for "
@@ -666,7 +663,7 @@ PADDLE_DEFINE_EXPORTED_double(
     "you should set FLAGS_local_exe_sub_scope_limit=-1. "
     "The default value is 256 MBytes.");
 
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     reader_queue_speed_test_mode,
     false,
     "If set true, the queue.pop will only get data from queue but not "
@@ -680,7 +677,7 @@ PADDLE_DEFINE_EXPORTED_bool(
  * Example:
  * Note:
  */
-PADDLE_DEFINE_EXPORTED_bool(use_mkldnn, false, "Use MKLDNN to run");
+PHI_DEFINE_EXPORTED_bool(use_mkldnn, false, "Use MKLDNN to run");
 
 /**
  * Debug related FLAG
@@ -702,7 +699,7 @@ static const int32_t kDefaultCallStackLevel = 2;
 static const int32_t kDefaultCallStackLevel = 1;
 #endif
 
-PADDLE_DEFINE_EXPORTED_int32(
+PHI_DEFINE_EXPORTED_int32(
     call_stack_level,
     kDefaultCallStackLevel,
     "Determine the call stack to print when error or exeception happens."
@@ -723,10 +720,10 @@ PADDLE_DEFINE_EXPORTED_int32(
  * Note: If True, gradients are summed by the reverse order of
  * the forward execution sequence.
  */
-PADDLE_DEFINE_EXPORTED_bool(sort_sum_gradient,
-                            false,
-                            "Sum gradients by the reverse order of "
-                            "the forward execution sequence.");
+PHI_DEFINE_EXPORTED_bool(sort_sum_gradient,
+                         false,
+                         "Sum gradients by the reverse order of "
+                         "the forward execution sequence.");
 
 /**
  * Performance related FLAG
@@ -736,7 +733,7 @@ PADDLE_DEFINE_EXPORTED_bool(sort_sum_gradient,
  * Example:
  * Note: The maximum number of inplace grad_add.
  */
-PADDLE_DEFINE_EXPORTED_int32(
+PHI_DEFINE_EXPORTED_int32(
     max_inplace_grad_add,
     0,
     "The maximum number of inplace grad_add. When doing "
@@ -752,7 +749,7 @@ PADDLE_DEFINE_EXPORTED_int32(
  * Now, just set true by default in 2.5 transition time
  * which will be removed in future (2.6 or 2.7) .
  */
-PADDLE_DEFINE_EXPORTED_bool(set_to_1d, true, "set 0D Tensor to 1D numpy");
+PHI_DEFINE_EXPORTED_bool(set_to_1d, true, "set 0D Tensor to 1D numpy");
 
 /**
  * Debug related FLAG
@@ -762,9 +759,9 @@ PADDLE_DEFINE_EXPORTED_bool(set_to_1d, true, "set 0D Tensor to 1D numpy");
  * Example:
  * Note: Holds list of operation types with OneDNN kernels to be enabled.
  */
-PADDLE_DEFINE_EXPORTED_string(tracer_mkldnn_ops_on,
-                              "",
-                              "List of OneDNN operation types to be turned on");
+PHI_DEFINE_EXPORTED_string(tracer_mkldnn_ops_on,
+                           "",
+                           "List of OneDNN operation types to be turned on");
 
 /**
  * Debug related FLAG
@@ -774,10 +771,9 @@ PADDLE_DEFINE_EXPORTED_string(tracer_mkldnn_ops_on,
  * Example:
  * Note: Holds list of operation types with OneDNN kernels to be disabled.
  */
-PADDLE_DEFINE_EXPORTED_string(
-    tracer_mkldnn_ops_off,
-    "",
-    "List of OneDNN operation types to be turned off");
+PHI_DEFINE_EXPORTED_string(tracer_mkldnn_ops_off,
+                           "",
+                           "List of OneDNN operation types to be turned off");
 
 /**
  * Debug related FLAG
@@ -788,7 +784,7 @@ PADDLE_DEFINE_EXPORTED_string(
  * Note: Check kernel launch status after every kernel compute.
  */
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     check_kernel_launch,
     false,
     "Check kernel launch status after every kernel compute");
@@ -803,13 +799,13 @@ PADDLE_DEFINE_EXPORTED_bool(
  * Note: Disable cudnn in conv2d.
  */
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-PADDLE_DEFINE_EXPORTED_bool(conv2d_disable_cudnn,
-                            false,
-                            "Disable cudnn in conv2d");
+PHI_DEFINE_EXPORTED_bool(conv2d_disable_cudnn,
+                         false,
+                         "Disable cudnn in conv2d");
 
-PADDLE_DEFINE_EXPORTED_bool(use_fast_math,
-                            false,
-                            "Whether to use fast math GPU functions.");
+PHI_DEFINE_EXPORTED_bool(use_fast_math,
+                         false,
+                         "Whether to use fast math GPU functions.");
 #endif
 
 /**
@@ -822,9 +818,9 @@ PADDLE_DEFINE_EXPORTED_bool(use_fast_math,
  */
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_XPU) || \
     defined(PADDLE_WITH_HIP)
-PADDLE_DEFINE_EXPORTED_int32(get_host_by_name_time,
-                             120,
-                             "The maximum time for get host by name time");
+PHI_DEFINE_EXPORTED_int32(get_host_by_name_time,
+                          120,
+                          "The maximum time for get host by name time");
 #endif
 
 /**
@@ -836,7 +832,7 @@ PADDLE_DEFINE_EXPORTED_int32(get_host_by_name_time,
  *          program when using Fleet APIs.
  * Note: Apply IR pass to program. Be only useful when using Fleet APIs.
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     apply_pass_to_program,
     false,
     "It controls whether to apply IR pass to program when using Fleet APIs");
@@ -850,10 +846,10 @@ PADDLE_DEFINE_EXPORTED_bool(
  * Note: Control whether load graph node and edge with multi threads parallely
  *       If it is not set, load graph data with one thread
  */
-PADDLE_DEFINE_EXPORTED_bool(graph_load_in_parallel,
-                            false,
-                            "It controls whether load graph node and edge with "
-                            "mutli threads parallely.");
+PHI_DEFINE_EXPORTED_bool(graph_load_in_parallel,
+                         false,
+                         "It controls whether load graph node and edge with "
+                         "mutli threads parallely.");
 
 /**
  * Distributed related FLAG
@@ -864,10 +860,10 @@ PADDLE_DEFINE_EXPORTED_bool(graph_load_in_parallel,
  * Note: Control whether load graph node and edge with multi threads parallely
  *       If it is not set, load graph data with one thread
  */
-PADDLE_DEFINE_EXPORTED_bool(graph_metapath_split_opt,
-                            false,
-                            "It controls whether load graph node and edge with "
-                            "mutli threads parallely.");
+PHI_DEFINE_EXPORTED_bool(graph_metapath_split_opt,
+                         false,
+                         "It controls whether load graph node and edge with "
+                         "mutli threads parallely.");
 
 /**
  * Distributed related FLAG
@@ -878,7 +874,7 @@ PADDLE_DEFINE_EXPORTED_bool(graph_metapath_split_opt,
  * Note: Control get all neighbor id when running sub part graph
  *       If it is not set, do not need get neighbor id when run all part graph
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     graph_get_neighbor_id,
     false,
     "It controls get all neighbor id when running sub part graph.");
@@ -892,7 +888,7 @@ PADDLE_DEFINE_EXPORTED_bool(
  * Note: Control  whether exit trainer when an worker has no ins.
  *       If it is not set, trainer will exit until all worker finish train.
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     enable_exit_when_partial_worker,
     false,
     "It controls whether exit trainer when an worker has no ins.");
@@ -905,9 +901,9 @@ PADDLE_DEFINE_EXPORTED_bool(
  * Example:
  * Note: represent gpugraph storage mode, 1 for full hbm, 2 for hbm + mem + ssd.
  */
-PADDLE_DEFINE_EXPORTED_int32(gpugraph_storage_mode,
-                             1,
-                             "gpugraph storage mode, default 1");
+PHI_DEFINE_EXPORTED_int32(gpugraph_storage_mode,
+                          1,
+                          "gpugraph storage mode, default 1");
 
 /**
  * KP kernel related FLAG
@@ -918,9 +914,9 @@ PADDLE_DEFINE_EXPORTED_int32(gpugraph_storage_mode,
  * Op.
  * Note:
  */
-PADDLE_DEFINE_EXPORTED_bool(run_kp_kernel,
-                            false,
-                            "It controls whether to run PaddlePaddle using KP");
+PHI_DEFINE_EXPORTED_bool(run_kp_kernel,
+                         false,
+                         "It controls whether to run PaddlePaddle using KP");
 
 /**
  * Distributed related FLAG
@@ -933,12 +929,12 @@ PADDLE_DEFINE_EXPORTED_bool(run_kp_kernel,
  *       multiple events. Currently, only fuse allreduce supports this.
  *       Otherwise, the precision may be wrong.
  */
-PADDLE_DEFINE_EXPORTED_bool(allreduce_record_one_event,
-                            false,
-                            "It controls whether the allreduce operations "
-                            "would only wait one event instead of multiple "
-                            "events. Currently, only fuse allreduce supports "
-                            "this. Otherwise, the precision may be wrong.");
+PHI_DEFINE_EXPORTED_bool(allreduce_record_one_event,
+                         false,
+                         "It controls whether the allreduce operations "
+                         "would only wait one event instead of multiple "
+                         "events. Currently, only fuse allreduce supports "
+                         "this. Otherwise, the precision may be wrong.");
 
 #ifdef PADDLE_WITH_CINN
 /*
@@ -948,8 +944,9 @@ PADDLE_DEFINE_EXPORTED_bool(allreduce_record_one_event,
  * Value Range: bool, default=false
  * Example: FLAGS_use_cinn=true would run PaddlePaddle using CINN
  */
-PADDLE_DEFINE_EXPORTED_bool(
-    use_cinn, false, "It controls whether to run PaddlePaddle using CINN");
+PHI_DEFINE_EXPORTED_bool(use_cinn,
+                         false,
+                         "It controls whether to run PaddlePaddle using CINN");
 
 /*
  * CINN related FLAG
@@ -959,10 +956,10 @@ PADDLE_DEFINE_EXPORTED_bool(
  * Example: FLAGS_allow_cinn_ops="mul;relu" would only cover `mul` and `relu`
  * when using CINN
  */
-PADDLE_DEFINE_EXPORTED_string(allow_cinn_ops,
-                              "",
-                              "It controls the cinn op subset to be used, "
-                              "which has the highest priority.");
+PHI_DEFINE_EXPORTED_string(allow_cinn_ops,
+                           "",
+                           "It controls the cinn op subset to be used, "
+                           "which has the highest priority.");
 
 /*
  * CINN related FLAG
@@ -972,9 +969,9 @@ PADDLE_DEFINE_EXPORTED_string(allow_cinn_ops,
  * Example: FLAGS_deny_cinn_ops="mul;relu" would block `mul` and `relu` two ops
  * when using CINN
  */
-PADDLE_DEFINE_EXPORTED_string(deny_cinn_ops,
-                              "",
-                              "It controls the cinn op subset to be not used.");
+PHI_DEFINE_EXPORTED_string(deny_cinn_ops,
+                           "",
+                           "It controls the cinn op subset to be not used.");
 
 /*
  * CINN related FLAG
@@ -985,10 +982,10 @@ PADDLE_DEFINE_EXPORTED_string(deny_cinn_ops,
  * instructions of a paddle graph with ParallelExecutor, otherwise with the
  * CINN compiled runtime program in sequential order.
  */
-PADDLE_DEFINE_EXPORTED_bool(enable_pe_launch_cinn,
-                            true,
-                            "It controls whether to execute cinn compiled "
-                            "program with ParallelExecutor");
+PHI_DEFINE_EXPORTED_bool(enable_pe_launch_cinn,
+                         true,
+                         "It controls whether to execute cinn compiled "
+                         "program with ParallelExecutor");
 
 /*
  * CINN related FLAG
@@ -999,10 +996,10 @@ PADDLE_DEFINE_EXPORTED_bool(enable_pe_launch_cinn,
  * compiled instructions of a paddle graph with InterpreterCore, otherwise with
  * the CINN compiled runtime program in sequential order.
  */
-PADDLE_DEFINE_EXPORTED_bool(enable_interpretercore_launch_cinn,
-                            true,
-                            "It controls whether to execute cinn compiled "
-                            "program with InterpreterCore");
+PHI_DEFINE_EXPORTED_bool(enable_interpretercore_launch_cinn,
+                         true,
+                         "It controls whether to execute cinn compiled "
+                         "program with InterpreterCore");
 
 /*
  * CINN related FLAG
@@ -1012,10 +1009,10 @@ PADDLE_DEFINE_EXPORTED_bool(enable_interpretercore_launch_cinn,
  * Example: FLAGS_enable_cinn_auto_tune=true would use CINN with its
  * auto-tune feature enabled
  */
-PADDLE_DEFINE_EXPORTED_bool(enable_cinn_auto_tune,
-                            false,
-                            "It controls whether to use cinn with "
-                            "its auto-tune feature enabled");
+PHI_DEFINE_EXPORTED_bool(enable_cinn_auto_tune,
+                         false,
+                         "It controls whether to use cinn with "
+                         "its auto-tune feature enabled");
 
 /*
  * CINN related FLAG
@@ -1026,10 +1023,10 @@ PADDLE_DEFINE_EXPORTED_bool(enable_cinn_auto_tune,
  * CINN sub-graph into "./cinn_graph/", and each sub-graph will save into
  * "fusion_groups_*"" directory
  */
-PADDLE_DEFINE_EXPORTED_string(cinn_subgraph_graphviz_dir,
-                              "",
-                              "Specify the directory path of dot file of "
-                              "graph, which is used for debug.");
+PHI_DEFINE_EXPORTED_string(cinn_subgraph_graphviz_dir,
+                           "",
+                           "Specify the directory path of dot file of "
+                           "graph, which is used for debug.");
 
 #endif
 
@@ -1041,9 +1038,9 @@ PADDLE_DEFINE_EXPORTED_string(cinn_subgraph_graphviz_dir,
  * Example: FLAGS_new_executor_use_cuda_graph=true would allow
  * new executor to use CUDA Graph.
  */
-PADDLE_DEFINE_EXPORTED_bool(new_executor_use_cuda_graph,
-                            false,
-                            "Use CUDA Graph in new executor");
+PHI_DEFINE_EXPORTED_bool(new_executor_use_cuda_graph,
+                         false,
+                         "Use CUDA Graph in new executor");
 
 DEFINE_int32(record_pool_max_size,
              2000000,
@@ -1058,56 +1055,56 @@ DEFINE_bool(enable_slotrecord_reset_shrink,
 DEFINE_bool(enable_ins_parser_file,
             false,
             "enable parser ins file, default false");
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     gpugraph_enable_hbm_table_collision_stat,
     false,
     "enable hash collisions stat for hbm table, default false");
-PADDLE_DEFINE_EXPORTED_double(gpugraph_hbm_table_load_factor,
-                              0.75,
-                              "the load factor of hbm table, default 0.75");
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_double(gpugraph_hbm_table_load_factor,
+                           0.75,
+                           "the load factor of hbm table, default 0.75");
+PHI_DEFINE_EXPORTED_bool(
     gpugraph_enable_gpu_direct_access,
     false,
     "enable direct access bwtween multi gpu cards, default false");
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     gpugraph_enable_segment_merge_grads,
     false,
     "enable segment merge gradients while push sparse, default false");
-PADDLE_DEFINE_EXPORTED_uint64(
+PHI_DEFINE_EXPORTED_uint64(
     gpugraph_merge_grads_segment_size,
     128,
     "segment size with segment gradient merge, default 128");
-PADDLE_DEFINE_EXPORTED_uint64(gpugraph_slot_feasign_max_num,
-                              5,
-                              "max feasign number in one slot, default 5");
-PADDLE_DEFINE_EXPORTED_int32(
+PHI_DEFINE_EXPORTED_uint64(gpugraph_slot_feasign_max_num,
+                           5,
+                           "max feasign number in one slot, default 5");
+PHI_DEFINE_EXPORTED_int32(
     gpugraph_dedup_pull_push_mode,
     0,
     "enable dedup keys while pull push sparse, default 0");
-PADDLE_DEFINE_EXPORTED_bool(gpugraph_load_node_list_into_hbm,
-                            true,
-                            "enable load_node_list_into_hbm, default true");
-PADDLE_DEFINE_EXPORTED_int32(gpugraph_sparse_table_storage_mode,
-                             0,
-                             "parse_table_storage_mode, default 0");
-PADDLE_DEFINE_EXPORTED_bool(enable_auto_detect_gpu_topo,
-                            true,
-                            "enable auto detect gpu topo, default true");
-PADDLE_DEFINE_EXPORTED_bool(enable_auto_rdma_trans,
-                            true,
-                            "enable auto gpu rdma trans, default true");
-PADDLE_DEFINE_EXPORTED_bool(enable_tracker_all2all,
-                            false,
-                            "enable tracker all2all log, default false");
-PADDLE_DEFINE_EXPORTED_bool(enable_all2all_use_fp16,
-                            false,
-                            "enable all2all use fp16, default false");
-PADDLE_DEFINE_EXPORTED_bool(enable_sparse_inner_gather,
-                            false,
-                            "enable sparse inner gather, default false");
-PADDLE_DEFINE_EXPORTED_bool(gpugraph_debug_gpu_memory,
-                            false,
-                            "enable debug gpu memory, default false");
+PHI_DEFINE_EXPORTED_bool(gpugraph_load_node_list_into_hbm,
+                         true,
+                         "enable load_node_list_into_hbm, default true");
+PHI_DEFINE_EXPORTED_int32(gpugraph_sparse_table_storage_mode,
+                          0,
+                          "parse_table_storage_mode, default 0");
+PHI_DEFINE_EXPORTED_bool(enable_auto_detect_gpu_topo,
+                         true,
+                         "enable auto detect gpu topo, default true");
+PHI_DEFINE_EXPORTED_bool(enable_auto_rdma_trans,
+                         true,
+                         "enable auto gpu rdma trans, default true");
+PHI_DEFINE_EXPORTED_bool(enable_tracker_all2all,
+                         false,
+                         "enable tracker all2all log, default false");
+PHI_DEFINE_EXPORTED_bool(enable_all2all_use_fp16,
+                         false,
+                         "enable all2all use fp16, default false");
+PHI_DEFINE_EXPORTED_bool(enable_sparse_inner_gather,
+                         false,
+                         "enable sparse inner gather, default false");
+PHI_DEFINE_EXPORTED_bool(gpugraph_debug_gpu_memory,
+                         false,
+                         "enable debug gpu memory, default false");
 /**
  * ProcessGroupNCCL related FLAG
  * Name: nccl_blocking_wait
@@ -1117,7 +1114,7 @@ PADDLE_DEFINE_EXPORTED_bool(gpugraph_debug_gpu_memory,
  * Note: nccl blocking wait.
  */
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-PADDLE_DEFINE_EXPORTED_bool(nccl_blocking_wait, false, "nccl blocking wait");
+PHI_DEFINE_EXPORTED_bool(nccl_blocking_wait, false, "nccl blocking wait");
 #endif
 
 /**
@@ -1127,7 +1124,7 @@ PADDLE_DEFINE_EXPORTED_bool(nccl_blocking_wait, false, "nccl blocking wait");
  * Value Range: bool, default=false
  * Example:
  */
-PADDLE_DEFINE_EXPORTED_bool(use_autotune, false, "Whether enable autotune.");
+PHI_DEFINE_EXPORTED_bool(use_autotune, false, "Whether enable autotune.");
 
 /**
  * Conv Search cache max number related FLAG
@@ -1136,9 +1133,9 @@ PADDLE_DEFINE_EXPORTED_bool(use_autotune, false, "Whether enable autotune.");
  * Value Range: int32, default=1000000
  * Example:
  */
-PADDLE_DEFINE_EXPORTED_int32(search_cache_max_number,
-                             1000000,
-                             "search_cache_max_number.");
+PHI_DEFINE_EXPORTED_int32(search_cache_max_number,
+                          1000000,
+                          "search_cache_max_number.");
 
 /**
  * Preformance related FLAG
@@ -1149,7 +1146,7 @@ PADDLE_DEFINE_EXPORTED_int32(search_cache_max_number,
  * Note: If True, EinsumOp will be optimimzed by innercache reuse, which
  * uses more gpu memory.
  */
-PADDLE_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_bool(
     einsum_opt,
     false,
     "EinsumOp backward will be speedup at the expense of more gpu memory.");
@@ -1165,9 +1162,9 @@ PADDLE_DEFINE_EXPORTED_bool(
  * FLAGS_jit_engine_type == New, using InterpreterEngine by default
  * FLAGS_jit_engine_type == Predictor, using inference Predictor by default
  */
-PADDLE_DEFINE_EXPORTED_string(jit_engine_type,
-                              "Predictor",
-                              "Choose default funciton type in JitLayer.");
+PHI_DEFINE_EXPORTED_string(jit_engine_type,
+                           "Predictor",
+                           "Choose default funciton type in JitLayer.");
 
 /**
  * Custom Device NPU related FLAG
@@ -1177,7 +1174,7 @@ PADDLE_DEFINE_EXPORTED_string(jit_engine_type,
  * Example:
  * Note: Enable NPU Storage Format for Ascend910 performance improvement.
  */
-PADDLE_DEFINE_EXPORTED_bool(npu_storage_format, false, "");
+PHI_DEFINE_EXPORTED_bool(npu_storage_format, false, "");
 
 #ifdef PADDLE_WITH_CUDNN_FRONTEND
 /**
@@ -1188,7 +1185,7 @@ PADDLE_DEFINE_EXPORTED_bool(npu_storage_format, false, "");
  * Example:
  * Note: Enable CUDNNv8 Frontend API for CUDNN kernels.
  */
-PADDLE_DEFINE_EXPORTED_bool(enable_cudnn_frontend, false, "");
+PHI_DEFINE_EXPORTED_bool(enable_cudnn_frontend, false, "");
 
 /**
  * CUDNNv8 related FLAG
@@ -1201,7 +1198,7 @@ PADDLE_DEFINE_EXPORTED_bool(enable_cudnn_frontend, false, "");
  * N times before it is actually added in the cache. It is useful when
  * the result of exhaustive search is unstable.
  */
-PADDLE_DEFINE_EXPORTED_int32(cudnn_cache_saturation_count, 1, "");
+PHI_DEFINE_EXPORTED_int32(cudnn_cache_saturation_count, 1, "");
 #endif  // PADDLE_WITH_CUDNN_FRONTEND
 
 /**
@@ -1213,9 +1210,9 @@ PADDLE_DEFINE_EXPORTED_int32(cudnn_cache_saturation_count, 1, "");
  * Note: This FLAG is only enabled when CI is running. If True, a persistent
  * IBuilder is added to avoid TensorRT unload/reload kernels.
  */
-PADDLE_DEFINE_EXPORTED_bool(trt_ibuilder_cache,
-                            false,
-                            "Add a persistent ibuilder.");
+PHI_DEFINE_EXPORTED_bool(trt_ibuilder_cache,
+                         false,
+                         "Add a persistent ibuilder.");
 
 /**
  * mmap_allocator related FLAG
@@ -1226,9 +1223,9 @@ PADDLE_DEFINE_EXPORTED_bool(trt_ibuilder_cache,
  * Note: . If True, mmap_allocator will cache shm file to decrease munmap
  * operation.
  */
-PADDLE_DEFINE_EXPORTED_bool(use_shm_cache,
-                            false,
-                            "Use shm cache in mmap_allocator.");
+PHI_DEFINE_EXPORTED_bool(use_shm_cache,
+                         false,
+                         "Use shm cache in mmap_allocator.");
 
 /**
  * Tensor operants related FLAG
@@ -1242,6 +1239,6 @@ PADDLE_DEFINE_EXPORTED_bool(use_shm_cache,
  *       - phi mode: tensor operants with only phi forward API;
  *       - static mode: tensor operants within static graph.
  */
-PADDLE_DEFINE_EXPORTED_string(tensor_operants_mode,
-                              "eager",
-                              "Tensor operants mode");
+PHI_DEFINE_EXPORTED_string(tensor_operants_mode,
+                           "eager",
+                           "Tensor operants mode");
diff --git a/paddle/phi/core/flags.h b/paddle/phi/core/flags.h
index e9ace6206255d..0112be93b7f92 100644
--- a/paddle/phi/core/flags.h
+++ b/paddle/phi/core/flags.h
@@ -24,25 +24,127 @@
 
 #include "paddle/utils/variant.h"
 
+#if defined(_WIN32) && defined(BUILD_PHI_SHARED)
+#define PHI_EXPORT_FLAG __declspec(dllexport)
+#define PHI_IMPORT_FLAG __declspec(dllimport)
+#else
+#define PHI_EXPORT_FLAG
+#define PHI_IMPORT_FLAG
+#endif  // _WIN32
+
+// We redefine the gflags' macro for exporting global variable
+
+// ----------------------------DECLARE FLAGS----------------------------
+// clang-format off
+#define PHI_DECLARE_VARIABLE(type, shorttype, name) \
+  namespace fL##shorttype {                         \
+    extern PHI_IMPORT_FLAG type FLAGS_##name;       \
+  }                                                 \
+  using fL##shorttype::FLAGS_##name
+// clang-format on
+
+#define PHI_DECLARE_bool(name) PHI_DECLARE_VARIABLE(bool, B, name)
+
+#define PHI_DECLARE_int32(name) \
+  PHI_DECLARE_VARIABLE(::GFLAGS_NAMESPACE::int32, I, name)
+
+#define PHI_DECLARE_uint32(name) \
+  PHI_DECLARE_VARIABLE(::GFLAGS_NAMESPACE::uint32, U, name)
+
+#define PHI_DECLARE_int64(name) \
+  PHI_DECLARE_VARIABLE(::GFLAGS_NAMESPACE::int64, I64, name)
+
+#define PHI_DECLARE_uint64(name) \
+  PHI_DECLARE_VARIABLE(::GFLAGS_NAMESPACE::uint64, U64, name)
+
+#define PHI_DECLARE_double(name) PHI_DECLARE_VARIABLE(double, D, name)
+
+#define PHI_DECLARE_string(name)                               \
+  /* We always want to import declared variables, dll or no */ \
+  namespace fLS {                                              \
+  extern PHI_IMPORT_FLAG ::fLS::clstring& FLAGS_##name;        \
+  }                                                            \
+  using fLS::FLAGS_##name
+
+// ----------------------------DEFINE FLAGS----------------------------
+#define PHI_DEFINE_VARIABLE(type, shorttype, name, value, help) \
+  namespace fL##shorttype {                                     \
+    static const type FLAGS_nono##name = value;                 \
+    PHI_EXPORT_FLAG type FLAGS_##name = FLAGS_nono##name;       \
+    static type FLAGS_no##name = FLAGS_nono##name;              \
+    static GFLAGS_NAMESPACE::FlagRegisterer o_##name(           \
+        #name,                                                  \
+        MAYBE_STRIPPED_HELP(help),                              \
+        __FILE__,                                               \
+        &FLAGS_##name,                                          \
+        &FLAGS_no##name);                                       \
+  } /* NOLINT */                                                \
+  using fL##shorttype::FLAGS_##name
+
+#define PHI_DEFINE_bool(name, val, txt)                              \
+  namespace fLB {                                                    \
+  typedef ::fLB::CompileAssert FLAG_##name##_value_is_not_a_bool     \
+      [(sizeof(::fLB::IsBoolFlag(val)) != sizeof(double)) ? 1 : -1]; \
+  }                                                                  \
+  PHI_DEFINE_VARIABLE(bool, B, name, val, txt)
+
+#define PHI_DEFINE_int32(name, val, txt) \
+  PHI_DEFINE_VARIABLE(GFLAGS_NAMESPACE::int32, I, name, val, txt)
+
+#define PHI_DEFINE_uint32(name, val, txt) \
+  PHI_DEFINE_VARIABLE(GFLAGS_NAMESPACE::uint32, U, name, val, txt)
+
+#define PHI_DEFINE_int64(name, val, txt) \
+  PHI_DEFINE_VARIABLE(GFLAGS_NAMESPACE::int64, I64, name, val, txt)
+
+#define PHI_DEFINE_uint64(name, val, txt) \
+  PHI_DEFINE_VARIABLE(GFLAGS_NAMESPACE::uint64, U64, name, val, txt)
+
+#define PHI_DEFINE_double(name, val, txt) \
+  PHI_DEFINE_VARIABLE(double, D, name, val, txt)
+
+#define PHI_DEFINE_string(name, val, txt)                             \
+  namespace fLS {                                                     \
+  using ::fLS::clstring;                                              \
+  using ::fLS::StringFlagDestructor;                                  \
+  static union {                                                      \
+    void* align;                                                      \
+    char s[sizeof(clstring)];                                         \
+  } s_##name[2];                                                      \
+  clstring* const FLAGS_no##name =                                    \
+      ::fLS::dont_pass0toDEFINE_string(s_##name[0].s, val);           \
+  static GFLAGS_NAMESPACE::FlagRegisterer o_##name(                   \
+      #name,                                                          \
+      MAYBE_STRIPPED_HELP(txt),                                       \
+      __FILE__,                                                       \
+      FLAGS_no##name,                                                 \
+      new (s_##name[1].s) clstring(*FLAGS_no##name));                 \
+  static StringFlagDestructor d_##name(s_##name[0].s, s_##name[1].s); \
+  extern PHI_EXPORT_FLAG clstring& FLAGS_##name;                      \
+  using fLS::FLAGS_##name;                                            \
+  clstring& FLAGS_##name = *FLAGS_no##name;                           \
+  } /* NOLINT */                                                      \
+  using fLS::FLAGS_##name
+
 namespace phi {
 
 struct FlagInfo {
   using ValueType =
       paddle::variant<bool, int32_t, int64_t, uint64_t, double, std::string>;
   std::string name;
-  mutable void *value_ptr;
+  mutable void* value_ptr;
   ValueType default_value;
   std::string doc;
   bool is_writable;
 };
 
 using ExportedFlagInfoMap = std::map<std::string, FlagInfo>;
-const ExportedFlagInfoMap &GetExportedFlagInfoMap();
-ExportedFlagInfoMap *GetMutableExportedFlagInfoMap();
+const ExportedFlagInfoMap& GetExportedFlagInfoMap();
+ExportedFlagInfoMap* GetMutableExportedFlagInfoMap();
 
-#define __PADDLE_DEFINE_EXPORTED_FLAG(                                        \
+#define __PHI_DEFINE_EXPORTED_FLAG(                                           \
     __name, __is_writable, __cpp_type, __gflag_type, __default_value, __doc)  \
-  DEFINE_##__gflag_type(__name, __default_value, __doc);                      \
+  PHI_DEFINE_##__gflag_type(__name, __default_value, __doc);                  \
   struct __PaddleRegisterFlag_##__name {                                      \
     __PaddleRegisterFlag_##__name() {                                         \
       using FlagDeclaredType =                                                \
@@ -50,8 +152,8 @@ ExportedFlagInfoMap *GetMutableExportedFlagInfoMap();
       static_assert(std::is_same<FlagDeclaredType, ::std::string>::value ||   \
                         std::is_arithmetic<FlagDeclaredType>::value,          \
                     "FLAGS should be std::string or arithmetic type");        \
-      auto *instance = ::phi::GetMutableExportedFlagInfoMap();                \
-      auto &info = (*instance)[#__name];                                      \
+      auto* instance = ::phi::GetMutableExportedFlagInfoMap();                \
+      auto& info = (*instance)[#__name];                                      \
       info.name = #__name;                                                    \
       info.value_ptr = &(FLAGS_##__name);                                     \
       info.default_value = static_cast<__cpp_type>(__default_value);          \
@@ -73,26 +175,25 @@ ExportedFlagInfoMap *GetMutableExportedFlagInfoMap();
   UNUSED static int __paddle_use_flag_##__name = \
       TouchPaddleFlagRegister_##__name()
 
-#define PADDLE_DEFINE_EXPORTED_bool(name, default_value, doc) \
-  __PADDLE_DEFINE_EXPORTED_FLAG(name, true, bool, bool, default_value, doc)
-#define PADDLE_DEFINE_EXPORTED_READONLY_bool(name, default_value, doc) \
-  __PADDLE_DEFINE_EXPORTED_FLAG(name, false, bool, bool, default_value, doc)
+#define PHI_DEFINE_EXPORTED_bool(name, default_value, doc) \
+  __PHI_DEFINE_EXPORTED_FLAG(name, true, bool, bool, default_value, doc)
+#define PHI_DEFINE_EXPORTED_READONLY_bool(name, default_value, doc) \
+  __PHI_DEFINE_EXPORTED_FLAG(name, false, bool, bool, default_value, doc)
 
-#define PADDLE_DEFINE_EXPORTED_int32(name, default_value, doc) \
-  __PADDLE_DEFINE_EXPORTED_FLAG(name, true, int32_t, int32, default_value, doc)
+#define PHI_DEFINE_EXPORTED_int32(name, default_value, doc) \
+  __PHI_DEFINE_EXPORTED_FLAG(name, true, int32_t, int32, default_value, doc)
 
-#define PADDLE_DEFINE_EXPORTED_int64(name, default_value, doc) \
-  __PADDLE_DEFINE_EXPORTED_FLAG(name, true, int64_t, int64, default_value, doc)
+#define PHI_DEFINE_EXPORTED_int64(name, default_value, doc) \
+  __PHI_DEFINE_EXPORTED_FLAG(name, true, int64_t, int64, default_value, doc)
 
-#define PADDLE_DEFINE_EXPORTED_uint64(name, default_value, doc) \
-  __PADDLE_DEFINE_EXPORTED_FLAG(                                \
-      name, true, uint64_t, uint64, default_value, doc)
+#define PHI_DEFINE_EXPORTED_uint64(name, default_value, doc) \
+  __PHI_DEFINE_EXPORTED_FLAG(name, true, uint64_t, uint64, default_value, doc)
 
-#define PADDLE_DEFINE_EXPORTED_double(name, default_value, doc) \
-  __PADDLE_DEFINE_EXPORTED_FLAG(name, true, double, double, default_value, doc)
+#define PHI_DEFINE_EXPORTED_double(name, default_value, doc) \
+  __PHI_DEFINE_EXPORTED_FLAG(name, true, double, double, default_value, doc)
 
-#define PADDLE_DEFINE_EXPORTED_string(name, default_value, doc) \
-  __PADDLE_DEFINE_EXPORTED_FLAG(                                \
+#define PHI_DEFINE_EXPORTED_string(name, default_value, doc) \
+  __PHI_DEFINE_EXPORTED_FLAG(                                \
       name, true, ::std::string, string, default_value, doc)
 
 }  // namespace phi
diff --git a/paddle/phi/kernels/funcs/jit/gen_base.cc b/paddle/phi/kernels/funcs/jit/gen_base.cc
index eadf636179636..1e5731e60ceaf 100644
--- a/paddle/phi/kernels/funcs/jit/gen_base.cc
+++ b/paddle/phi/kernels/funcs/jit/gen_base.cc
@@ -25,7 +25,7 @@
 #define posix_memalign_free free
 #endif
 
-DEFINE_bool(dump_jitcode, false, "Whether to dump the jitcode to file");
+PHI_DEFINE_bool(dump_jitcode, false, "Whether to dump the jitcode to file");
 
 namespace phi {
 namespace jit {
diff --git a/paddle/phi/kernels/funcs/jit/gen_base.h b/paddle/phi/kernels/funcs/jit/gen_base.h
index 8795d36fbc12d..c72c0c52792c7 100644
--- a/paddle/phi/kernels/funcs/jit/gen_base.h
+++ b/paddle/phi/kernels/funcs/jit/gen_base.h
@@ -23,9 +23,10 @@
 #endif
 
 #include "gflags/gflags.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/kernels/funcs/jit/kernel_base.h"
 
-DECLARE_bool(dump_jitcode);
+PHI_DECLARE_bool(dump_jitcode);
 
 namespace phi {
 namespace jit {
diff --git a/paddle/utils/pybind.cc b/paddle/utils/pybind.cc
index 7e93dcff1400a..24f4c2bd22a8d 100644
--- a/paddle/utils/pybind.cc
+++ b/paddle/utils/pybind.cc
@@ -14,10 +14,10 @@
 
 #include "paddle/utils/pybind.h"
 
-#include "gflags/gflags.h"
 #include "paddle/phi/core/enforce.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_string(tensor_operants_mode);
+PHI_DECLARE_string(tensor_operants_mode);
 namespace paddle {
 namespace pybind {
 
diff --git a/test/cpp/inference/test_helper.h b/test/cpp/inference/test_helper.h
index f0ed56d1e19f6..0685b90de1fa2 100644
--- a/test/cpp/inference/test_helper.h
+++ b/test/cpp/inference/test_helper.h
@@ -24,8 +24,9 @@ limitations under the License. */
 #include "paddle/fluid/platform/errors.h"
 #include "paddle/fluid/platform/profiler.h"
 #include "paddle/phi/backends/dynload/port.h"
+#include "paddle/phi/core/flags.h"
 
-DECLARE_bool(use_mkldnn);
+PHI_DECLARE_bool(use_mkldnn);
 
 namespace paddle {
 bool gpu_place_used(const paddle::PaddlePlace& place) {
diff --git a/test/cpp/prim/test_eager_prim.cc b/test/cpp/prim/test_eager_prim.cc
index d2cdeb80495c1..9d2166d457643 100644
--- a/test/cpp/prim/test_eager_prim.cc
+++ b/test/cpp/prim/test_eager_prim.cc
@@ -14,7 +14,6 @@
 
 #include <sstream>
 
-#include "gflags/gflags.h"
 #include "glog/logging.h"
 #include "gtest/gtest.h"
 #include "paddle/fluid/eager/api/generated/eager_generated/forwards/dygraph_functions.h"
@@ -22,12 +21,13 @@
 #include "paddle/fluid/eager/backward.h"
 #include "paddle/fluid/prim/utils/utils.h"
 #include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/core/tensor_meta.h"
 #include "test/cpp/eager/test_utils.h"
 #include "test/cpp/prim/init_env_utils.h"
 
-DECLARE_string(tensor_operants_mode);
+PHI_DECLARE_string(tensor_operants_mode);
 
 PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(tanh, CPU, ALL_LAYOUT);
diff --git a/test/cpp/prim/test_static_prim.cc b/test/cpp/prim/test_static_prim.cc
index 1ae7303d10d75..936ae9babfe36 100644
--- a/test/cpp/prim/test_static_prim.cc
+++ b/test/cpp/prim/test_static_prim.cc
@@ -25,10 +25,11 @@
 #include "paddle/fluid/prim/utils/utils.h"
 #include "paddle/phi/api/include/operants_manager.h"
 #include "paddle/phi/core/enforce.h"
+#include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/kernel_registry.h"
 
 DECLARE_bool(prim_enabled);
-DECLARE_string(tensor_operants_mode);
+PHI_DECLARE_string(tensor_operants_mode);
 
 PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(tanh, CPU, ALL_LAYOUT);

From c1a61fc0447f16c419a5b86ee35649adda929996 Mon Sep 17 00:00:00 2001
From: Zero Rains <me@zerorains.top>
Date: Tue, 25 Apr 2023 16:01:18 +0800
Subject: [PATCH 055/405] =?UTF-8?q?=E3=80=90PaddlePaddle=20Hackathon=204?=
 =?UTF-8?q?=20No.33=E3=80=91=E4=B8=BA=20Paddle=20=E4=BC=98=E5=8C=96=20Hist?=
 =?UTF-8?q?ogram=20op=20=E5=9C=A8=20GPU=20=E4=B8=8A=E7=9A=84=E8=AE=A1?=
 =?UTF-8?q?=E7=AE=97=E6=80=A7=E8=83=BD=20(#53112)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* create KernelMinMax to optimize the performance of histogram op in GPU

* change to block and warp wise operation

* remove the time in DtoH

* fix a bug
---
 paddle/phi/kernels/gpu/histogram_kernel.cu | 105 ++++++++++++++-------
 1 file changed, 73 insertions(+), 32 deletions(-)

diff --git a/paddle/phi/kernels/gpu/histogram_kernel.cu b/paddle/phi/kernels/gpu/histogram_kernel.cu
index 111b13f11dd0e..aa10aea35f867 100644
--- a/paddle/phi/kernels/gpu/histogram_kernel.cu
+++ b/paddle/phi/kernels/gpu/histogram_kernel.cu
@@ -18,8 +18,7 @@
 #include "paddle/phi/backends/gpu/gpu_launch_config.h"
 #include "paddle/phi/backends/gpu/gpu_primitives.h"
 #include "paddle/phi/core/kernel_registry.h"
-#include "paddle/phi/kernels/funcs/eigen/common.h"
-#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
+#include "paddle/phi/kernels/funcs/math_cuda_utils.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
 
 namespace phi {
@@ -46,8 +45,8 @@ template <typename T, typename IndexType>
 __global__ void KernelHistogram(const T* input,
                                 const int total_elements,
                                 const int64_t nbins,
-                                const T min_value,
-                                const T max_value,
+                                const T* min_value,
+                                const T* max_value,
                                 int64_t* output) {
   extern __shared__ int64_t buf_hist[];
   for (int i = threadIdx.x; i < nbins; i += blockDim.x) {
@@ -58,9 +57,9 @@ __global__ void KernelHistogram(const T* input,
   CUDA_KERNEL_LOOP(input_index, total_elements) {
     // const IndexType input_index = threadIdx.x + blockIdx.x * blockDim.x;
     const auto input_value = input[input_index];
-    if (input_value >= min_value && input_value <= max_value) {
+    if (input_value >= *min_value && input_value <= *max_value) {
       const IndexType output_index =
-          GetBin<T, IndexType>(input_value, min_value, max_value, nbins);
+          GetBin<T, IndexType>(input_value, *min_value, *max_value, nbins);
       phi::CudaAtomicAdd(&buf_hist[output_index], 1);
     }
   }
@@ -71,6 +70,60 @@ __global__ void KernelHistogram(const T* input,
   }
 }
 
+template <typename T>
+__global__ void KernelMinMax(const T* input,
+                             const int numel,
+                             const int block_num,
+                             T* min_ptr,
+                             T* max_ptr) {
+  int64_t index = threadIdx.x + blockIdx.x * blockDim.x;
+  int64_t i = index;
+  T min_value = static_cast<T>(i < numel ? input[i] : input[0]);
+  T max_value = static_cast<T>(i < numel ? input[i] : input[0]);
+
+  for (; i < numel; i += blockDim.x * gridDim.x) {
+    T value = static_cast<T>(input[i]);
+    min_value = value < min_value ? value : min_value;
+    max_value = value > max_value ? value : max_value;
+  }
+  if (max_ptr && min_ptr) {
+    __syncthreads();
+    T block_min_value = phi::funcs::BlockReduceMin<T>(min_value, FINAL_MASK);
+    T block_max_value = phi::funcs::BlockReduceMax<T>(max_value, FINAL_MASK);
+
+    if (threadIdx.x == 0) {
+      min_ptr[blockIdx.x] = block_min_value;
+      max_ptr[blockIdx.x] = block_max_value;
+    }
+  }
+  __syncthreads();
+  if (index == 0) {
+    if (min_ptr && max_ptr) {
+      min_value = min_ptr[0];
+      max_value = max_ptr[0];
+      for (int64_t i = 1; i < block_num; i++) {
+        min_ptr[0] = min_ptr[i] < min_value ? min_ptr[i] : min_value;
+        max_ptr[0] = max_ptr[i] > max_value ? max_ptr[i] : max_value;
+      }
+      if (min_ptr[0] == max_ptr[0]) {
+        min_ptr[0] = min_ptr[0] - 1;
+        max_ptr[0] = max_ptr[0] + 1;
+      }
+    }
+  }
+}
+
+template <typename T>
+__global__ void KernelMinMax(const T min_value,
+                             const T max_value,
+                             T* min_ptr,
+                             T* max_ptr) {
+  if (threadIdx.x == 0 && blockIdx.x == 0) {
+    min_ptr[0] = min_value;
+    max_ptr[0] = max_value;
+  }
+}
+
 template <typename T, typename Context>
 void HistogramKernel(const Context& dev_ctx,
                      const DenseTensor& input,
@@ -93,32 +146,20 @@ void HistogramKernel(const Context& dev_ctx,
 
   T output_min = static_cast<T>(minval);
   T output_max = static_cast<T>(maxval);
-
-  if (output_min == output_max) {
-    auto input_x = phi::EigenVector<T>::Flatten(input);
-
-    DenseTensor input_min_t, input_max_t;
-    input_min_t.Resize({1});
-    input_max_t.Resize({1});
-    auto* input_min_data = dev_ctx.template Alloc<T>(&input_min_t);
-    auto* input_max_data = dev_ctx.template Alloc<T>(&input_max_t);
-    auto input_min_scala = phi::EigenScalar<T>::From(input_min_t);
-    auto input_max_scala = phi::EigenScalar<T>::From(input_max_t);
-
-    auto* place = dev_ctx.eigen_device();
-    input_min_scala.device(*place) = input_x.minimum();
-    input_max_scala.device(*place) = input_x.maximum();
-
-    DenseTensor input_min_cpu, input_max_cpu;
-    phi::Copy(dev_ctx, input_min_t, phi::CPUPlace(), true, &input_min_cpu);
-    phi::Copy(dev_ctx, input_max_t, phi::CPUPlace(), true, &input_max_cpu);
-
-    output_min = input_min_cpu.data<T>()[0];
-    output_max = input_max_cpu.data<T>()[0];
-  }
+  DenseTensor min_max;
+  int block_num = GET_BLOCKS(input_numel);
+  min_max.Resize({2 * block_num});
+  auto* min_block_ptr = dev_ctx.template Alloc<T>(&min_max);
+  auto* max_block_ptr = min_block_ptr + block_num;
   if (output_min == output_max) {
-    output_min = output_min - 1;
-    output_max = output_max + 1;
+    KernelMinMax<T><<<GET_BLOCKS(input_numel),
+                      PADDLE_CUDA_NUM_THREADS,
+                      0,
+                      dev_ctx.stream()>>>(
+        input_data, input_numel, block_num, min_block_ptr, max_block_ptr);
+  } else {
+    KernelMinMax<T><<<1, 1, 0, dev_ctx.stream()>>>(
+        output_min, output_max, min_block_ptr, max_block_ptr);
   }
 
   PADDLE_ENFORCE_EQ((std::isinf(static_cast<float>(output_min)) ||
@@ -142,7 +183,7 @@ void HistogramKernel(const Context& dev_ctx,
                                   PADDLE_CUDA_NUM_THREADS,
                                   nbins * sizeof(int64_t),
                                   stream>>>(
-      input_data, input_numel, nbins, output_min, output_max, out_data);
+      input_data, input_numel, nbins, min_block_ptr, max_block_ptr, out_data);
 }
 
 }  // namespace phi

From f6f48780e44bbdf4773cd71144a8c1dd971c3d1f Mon Sep 17 00:00:00 2001
From: huangjiyi <43315610+huangjiyi@users.noreply.github.com>
Date: Tue, 25 Apr 2023 16:13:59 +0800
Subject: [PATCH 056/405] Register fluid xpu kerenls to phi [part 1] (#53187)

* update

* fix bug

* Revert "affine_channel_op"
---
 cmake/operators.cmake                         |  1 +
 .../operators/beam_search_decode_op_xpu.cc    | 22 ++++++++++---------
 paddle/fluid/operators/beam_search_op_xpu.cc  | 15 +++++++------
 .../collective/c_allgather_op_xpu.cc          | 17 ++++++++------
 .../collective/c_allreduce_max_op_xpu.cc      | 18 ++++++++++-----
 .../collective/c_allreduce_min_op_xpu.cc      | 16 +++++++++++---
 .../operators/collective/c_allreduce_op.h     |  4 ++++
 .../collective/c_allreduce_prod_op_xpu.cc     | 16 +++++++++++---
 .../collective/c_allreduce_sum_op_xpu.cc      | 18 ++++++++++-----
 .../collective/c_broadcast_op_xpu.cc          | 11 ++++++----
 .../operators/collective/c_concat_op_xpu.cc   |  7 +++---
 .../collective/c_embedding_op_xpu.cc          | 14 +++++-------
 .../operators/collective/c_identity_op_xpu.cc | 15 ++++++++-----
 .../collective/c_reduce_max_op_xpu.cc         | 10 +++++++--
 .../collective/c_reduce_min_op_xpu.cc         | 10 +++++++--
 .../fluid/operators/collective/c_reduce_op.h  |  4 ++++
 .../collective/c_reduce_prod_op_xpu.cc        | 10 +++++++--
 .../collective/c_reduce_sum_op_xpu.cc         | 11 +++++++---
 .../collective/mp_allreduce_sum_op_xpu.cc     | 18 ++++++++++-----
 19 files changed, 161 insertions(+), 76 deletions(-)

diff --git a/cmake/operators.cmake b/cmake/operators.cmake
index a9cac23e8f0b0..e22a747688b76 100644
--- a/cmake/operators.cmake
+++ b/cmake/operators.cmake
@@ -470,6 +470,7 @@ function(op_library TARGET)
       foreach(xpu_src ${xpu_cc_srcs})
         set(op_name "")
         find_register(${xpu_src} "REGISTER_OP_XPU_KERNEL" op_name)
+        find_phi_register(${xpu_src} ${pybind_file} "PD_REGISTER_STRUCT_KERNEL")
         if(NOT ${op_name} EQUAL "")
           file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${op_name}, XPU);\n")
           set(pybind_flag 1)
diff --git a/paddle/fluid/operators/beam_search_decode_op_xpu.cc b/paddle/fluid/operators/beam_search_decode_op_xpu.cc
index b78cec34cb900..5fd2b2fc6fa35 100644
--- a/paddle/fluid/operators/beam_search_decode_op_xpu.cc
+++ b/paddle/fluid/operators/beam_search_decode_op_xpu.cc
@@ -20,7 +20,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename DeviceContext, typename T>
+template <typename T, typename DeviceContext>
 class BeamSearchDecodeXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& context) const override {
@@ -111,13 +111,15 @@ class BeamSearchDecodeXPUKernel : public framework::OpKernel<T> {
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OP_XPU_KERNEL(
-    beam_search_decode,
-    ops::BeamSearchDecodeXPUKernel<paddle::platform::XPUDeviceContext, float>,
-    ops::BeamSearchDecodeXPUKernel<paddle::platform::XPUDeviceContext, double>,
-    ops::BeamSearchDecodeXPUKernel<paddle::platform::XPUDeviceContext,
-                                   paddle::platform::float16>,
-    ops::BeamSearchDecodeXPUKernel<paddle::platform::XPUDeviceContext, int>,
-    ops::BeamSearchDecodeXPUKernel<paddle::platform::XPUDeviceContext,
-                                   int64_t>);
+namespace plat = paddle::platform;
+
+PD_REGISTER_STRUCT_KERNEL(beam_search_decode,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::BeamSearchDecodeXPUKernel,
+                          float,
+                          double,
+                          plat::float16,
+                          int,
+                          int64_t) {}
 #endif
diff --git a/paddle/fluid/operators/beam_search_op_xpu.cc b/paddle/fluid/operators/beam_search_op_xpu.cc
index 9f1d1488d9a64..4140fc6f2910d 100644
--- a/paddle/fluid/operators/beam_search_op_xpu.cc
+++ b/paddle/fluid/operators/beam_search_op_xpu.cc
@@ -18,11 +18,12 @@ limitations under the License. */
 #include "paddle/fluid/operators/beam_search_op.h"
 
 namespace ops = paddle::operators;
-using XPUCtx = paddle::platform::XPUDeviceContext;
-
-REGISTER_OP_XPU_KERNEL(beam_search,
-                       ops::BeamSearchOpKernel<float, XPUCtx>,
-                       ops::BeamSearchOpKernel<double, XPUCtx>,
-                       ops::BeamSearchOpKernel<int, XPUCtx>,
-                       ops::BeamSearchOpKernel<int64_t, XPUCtx>);
+PD_REGISTER_STRUCT_KERNEL(beam_search,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::BeamSearchOpKernel,
+                          float,
+                          double,
+                          int,
+                          int64_t) {}
 #endif
diff --git a/paddle/fluid/operators/collective/c_allgather_op_xpu.cc b/paddle/fluid/operators/collective/c_allgather_op_xpu.cc
index 107f5ccd1b563..1e7d3f3a9fec1 100644
--- a/paddle/fluid/operators/collective/c_allgather_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_allgather_op_xpu.cc
@@ -22,7 +22,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename T>
+template <typename T, typename DeviceContext>
 class CAllGatherOpXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
@@ -80,9 +80,12 @@ class CAllGatherOpXPUKernel : public framework::OpKernel<T> {
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-REGISTER_OP_XPU_KERNEL(c_allgather,
-                       ops::CAllGatherOpXPUKernel<float>,
-                       ops::CAllGatherOpXPUKernel<double>,
-                       ops::CAllGatherOpXPUKernel<int>,
-                       ops::CAllGatherOpXPUKernel<int64_t>,
-                       ops::CAllGatherOpXPUKernel<plat::float16>);
+PD_REGISTER_STRUCT_KERNEL(c_allgather,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::CAllGatherOpXPUKernel,
+                          float,
+                          double,
+                          plat::float16,
+                          int,
+                          int64_t) {}
diff --git a/paddle/fluid/operators/collective/c_allreduce_max_op_xpu.cc b/paddle/fluid/operators/collective/c_allreduce_max_op_xpu.cc
index 86527cf0e6e1e..8c648b4ae4a37 100644
--- a/paddle/fluid/operators/collective/c_allreduce_max_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_allreduce_max_op_xpu.cc
@@ -14,10 +14,18 @@ limitations under the License. */
 
 #include "paddle/fluid/operators/collective/c_allreduce_op.h"
 
+namespace paddle {
+namespace operators {
+DEFINE_C_ALLREDUCE_XPU_KERNEL(CAllReduceMax, kRedMax)
+}  // namespace operators
+}  // namespace paddle
+
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
-
-REGISTER_OP_XPU_KERNEL(c_allreduce_max,
-                       ops::CAllReduceOpXPUKernel<ops::kRedMax, plat::float16>,
-                       ops::CAllReduceOpXPUKernel<ops::kRedMax, int>,
-                       ops::CAllReduceOpXPUKernel<ops::kRedMax, float>)
+PD_REGISTER_STRUCT_KERNEL(c_allreduce_max,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::CAllReduceMaxXPUKernel,
+                          float,
+                          int,
+                          plat::float16) {}
diff --git a/paddle/fluid/operators/collective/c_allreduce_min_op_xpu.cc b/paddle/fluid/operators/collective/c_allreduce_min_op_xpu.cc
index b8a8cfab3f75f..f9be16781af70 100644
--- a/paddle/fluid/operators/collective/c_allreduce_min_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_allreduce_min_op_xpu.cc
@@ -14,8 +14,18 @@ limitations under the License. */
 
 #include "paddle/fluid/operators/collective/c_allreduce_op.h"
 
+namespace paddle {
+namespace operators {
+DEFINE_C_ALLREDUCE_XPU_KERNEL(CAllReduceMin, kRedMin)
+}  // namespace operators
+}  // namespace paddle
+
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
-
-REGISTER_OP_XPU_KERNEL(c_allreduce_min,
-                       ops::CAllReduceOpXPUKernel<ops::kRedMin, float>)
+PD_REGISTER_STRUCT_KERNEL(c_allreduce_min,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::CAllReduceMinXPUKernel,
+                          float,
+                          int,
+                          plat::float16) {}
diff --git a/paddle/fluid/operators/collective/c_allreduce_op.h b/paddle/fluid/operators/collective/c_allreduce_op.h
index a57486584463d..82cb3a3772e3a 100644
--- a/paddle/fluid/operators/collective/c_allreduce_op.h
+++ b/paddle/fluid/operators/collective/c_allreduce_op.h
@@ -223,6 +223,10 @@ class CAllReduceOpXPUKernel : public framework::OpKernel<T> {
   }
 };
 
+#define DEFINE_C_ALLREDUCE_XPU_KERNEL(op_name, red_type) \
+  template <typename T, typename DeviceContext>          \
+  class op_name##XPUKernel : public CAllReduceOpXPUKernel<red_type, T> {};
+
 template <ReduceType red_type, typename T>
 class CAllReduceOpCUDAKernel : public framework::OpKernel<T> {
  public:
diff --git a/paddle/fluid/operators/collective/c_allreduce_prod_op_xpu.cc b/paddle/fluid/operators/collective/c_allreduce_prod_op_xpu.cc
index 6e2b2df297313..5558b1722093a 100644
--- a/paddle/fluid/operators/collective/c_allreduce_prod_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_allreduce_prod_op_xpu.cc
@@ -14,8 +14,18 @@ limitations under the License. */
 
 #include "paddle/fluid/operators/collective/c_allreduce_op.h"
 
+namespace paddle {
+namespace operators {
+DEFINE_C_ALLREDUCE_XPU_KERNEL(CAllReduceProd, kRedProd)
+}  // namespace operators
+}  // namespace paddle
+
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
-
-REGISTER_OP_XPU_KERNEL(c_allreduce_prod,
-                       ops::CAllReduceOpXPUKernel<ops::kRedProd, float>)
+PD_REGISTER_STRUCT_KERNEL(c_allreduce_prod,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::CAllReduceProdXPUKernel,
+                          float,
+                          int,
+                          plat::float16) {}
diff --git a/paddle/fluid/operators/collective/c_allreduce_sum_op_xpu.cc b/paddle/fluid/operators/collective/c_allreduce_sum_op_xpu.cc
index a4d1c62e821ec..1d4c5f63b5850 100644
--- a/paddle/fluid/operators/collective/c_allreduce_sum_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_allreduce_sum_op_xpu.cc
@@ -14,10 +14,18 @@ limitations under the License. */
 
 #include "paddle/fluid/operators/collective/c_allreduce_op.h"
 
+namespace paddle {
+namespace operators {
+DEFINE_C_ALLREDUCE_XPU_KERNEL(CAllReduceSum, kRedSum)
+}  // namespace operators
+}  // namespace paddle
+
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
-
-REGISTER_OP_XPU_KERNEL(c_allreduce_sum,
-                       ops::CAllReduceOpXPUKernel<ops::kRedSum, float>,
-                       ops::CAllReduceOpXPUKernel<ops::kRedSum, plat::float16>,
-                       ops::CAllReduceOpXPUKernel<ops::kRedSum, int>)
+PD_REGISTER_STRUCT_KERNEL(c_allreduce_sum,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::CAllReduceSumXPUKernel,
+                          float,
+                          int,
+                          plat::float16) {}
diff --git a/paddle/fluid/operators/collective/c_broadcast_op_xpu.cc b/paddle/fluid/operators/collective/c_broadcast_op_xpu.cc
index 3f12adcea3f8f..676743e22c6cd 100644
--- a/paddle/fluid/operators/collective/c_broadcast_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_broadcast_op_xpu.cc
@@ -22,7 +22,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename T>
+template <typename T, typename DeviceContext>
 class CBroadcastOpXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
@@ -113,6 +113,9 @@ class CBroadcastOpXPUKernel : public framework::OpKernel<T> {
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-REGISTER_OP_XPU_KERNEL(c_broadcast,
-                       ops::CBroadcastOpXPUKernel<float>,
-                       ops::CBroadcastOpXPUKernel<plat::float16>);
+PD_REGISTER_STRUCT_KERNEL(c_broadcast,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::CBroadcastOpXPUKernel,
+                          float,
+                          plat::float16) {}
diff --git a/paddle/fluid/operators/collective/c_concat_op_xpu.cc b/paddle/fluid/operators/collective/c_concat_op_xpu.cc
index b31be9bde44dc..7ecf5d08dba84 100644
--- a/paddle/fluid/operators/collective/c_concat_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_concat_op_xpu.cc
@@ -28,7 +28,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename T>
+template <typename T, typename DeviceContext>
 class CConcatOpXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
@@ -118,6 +118,5 @@ class CConcatOpXPUKernel : public framework::OpKernel<T> {
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-REGISTER_OP_XPU_KERNEL(c_concat,
-                       ops::CConcatOpXPUKernel<float>,
-                       ops::CConcatOpXPUKernel<plat::float16>);
+PD_REGISTER_STRUCT_KERNEL(
+    c_concat, XPU, ALL_LAYOUT, ops::CConcatOpXPUKernel, float, plat::float16) {}
diff --git a/paddle/fluid/operators/collective/c_embedding_op_xpu.cc b/paddle/fluid/operators/collective/c_embedding_op_xpu.cc
index 8590ff257305f..b46a561532d99 100644
--- a/paddle/fluid/operators/collective/c_embedding_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_embedding_op_xpu.cc
@@ -15,7 +15,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename DeviceContext, typename T>
+template <typename T, typename DeviceContext>
 class CEmbeddingOpXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
@@ -68,7 +68,7 @@ class CEmbeddingOpXPUKernel : public framework::OpKernel<T> {
   }
 };
 
-template <typename DeviceContext, typename T>
+template <typename T, typename DeviceContext>
 class CEmbeddingGradOpXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& context) const override {
@@ -140,9 +140,7 @@ class CEmbeddingGradOpXPUKernel : public framework::OpKernel<T> {
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-REGISTER_OP_XPU_KERNEL(
-    c_embedding,
-    ops::CEmbeddingOpXPUKernel<paddle::platform::XPUDeviceContext, float>);
-REGISTER_OP_XPU_KERNEL(
-    c_embedding_grad,
-    ops::CEmbeddingGradOpXPUKernel<paddle::platform::XPUDeviceContext, float>);
+PD_REGISTER_STRUCT_KERNEL(
+    c_embedding, XPU, ALL_LAYOUT, ops::CEmbeddingOpXPUKernel, float) {}
+PD_REGISTER_STRUCT_KERNEL(
+    c_embedding_grad, XPU, ALL_LAYOUT, ops::CEmbeddingGradOpXPUKernel, float) {}
diff --git a/paddle/fluid/operators/collective/c_identity_op_xpu.cc b/paddle/fluid/operators/collective/c_identity_op_xpu.cc
index d92c90e3f6385..a627d2748b6d7 100644
--- a/paddle/fluid/operators/collective/c_identity_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_identity_op_xpu.cc
@@ -14,9 +14,12 @@ limitations under the License. */
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-REGISTER_OP_XPU_KERNEL(c_identity,
-                       ops::CIdentityOpKernel<float, plat::XPUPlace>,
-                       ops::CIdentityOpKernel<double, plat::XPUPlace>,
-                       ops::CIdentityOpKernel<int, plat::XPUPlace>,
-                       ops::CIdentityOpKernel<int64_t, plat::XPUPlace>,
-                       ops::CIdentityOpKernel<plat::float16, plat::XPUPlace>);
+PD_REGISTER_STRUCT_KERNEL(c_identity,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::CIdentityOpKernel,
+                          float,
+                          double,
+                          int,
+                          int64_t,
+                          plat::float16) {}
diff --git a/paddle/fluid/operators/collective/c_reduce_max_op_xpu.cc b/paddle/fluid/operators/collective/c_reduce_max_op_xpu.cc
index 3ae1af1d08315..6712a6eb500ee 100644
--- a/paddle/fluid/operators/collective/c_reduce_max_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_reduce_max_op_xpu.cc
@@ -14,8 +14,14 @@ limitations under the License. */
 
 #include "paddle/fluid/operators/collective/c_reduce_op.h"
 
+namespace paddle {
+namespace operators {
+DEFINE_C_REDUCE_XPU_KERNEL(CReduceMax, kRedMax);
+}  // namespace operators
+}  // namespace paddle
+
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-REGISTER_OP_XPU_KERNEL(c_reduce_max,
-                       ops::CReduceOpXPUKernel<ops::kRedMax, float>)
+PD_REGISTER_STRUCT_KERNEL(
+    c_reduce_max, XPU, ALL_LAYOUT, ops::CReduceMaxXPUKernel, float) {}
diff --git a/paddle/fluid/operators/collective/c_reduce_min_op_xpu.cc b/paddle/fluid/operators/collective/c_reduce_min_op_xpu.cc
index 500ea2abe6ab5..440c2b85acde3 100644
--- a/paddle/fluid/operators/collective/c_reduce_min_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_reduce_min_op_xpu.cc
@@ -14,8 +14,14 @@ limitations under the License. */
 
 #include "paddle/fluid/operators/collective/c_reduce_op.h"
 
+namespace paddle {
+namespace operators {
+DEFINE_C_REDUCE_XPU_KERNEL(CReduceMin, kRedMin);
+}  // namespace operators
+}  // namespace paddle
+
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-REGISTER_OP_XPU_KERNEL(c_reduce_min,
-                       ops::CReduceOpXPUKernel<ops::kRedMin, float>)
+PD_REGISTER_STRUCT_KERNEL(
+    c_reduce_min, XPU, ALL_LAYOUT, ops::CReduceMinXPUKernel, float) {}
diff --git a/paddle/fluid/operators/collective/c_reduce_op.h b/paddle/fluid/operators/collective/c_reduce_op.h
index 4ff84af7a6ee2..09b9b9f5a5b2b 100644
--- a/paddle/fluid/operators/collective/c_reduce_op.h
+++ b/paddle/fluid/operators/collective/c_reduce_op.h
@@ -198,6 +198,10 @@ class CReduceOpXPUKernel : public framework::OpKernel<T> {
   }
 };
 
+#define DEFINE_C_REDUCE_XPU_KERNEL(op_name, red_type) \
+  template <typename T, typename DeviceContext>       \
+  class op_name##XPUKernel : public CReduceOpXPUKernel<red_type, T> {};
+
 template <ReduceType red_type, typename T>
 class CReduceOpCUDAKernel : public framework::OpKernel<T> {
  public:
diff --git a/paddle/fluid/operators/collective/c_reduce_prod_op_xpu.cc b/paddle/fluid/operators/collective/c_reduce_prod_op_xpu.cc
index 15ef32289655e..1541918396d07 100644
--- a/paddle/fluid/operators/collective/c_reduce_prod_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_reduce_prod_op_xpu.cc
@@ -14,8 +14,14 @@ limitations under the License. */
 
 #include "paddle/fluid/operators/collective/c_reduce_op.h"
 
+namespace paddle {
+namespace operators {
+DEFINE_C_REDUCE_XPU_KERNEL(CReduceProd, kRedProd);
+}  // namespace operators
+}  // namespace paddle
+
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-REGISTER_OP_XPU_KERNEL(c_reduce_prod,
-                       ops::CReduceOpXPUKernel<ops::kRedProd, float>)
+PD_REGISTER_STRUCT_KERNEL(
+    c_reduce_prod, XPU, ALL_LAYOUT, ops::CReduceProdXPUKernel, float) {}
diff --git a/paddle/fluid/operators/collective/c_reduce_sum_op_xpu.cc b/paddle/fluid/operators/collective/c_reduce_sum_op_xpu.cc
index 9f35a6866eeb1..230dca3503538 100644
--- a/paddle/fluid/operators/collective/c_reduce_sum_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_reduce_sum_op_xpu.cc
@@ -14,9 +14,14 @@ limitations under the License. */
 
 #include "paddle/fluid/operators/collective/c_reduce_op.h"
 
+namespace paddle {
+namespace operators {
+DEFINE_C_REDUCE_XPU_KERNEL(CReduceSum, kRedSum);
+}  // namespace operators
+}  // namespace paddle
+
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-REGISTER_OP_XPU_KERNEL(c_reduce_sum,
-                       ops::CReduceOpXPUKernel<ops::kRedSum, plat::float16>,
-                       ops::CReduceOpXPUKernel<ops::kRedSum, float>)
+PD_REGISTER_STRUCT_KERNEL(
+    c_reduce_sum, XPU, ALL_LAYOUT, ops::CReduceSumXPUKernel, float) {}
diff --git a/paddle/fluid/operators/collective/mp_allreduce_sum_op_xpu.cc b/paddle/fluid/operators/collective/mp_allreduce_sum_op_xpu.cc
index 39dcc3470f6ac..9638bf68d1717 100644
--- a/paddle/fluid/operators/collective/mp_allreduce_sum_op_xpu.cc
+++ b/paddle/fluid/operators/collective/mp_allreduce_sum_op_xpu.cc
@@ -14,10 +14,18 @@
 
 #include "paddle/fluid/operators/collective/c_allreduce_op.h"
 
+namespace paddle {
+namespace operators {
+DEFINE_C_ALLREDUCE_XPU_KERNEL(CAllReduceSum, kRedSum)
+}  // namespace operators
+}  // namespace paddle
+
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
-
-REGISTER_OP_XPU_KERNEL(mp_allreduce_sum,
-                       ops::CAllReduceOpXPUKernel<ops::kRedSum, float>,
-                       ops::CAllReduceOpXPUKernel<ops::kRedSum, plat::float16>,
-                       ops::CAllReduceOpXPUKernel<ops::kRedSum, int>)
+PD_REGISTER_STRUCT_KERNEL(mp_allreduce_sum,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::CAllReduceSumXPUKernel,
+                          float,
+                          int,
+                          plat::float16) {}

From 00f747f2cce7251d23f7688c334de98bd5692dfa Mon Sep 17 00:00:00 2001
From: Yuanle Liu <yuanlehome@163.com>
Date: Tue, 25 Apr 2023 16:20:07 +0800
Subject: [PATCH 057/405] [Paddle Inference] add generic plugin for p_norm
 (#53278)

---
 .../tensorrt/dynamic_shape_infermeta.cc       |  51 +++++--
 .../dynamic_shape_infermeta_registry.h        |   1 +
 .../tensorrt/plugin/generic_plugin.cu         |   2 +-
 test/ir/inference/test_trt_convert_p_norm.py  | 134 ++++++++++++++++++
 4 files changed, 177 insertions(+), 11 deletions(-)
 create mode 100644 test/ir/inference/test_trt_convert_p_norm.py

diff --git a/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta.cc b/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta.cc
index 8752bdf793329..fe0c4e0d18ef4 100644
--- a/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta.cc
+++ b/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta.cc
@@ -14,7 +14,7 @@
 
 #include "paddle/fluid/inference/tensorrt/dynamic_shape_infermeta_factory.h"
 #include "paddle/fluid/inference/tensorrt/helper.h"
-#include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/core/enforce.h"
 #include "paddle/phi/kernels/funcs/unfold_functor.h"
 
 namespace paddle {
@@ -322,20 +322,51 @@ nvinfer1::DimsExprs PNormInferMeta(
     int nb_inputs,
     nvinfer1::IExprBuilder& expr_builder,  // NOLINT
     const framework::OpDesc& op_desc) {
-  const nvinfer1::DimsExprs x_dim = inputs[0];
-  std::vector<const nvinfer1::IDimensionExpr*> reduce_dims;
-  std::vector<const nvinfer1::IDimensionExpr*> keep_dims;
-
   bool asvector = PADDLE_GET_CONST(bool, op_desc.GetAttr("asvector"));
   bool keepdim = PADDLE_GET_CONST(bool, op_desc.GetAttr("keepdim"));
   int axis = PADDLE_GET_CONST(int, op_desc.GetAttr("axis"));
 
+  auto x_dim = inputs[0];
+  auto x_rank = x_dim.nbDims;
+
+  PADDLE_ENFORCE_GE(axis,
+                    -x_rank,
+                    phi::errors::InvalidArgument(
+                        "Attr(axis) value should be in range [-R, R-1], R is "
+                        "the rank of Input(X). But received axis: %d, R: %d. "
+                        "Current Input(X)'s shape is=[%s].",
+                        axis,
+                        x_rank,
+                        x_dim.d));
+  PADDLE_ENFORCE_LT(axis,
+                    x_rank,
+                    phi::errors::InvalidArgument(
+                        "Attr(axis) value should be in range [-R, R-1], R is "
+                        "the rank of Input(X). But received axis: %d, R: %d. "
+                        "Current Input(X)'s shape is=[%s].",
+                        axis,
+                        x_rank,
+                        x_dim.d));
+
+  // TODO(liuyuanle): support asvector = True
+  PADDLE_ENFORCE_EQ(
+      asvector,
+      false,
+      phi::errors::InvalidArgument(
+          "p_norm only support asvector=false, but received asvector: %d.",
+          asvector));
+
+  std::vector<const nvinfer1::IDimensionExpr*> reduce_dims;
+
   if (asvector) {
     reduce_dims.emplace_back(expr_builder.constant(1));
-    keep_dims.emplace_back(expr_builder.constant(1));
     if (keepdim) {
       for (int i = 1; i < x_dim.nbDims; ++i) {
-        keep_dims.emplace_back(expr_builder.constant(1));
+        reduce_dims.emplace_back(expr_builder.constant(1));
+      }
+      x_dim.nbDims = reduce_dims.size();
+      for (size_t i = 0; i < reduce_dims.size(); i++) {
+        x_dim.d[i] = reduce_dims[i];
       }
     }
   } else {
@@ -347,12 +378,11 @@ nvinfer1::DimsExprs PNormInferMeta(
       reduce_dims.emplace_back(expr_builder.constant(1));
     }
   }
-  keep_dims[axis] = expr_builder.constant(1);
+  x_dim.d[axis] = expr_builder.constant(1);
 
   nvinfer1::DimsExprs output;
   if (keepdim) {
-    output.nbDims = keep_dims.size();
-    for (int i = 0; i < output.nbDims; i++) output.d[i] = keep_dims[i];
+    output = x_dim;
   } else {
     output.nbDims = reduce_dims.size();
     for (int i = 0; i < output.nbDims; i++) output.d[i] = reduce_dims[i];
@@ -396,6 +426,7 @@ PD_REGISTER_DYNAMIC_INFER_META_FN(inverse, UnchangedInferMeta);
 PD_REGISTER_DYNAMIC_INFER_META_FN(moe, MoeInferMeta);
 PD_REGISTER_DYNAMIC_INFER_META_FN(pad3d, Pad3dInferMeta);
 PD_REGISTER_DYNAMIC_INFER_META_FN(grid_sampler, GridSamplerInferMeta);
+PD_REGISTER_DYNAMIC_INFER_META_FN(p_norm, PNormInferMeta);
 }  // namespace tensorrt
 }  // namespace inference
 }  // namespace paddle
diff --git a/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta_registry.h b/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta_registry.h
index c0ddaf5d983ef..db9b6e9313b5d 100644
--- a/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta_registry.h
+++ b/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta_registry.h
@@ -28,6 +28,7 @@ USE_TRT_DYNAMIC_INFER_META_FN(scatter_nd_add);
 USE_TRT_DYNAMIC_INFER_META_FN(pad3d);
 USE_TRT_DYNAMIC_INFER_META_FN(inverse);
 USE_TRT_DYNAMIC_INFER_META_FN(grid_sampler);
+USE_TRT_DYNAMIC_INFER_META_FN(p_norm);
 }  // namespace tensorrt
 }  // namespace inference
 }  // namespace paddle
diff --git a/paddle/fluid/inference/tensorrt/plugin/generic_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/generic_plugin.cu
index e97aa95a6aa03..f9bbb27958676 100644
--- a/paddle/fluid/inference/tensorrt/plugin/generic_plugin.cu
+++ b/paddle/fluid/inference/tensorrt/plugin/generic_plugin.cu
@@ -472,7 +472,7 @@ int GenericPlugin::enqueue(const nvinfer1::PluginTensorDesc* input_desc,
                            cudaStream_t stream) TRT_NOEXCEPT {
   platform::CUDAPlace place(platform::GetCurrentDeviceId());
 
-  // [TODO]now generic plugin do not support FP16 and INT8 precision
+  // [TODO]now generic plugin do not support INT8 precision
   auto protoType2PhiType =
       [&](int proto_type,
           nvinfer1::DataType nv_dtype) -> std::pair<phi::DataType, int> {
diff --git a/test/ir/inference/test_trt_convert_p_norm.py b/test/ir/inference/test_trt_convert_p_norm.py
new file mode 100644
index 0000000000000..5aa48135a89f6
--- /dev/null
+++ b/test/ir/inference/test_trt_convert_p_norm.py
@@ -0,0 +1,134 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from functools import partial
+from typing import Any, Dict, List
+
+import numpy as np
+from program_config import ProgramConfig, TensorConfig
+from trt_layer_auto_scan_test import TrtLayerAutoScanTest
+
+import paddle.inference as paddle_infer
+
+
+class TrtConvertCeluTest(TrtLayerAutoScanTest):
+    def sample_program_configs(self):
+        def generate_input1(dims, attrs: List[Dict[str, Any]]):
+            if dims == 1:
+                return np.ones([3]).astype(np.float32)
+            elif dims == 2:
+                return np.ones([3, 64]).astype(np.float32)
+            elif dims == 3:
+                return np.ones([3, 64, 64]).astype(np.float32)
+            else:
+                return np.ones([1, 3, 64, 64]).astype(np.float32)
+
+        for dims in [2, 3, 4]:
+            # TODO(liuyuanle): support asvector = True
+            for asvector in [False]:
+                for keepdim in [False, True]:
+                    for porder in [0, 1, 2, 3]:
+                        for axis in [-1]:
+                            self.dims = dims
+
+                            dics = [
+                                {
+                                    "asvector": asvector,
+                                    "keepdim": keepdim,
+                                    "axis": axis,
+                                    "porder": porder,
+                                }
+                            ]
+
+                            ops_config = [
+                                {
+                                    "op_type": "p_norm",
+                                    "op_inputs": {
+                                        "X": ["input_data"],
+                                    },
+                                    "op_outputs": {"Out": ["output_data"]},
+                                    "op_attrs": dics[0],
+                                }
+                            ]
+                            ops = self.generate_op_config(ops_config)
+
+                            program_config = ProgramConfig(
+                                ops=ops,
+                                weights={},
+                                inputs={
+                                    "input_data": TensorConfig(
+                                        data_gen=partial(
+                                            generate_input1, dims, dics
+                                        )
+                                    )
+                                },
+                                outputs=["output_data"],
+                            )
+
+                            yield program_config
+
+    def sample_predictor_configs(
+        self, program_config
+    ) -> (paddle_infer.Config, List[int], float):
+        def generate_dynamic_shape(attrs):
+            if self.dims == 1:
+                self.dynamic_shape.min_input_shape = {"input_data": [1]}
+                self.dynamic_shape.max_input_shape = {"input_data": [128]}
+                self.dynamic_shape.opt_input_shape = {"input_data": [64]}
+            elif self.dims == 2:
+                self.dynamic_shape.min_input_shape = {"input_data": [1, 32]}
+                self.dynamic_shape.max_input_shape = {"input_data": [4, 64]}
+                self.dynamic_shape.opt_input_shape = {"input_data": [3, 64]}
+            elif self.dims == 3:
+                self.dynamic_shape.min_input_shape = {"input_data": [1, 32, 32]}
+                self.dynamic_shape.max_input_shape = {
+                    "input_data": [10, 64, 64]
+                }
+                self.dynamic_shape.opt_input_shape = {"input_data": [3, 64, 64]}
+            else:
+                self.dynamic_shape.min_input_shape = {
+                    "input_data": [1, 3, 32, 32]
+                }
+                self.dynamic_shape.max_input_shape = {
+                    "input_data": [4, 3, 64, 64]
+                }
+                self.dynamic_shape.opt_input_shape = {
+                    "input_data": [1, 3, 64, 64]
+                }
+
+        def generate_trt_nodes_num(attrs, dynamic_shape):
+            return 1, 2
+
+        attrs = [
+            program_config.ops[i].attrs for i in range(len(program_config.ops))
+        ]
+
+        # for dynamic_shape mode
+        generate_dynamic_shape(attrs)
+        self.trt_param.precision = paddle_infer.PrecisionType.Float32
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), 1e-5
+        self.trt_param.precision = paddle_infer.PrecisionType.Half
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), (1e-3, 1e-3)
+
+    def test(self):
+        self.run_test()
+
+
+if __name__ == "__main__":
+    unittest.main()

From 503f422eaa7a7c4aaeb87a202e3877168c77a1b9 Mon Sep 17 00:00:00 2001
From: wuhuachaocoding <77733235+wuhuachaocoding@users.noreply.github.com>
Date: Tue, 25 Apr 2023 16:22:20 +0800
Subject: [PATCH 058/405] add mp_sync config. (#53254)

---
 .../framework/distributed_strategy.proto      |  1 +
 .../fleet/base/distributed_strategy.py        |  6 ++
 .../hybrid_parallel_optimizer.py              | 95 +++++++++++++------
 .../fleet/hybrid_parallel_mp_model.py         | 52 +++++++++-
 .../fleet/test_fleet_distributed_strategy.py  | 30 ++++++
 5 files changed, 150 insertions(+), 34 deletions(-)

diff --git a/paddle/fluid/framework/distributed_strategy.proto b/paddle/fluid/framework/distributed_strategy.proto
index d0e494ad49491..6b093e9ee03b8 100755
--- a/paddle/fluid/framework/distributed_strategy.proto
+++ b/paddle/fluid/framework/distributed_strategy.proto
@@ -55,6 +55,7 @@ message MpConfig {
     optional bool sync_param= 1 [ default = false ];
     optional bool sync_grad= 2 [ default = false ];
     optional bool sync_moment= 3 [ default = false ];
+    optional string sync_mode= 4 [ default = 'broadcast' ];
 }
 
 message PpConfig {
diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py
index 14e2fc09d3390..b751989102483 100755
--- a/python/paddle/distributed/fleet/base/distributed_strategy.py
+++ b/python/paddle/distributed/fleet/base/distributed_strategy.py
@@ -146,6 +146,8 @@ def __init__(self):
             self.strategy.sync_nccl_allreduce = bool(_global_flags()[key])
 
         self.hybrid_parallel_order = ['dp', 'pp', 'sharding', 'mp']
+        self.sync_param_name = ["embedding", "layer_norm", ".b_"]
+
         self.__lock_attr = True
         logger.info("distributed strategy initialized")
 
@@ -1698,6 +1700,10 @@ def hybrid_configs(self, configs):
         )
 
         if "mp_configs" in configs:
+            if "sync_param_name" in configs["mp_configs"]:
+                self.sync_param_name = configs["mp_configs"]["sync_param_name"]
+                configs["mp_configs"].pop("sync_param_name")
+
             assign_configs_value(
                 self.strategy.hybrid_configs.mp_configs, configs["mp_configs"]
             )
diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py
index ab1b270e2fd88..3254bffb254c6 100755
--- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py
+++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py
@@ -303,9 +303,20 @@ def __init__(self, optimizer, hcg, strategy):
                                 inner_opt._grad_clip, hcg
                             )
 
-    def _filter_fn(self, param):
+    def _insert_sync(self, sync_var, src, mp_group, sync_mode):
+        if sync_mode == "broadcast":
+            paddle.distributed.broadcast(
+                sync_var, src=src, group=mp_group, sync_op=True
+            )
+        else:
+            paddle.distributed.all_reduce(
+                sync_var, group=mp_group, sync_op=True
+            )
+            sync_var.scale_(1.0 / mp_group.nranks)
+
+    def _filter_fn(self, param, strategy):
         p_name = param.name
-        tar_param = ["embedding", "layer_norm", ".b_"]
+        tar_param = strategy.sync_param_name
         if param.is_distributed is False:
             for tar in tar_param:
                 if tar in p_name:
@@ -329,26 +340,48 @@ def _step(self, parameters_list):
             or mp_configs.sync_moment
         ):
             params = sorted(
-                [p for p in parameters_list if self._filter_fn(p)],
+                [
+                    p
+                    for p in parameters_list
+                    if self._filter_fn(p, fleet.fleet._user_defined_strategy)
+                ],
                 key=lambda p: p.name,
             )
 
+        # Grad sync before opt
         if mp_group.nranks > 1 and mp_configs and mp_configs.sync_grad:
             for p in params:
-                if p.grad is None:
-                    continue
-                paddle.distributed.broadcast(
-                    p.grad, src=src_rank, group=mp_group, sync_op=True
-                )
+                if hasattr(p, "main_grad") and p.main_grad is not None:
+                    assert p.grad is None
+                    self._insert_sync(
+                        p.main_grad, src_rank, mp_group, mp_configs.sync_mode
+                    )
+                elif p.grad is not None:
+                    self._insert_sync(
+                        p.grad, src_rank, mp_group, mp_configs.sync_mode
+                    )
 
         self._inner_opt.step()
 
         if mp_group.nranks > 1 and mp_configs and mp_configs.sync_param:
             for p in params:
-                paddle.distributed.broadcast(
-                    p, src=src_rank, group=mp_group, sync_op=True
-                )
+                # Param sync after opt
+                self._insert_sync(p, src_rank, mp_group, mp_configs.sync_mode)
+
+                # Master param sync after opt
+                if (
+                    hasattr(self._inner_opt, "_multi_precision")
+                    and self._inner_opt._multi_precision
+                    and p.name in self._inner_opt._master_weights
+                ):
+                    self._insert_sync(
+                        self._inner_opt._master_weights[p.name],
+                        src_rank,
+                        mp_group,
+                        mp_configs.sync_mode,
+                    )
 
+        # Moment sync after opt
         if mp_group.nranks > 1 and mp_configs and mp_configs.sync_moment:
             for p in params:
                 # support opt state of adam and adamw to broadcast now.
@@ -357,28 +390,30 @@ def _step(self, parameters_list):
                     (paddle.optimizer.Adam, paddle.optimizer.AdamW),
                 ):
                     if (
-                        self._inner_opt._multi_precision
-                        and p.name in self._master_weights
+                        p.name
+                        in self._inner_opt._accumulators[
+                            self._inner_opt._moment1_acc_str
+                        ]
                     ):
-                        paddle.distributed.broadcast(
-                            self._inner_opt._master_weights[p.name],
-                            src=src_rank,
-                            group=mp_group,
-                            sync_op=True,
+                        moment1 = self._inner_opt._get_accumulator(
+                            self._inner_opt._moment1_acc_str, p
+                        )
+                        self._insert_sync(
+                            moment1, src_rank, mp_group, mp_configs.sync_mode
                         )
 
-                    moment1 = self._inner_opt._get_accumulator(
-                        self._inner_opt._moment1_acc_str, p
-                    )
-                    moment2 = self._inner_opt._get_accumulator(
-                        self._inner_opt._moment2_acc_str, p
-                    )
-                    paddle.distributed.broadcast(
-                        moment1, src=src_rank, group=mp_group, sync_op=True
-                    )
-                    paddle.distributed.broadcast(
-                        moment2, src=src_rank, group=mp_group, sync_op=True
-                    )
+                    if (
+                        p.name
+                        in self._inner_opt._accumulators[
+                            self._inner_opt._moment2_acc_str
+                        ]
+                    ):
+                        moment2 = self._inner_opt._get_accumulator(
+                            self._inner_opt._moment2_acc_str, p
+                        )
+                        self._insert_sync(
+                            moment2, src_rank, mp_group, mp_configs.sync_mode
+                        )
 
     @no_grad()
     @framework.dygraph_only
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_model.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_model.py
index 26e740bfa6b79..82efb6fa46657 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_model.py
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_model.py
@@ -202,6 +202,7 @@ def build_model_optimizer_train(
         self,
         batchs,
         fp16=False,
+        amp_level="O1",
         mp_sync_param=False,
         mp_sync_grad=False,
         mp_sync_moment=False,
@@ -232,6 +233,11 @@ def build_model_optimizer_train(
             learning_rate=0.1, parameters=model.parameters()
         )
 
+        if fp16 and amp_level == "O2":
+            model, optimizer = paddle.amp.decorate(
+                models=model, optimizers=optimizer, level='O2'
+            )
+
         strategy = fleet.fleet._user_defined_strategy
         strategy.hybrid_configs = {
             "dp_degree": self.data_parallel_size,
@@ -246,15 +252,15 @@ def build_model_optimizer_train(
 
         model = fleet.distributed_model(model)
         optimizer = fleet.distributed_optimizer(optimizer)
-        return self.train_batch(batchs, model, optimizer, fp16)
+        return self.train_batch(batchs, model, optimizer, fp16, amp_level)
 
-    def train_batch(self, batchs, model, optimizer, fp16=False):
+    def train_batch(self, batchs, model, optimizer, fp16=False, amp_level="O1"):
         losses = []
         if fp16:
             scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
             scaler = fleet.distributed_scaler(scaler)
         for batch in batchs:
-            with paddle.amp.auto_cast(enable=fp16, level='O1'):
+            with paddle.amp.auto_cast(enable=fp16, level=amp_level):
                 output = model(batch)
                 loss = output.mean()
                 losses.append(loss.numpy())
@@ -295,7 +301,7 @@ def mp_sync_base(
         for i in range(len(losses)):
             np.testing.assert_allclose(losses[i], losses_sync[i], rtol=1e-6)
 
-        # test fp16
+        # test fp16 O1
         losses_fp16 = self.build_model_optimizer_train(batchs, fp16=True)
         losses_sync_fp16 = self.build_model_optimizer_train(
             batchs,
@@ -310,6 +316,24 @@ def mp_sync_base(
                 losses_fp16[i], losses_sync_fp16[i], rtol=1e-6
             )
 
+        # test fp16 O2
+        losses_fp16_O2 = self.build_model_optimizer_train(
+            batchs, fp16=True, amp_level="O2"
+        )
+        losses_sync_fp16_O2 = self.build_model_optimizer_train(
+            batchs,
+            fp16=True,
+            amp_level="O2",
+            mp_sync_param=mp_sync_param,
+            mp_sync_grad=mp_sync_grad,
+            mp_sync_moment=mp_sync_moment,
+        )
+
+        for i in range(len(losses_fp16_O2)):
+            np.testing.assert_allclose(
+                losses_fp16_O2[i], losses_sync_fp16_O2[i], rtol=1e-6
+            )
+
     def test_mp_sync_param(self):
         self.mp_sync_base(mp_sync_param=True)
 
@@ -325,6 +349,26 @@ def test_mp_sync_all(self):
         )
 
 
+class TestDistMPSyncModelTraning(TestDistMPSyncTraning):
+    def setUp(self):
+        strategy = fleet.DistributedStrategy()
+        self.model_parallel_size = 2
+        self.data_parallel_size = 1
+        strategy.hybrid_configs = {
+            "dp_degree": self.data_parallel_size,
+            "mp_degree": self.model_parallel_size,
+            "pp_degree": 1,
+            "mp_configs": {
+                "sync_param": False,
+                "sync_grad": False,
+                "sync_moment": False,
+                "sync_mode": "average",
+                "sync_param_name": ["embedding", "layer_norm", ".b_"],
+            },
+        }
+        fleet.init(is_collective=True, strategy=strategy)
+
+
 class TestDistMPTraning(unittest.TestCase):
     def setUp(self):
         strategy = fleet.DistributedStrategy()
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_distributed_strategy.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_distributed_strategy.py
index 99f235b5887fd..ba49cbf125a62 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_distributed_strategy.py
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_distributed_strategy.py
@@ -84,6 +84,36 @@ def test_hybrid_parallel_configs(self):
         self.assertEqual(strategy.hybrid_configs["mp_degree"], 2)
         self.assertEqual(strategy.hybrid_configs["pp_degree"], 4)
 
+    def test_hybrid_parallel_mp_configs(self):
+        strategy = paddle.distributed.fleet.DistributedStrategy()
+        strategy.hybrid_configs = {
+            "dp_degree": 1,
+            "mp_degree": 2,
+            "pp_degree": 4,
+            "mp_configs": {
+                "sync_param": True,
+                "sync_grad": False,
+                "sync_moment": False,
+                "sync_mode": "broadcast",
+                "sync_param_name": ["embedding", "layer_norm", ".w", ".b_"],
+            },
+        }
+        self.assertEqual(strategy.hybrid_configs["dp_degree"], 1)
+        self.assertEqual(strategy.hybrid_configs["mp_degree"], 2)
+        self.assertEqual(strategy.hybrid_configs["pp_degree"], 4)
+        self.assertEqual(strategy.hybrid_configs["mp_configs"].sync_param, True)
+        self.assertEqual(strategy.hybrid_configs["mp_configs"].sync_grad, False)
+        self.assertEqual(
+            strategy.hybrid_configs["mp_configs"].sync_moment, False
+        )
+        self.assertEqual(
+            strategy.hybrid_configs["mp_configs"].sync_mode, "broadcast"
+        )
+
+        self.assertEqual(
+            strategy.sync_param_name, ["embedding", "layer_norm", ".w", ".b_"]
+        )
+
     def test_hybrid_parallel_configs_order(self):
         strategy = paddle.distributed.fleet.DistributedStrategy()
         strategy.hybrid_configs = {

From be1b3fc3649b694af3aaaa468d2df20e35ede3c7 Mon Sep 17 00:00:00 2001
From: sprouteer <89541335+sprouteer@users.noreply.github.com>
Date: Tue, 25 Apr 2023 16:38:41 +0800
Subject: [PATCH 059/405] [XPU][BUG] Fix link_xpu_op_max_pass bug  (#53258)

---
 .../framework/ir/xpu/conv2d_xpu_fuse_pass.cc  |  17 +--
 .../framework/ir/xpu/link_xpu_op_max_pass.cc  | 105 +++++++++++++-----
 2 files changed, 88 insertions(+), 34 deletions(-)

diff --git a/paddle/fluid/framework/ir/xpu/conv2d_xpu_fuse_pass.cc b/paddle/fluid/framework/ir/xpu/conv2d_xpu_fuse_pass.cc
index af5ec2c651c8a..263ea5a09ca76 100644
--- a/paddle/fluid/framework/ir/xpu/conv2d_xpu_fuse_pass.cc
+++ b/paddle/fluid/framework/ir/xpu/conv2d_xpu_fuse_pass.cc
@@ -110,7 +110,8 @@ Conv2dXPUPattern::Conv2dXPUPattern(PDPattern* pattern,
                          ->assert_is_op_input(conv_type_, "Filter")
                          ->AsInput();
   auto conv_out = pattern->NewNode(conv_out_repr())
-                      ->assert_is_op_output(conv_type_, "Output");
+                      ->assert_is_op_output(conv_type_, "Output")
+                      ->assert_has_n_outputs(1);
   conv->LinksFrom({input, conv_filter}).LinksTo({conv_out});
   // ew_bias_add op
   PDNode* ew_bias_add = nullptr;
@@ -190,12 +191,12 @@ Conv2dXPUPattern::Conv2dXPUPattern(PDPattern* pattern,
   // ew_branch_add op
   if (with_branch_) {
     if (with_branch_x_) {
-      bn_out->assert_is_op_input("elementwise_add", "Y")->AsIntermediate();
+      bn_out->assert_is_op_input("elementwise_add", "Y");
       ew_branch_add_in = pattern->NewNode(ew_branch_add_in_repr())
                              ->assert_is_op_input("elementwise_add", "X")
                              ->AsInput();
     } else if (with_branch_y_) {
-      bn_out->assert_is_op_input("elementwise_add", "X")->AsIntermediate();
+      bn_out->assert_is_op_input("elementwise_add", "X");
       ew_branch_add_in = pattern->NewNode(ew_branch_add_in_repr())
                              ->assert_is_op_input("elementwise_add", "Y")
                              ->AsInput();
@@ -221,13 +222,15 @@ Conv2dXPUPattern::Conv2dXPUPattern(PDPattern* pattern,
   }
   // act op
   if (!act_type_.empty()) {
-    ew_branch_add_out->assert_is_op_input(act_type_, "X")->AsIntermediate();
+    ew_branch_add_out->assert_is_op_input(act_type_, "X");
     act = pattern->NewNode(act_repr())->assert_is_op(act_type_);
-    act_out = pattern->NewNode(act_out_repr())
-                  ->assert_is_op_output(act_type_, "Out")
-                  ->assert_var_not_persistable();
+    act_out =
+        pattern->NewNode(act_out_repr())->assert_is_op_output(act_type_, "Out");
     act->LinksFrom({ew_branch_add_out}).LinksTo({act_out});
+  } else {
+    act_out = ew_branch_add_out;
   }
+  act_out->AsOutput();
 }
 
 }  // namespace patterns
diff --git a/paddle/fluid/framework/ir/xpu/link_xpu_op_max_pass.cc b/paddle/fluid/framework/ir/xpu/link_xpu_op_max_pass.cc
index e100db36c2735..5b3f1332907a9 100644
--- a/paddle/fluid/framework/ir/xpu/link_xpu_op_max_pass.cc
+++ b/paddle/fluid/framework/ir/xpu/link_xpu_op_max_pass.cc
@@ -41,30 +41,39 @@ namespace patterns {
 struct FusionXPUOpPattern : public PatternBase {
   FusionXPUOpPattern(PDPattern* pattern,
                      const std::string& name_scope,
-                     const std::string& op_type);
+                     const std::string& op_type,
+                     bool with_branch);
 
   // declare operator node's name
   PATTERN_DECL_NODE(fusion_op);
   // declare variable node's name
-  PATTERN_DECL_NODE(out);
-  PATTERN_DECL_NODE(out_max);
+  PATTERN_DECL_NODE(input);
+  PATTERN_DECL_NODE(branch);
 
  private:
   std::string op_type_;
+  bool with_branch_{false};
 };
 
 FusionXPUOpPattern::FusionXPUOpPattern(PDPattern* pattern,
                                        const std::string& name_scope,
-                                       const std::string& op_type)
-    : PatternBase(pattern, name_scope, name_scope), op_type_(op_type) {
+                                       const std::string& op_type,
+                                       bool with_branch)
+    : PatternBase(pattern, name_scope, name_scope),
+      op_type_(op_type),
+      with_branch_(with_branch) {
   auto* fusion_op = pattern->NewNode(fusion_op_repr())->assert_is_op(op_type_);
-  auto* out = pattern->NewNode(out_repr())
-                  ->assert_is_op_output(op_type_, "out")
-                  ->assert_var_not_persistable();
-  auto* out_max = pattern->NewNode(out_max_repr())
-                      ->assert_is_op_output(op_type_, "out_max")
-                      ->assert_var_not_persistable();
-  fusion_op->LinksTo({out, out_max});
+  auto* input =
+      pattern->NewNode(input_repr())->assert_is_op_input(op_type_, "x");
+
+  PDNode* branch = nullptr;
+  if (with_branch_) {
+    branch =
+        pattern->NewNode(branch_repr())->assert_is_op_input(op_type_, "branch");
+    fusion_op->LinksFrom({input, branch});
+  } else {
+    fusion_op->LinksFrom({input});
+  }
 }
 
 }  // namespace patterns
@@ -74,7 +83,9 @@ class LinkXPUOpMaxPass : public FusePassBase {
   void ApplyImpl(ir::Graph* graph) const override;
 
  private:
-  void ApplyImpl(ir::Graph* graph, const std::string& op_type) const;
+  void ApplyImpl(ir::Graph* graph,
+                 const std::string& op_type,
+                 bool with_branch) const;
 
   const std::string name_scope_{"multi_encoder_xpu_slice_fuse_pass"};
   // ops with x_max/out_max
@@ -89,8 +100,7 @@ Origin subgraph:
           out0   out0_max
             |
             \
-          fusion_xpu_op1
-
+            fusion_op
 Fused subgraph:
           fusion_xpu_op0
             /       \
@@ -98,36 +108,77 @@ Fused subgraph:
           out0   out0_max
             |       |
             \       /
-          fusion_xpu_op1
+            fusion_op
+
+Origin subgraph1:
+          fusion_xpu_op0     fusion_xpu_op1
+            /       \         /          \
+            |       |         |          |
+          out0   out0_max    out1      out1_max
+            |                 |
+        (x) \                / (branch)
+              fusion_xpu_op2
+Fused subgraph1:
+          fusion_xpu_op0     fusion_xpu_op1
+            /       \         /           \
+            |       |         |            |
+          out0   out0_max    out1      out1_max
+            |       |          |           |
+        (x) \       |(x_max)   |(branch)  /(branch_max)
+             \      |          |         /
+              \     |          |        /
+               \    |          |       /
+                   fusion_xpu_op2
 */
 void LinkXPUOpMaxPass::ApplyImpl(ir::Graph* graph) const {
   Init(name_scope_, graph);
   for (auto op_type : op_types_) {
-    ApplyImpl(graph, op_type);
+    for (auto with_branch : {true, false}) {
+      ApplyImpl(graph, op_type, with_branch);
+    }
   }
 }
 
 void LinkXPUOpMaxPass::ApplyImpl(ir::Graph* graph,
-                                 const std::string& op_type) const {
+                                 const std::string& op_type,
+                                 bool with_branch) const {
   PADDLE_ENFORCE_NOT_NULL(
       graph, platform::errors::PreconditionNotMet("graph should not be null."));
   GraphPatternDetector gpd;
   patterns::FusionXPUOpPattern pattern(
-      gpd.mutable_pattern(), name_scope_, op_type);
+      gpd.mutable_pattern(), name_scope_, op_type, with_branch);
 
   int found_subgraph_count = 0;
   auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph,
                      Graph* graph) {
     VLOG(4) << "handle LinkXPUOpMaxPass fuse";
     GET_IR_NODE(fusion_op);
-    GET_IR_NODE(out);
-    GET_IR_NODE(out_max);
-    for (auto next_op : out->outputs) {
-      auto* next_op_desc = next_op->Op();
-      if (op_types_.count(next_op_desc->Type()) == 0) continue;
-      next_op_desc->SetInput("x_max", {out_max->Name()});
-      IR_NODE_LINK_TO(out_max, next_op);
-      found_subgraph_count++;
+    GET_IR_NODE(input);
+    GET_IR_NODE(branch);
+
+    auto* fusion_op_desc = fusion_op->Op();
+    if (input->inputs[0]->Op()->HasOutput("out_max")) {
+      auto input_max_name = input->inputs[0]->Op()->Output("out_max");
+      for (auto max_node : input->inputs[0]->outputs) {
+        if (input_max_name[0] == max_node->Name()) {
+          fusion_op_desc->SetInput("x_max", {max_node->Name()});
+          IR_NODE_LINK_TO(max_node, fusion_op);
+          found_subgraph_count++;
+        }
+      }
+    }
+
+    if (with_branch) {
+      if (branch->inputs[0]->Op()->HasOutput("out_max")) {
+        auto branch_max_name = branch->inputs[0]->Op()->Output("out_max");
+        for (auto max_node : branch->inputs[0]->outputs) {
+          if (branch_max_name[0] == max_node->Name()) {
+            fusion_op_desc->SetInput("branch_max", {max_node->Name()});
+            IR_NODE_LINK_TO(max_node, fusion_op);
+            found_subgraph_count++;
+          }
+        }
+      }
     }
   };
 

From dda6b9d59c8b04ac92678bb98226f3b742c20258 Mon Sep 17 00:00:00 2001
From: ccrrong <101700995+ccrrong@users.noreply.github.com>
Date: Tue, 25 Apr 2023 16:47:19 +0800
Subject: [PATCH 060/405] update tile_grad composite rule (#53261)

---
 paddle/fluid/operators/tile_op.cc             | 19 +++++++++++++++----
 .../composite_backward_api.h                  | 17 +----------------
 paddle/phi/infermeta/unary.cc                 |  6 +++---
 3 files changed, 19 insertions(+), 23 deletions(-)

diff --git a/paddle/fluid/operators/tile_op.cc b/paddle/fluid/operators/tile_op.cc
index 2fcf702728501..26657ce42f303 100644
--- a/paddle/fluid/operators/tile_op.cc
+++ b/paddle/fluid/operators/tile_op.cc
@@ -171,14 +171,25 @@ class TileCompositeGradOpMaker : public prim::CompositeGradOpMakerBase {
     paddle::Tensor x = this->GetSingleForwardInput("X");
     paddle::Tensor out_grad = this->GetSingleOutputGrad("Out");
     paddle::Tensor x_grad = this->GetSingleInputGrad("X");
+    paddle::optional<paddle::Tensor> tensor_repeat_times =
+        this->GetOptionalSingleForwardInput("RepeatTimes");
+    paddle::optional<paddle::Tensor> tensor_repeat_times_attr =
+        this->GetOptionalSingleForwardInput("repeat_times_tensor");
 
     auto dx_ptr = this->GetOutputPtr(&x_grad);
     std::string dx_name = this->GetOutputName(x_grad);
     auto repeat_times = this->Attr<std::vector<int>>("repeat_times");
-    VLOG(6) << "Runing tile_grad composite func";
-    prim::tile_grad<prim::DescTensor>(
-        x, out_grad, paddle::experimental::IntArray(repeat_times), dx_ptr);
-    this->RecoverOutputName(x_grad, dx_name);
+    if (tensor_repeat_times.is_initialized() ||
+        tensor_repeat_times_attr.is_initialized()) {
+      PADDLE_THROW(platform::errors::Unimplemented(
+          "We don't support RepeatTimes from tensor or repeat_times_tensor for "
+          "tile composite grad for now. "));
+    } else {
+      VLOG(6) << "Runing tile_grad composite func";
+      prim::tile_grad<prim::DescTensor>(
+          x, out_grad, paddle::experimental::IntArray(repeat_times), dx_ptr);
+      this->RecoverOutputName(x_grad, dx_name);
+    }
   }
 };
 
diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
index a42c41c1ba229..5e1e490c1b73a 100644
--- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
+++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
@@ -1774,22 +1774,7 @@ void tile_grad(const Tensor& x,
   if (x_grad) {
     auto repeat_times_data = repeat_times.GetData();
     auto out_grad_shape = phi::vectorize<int>(out_grad.dims());
-    auto x_shape = phi::vectorize<int>(x.dims());
-
-    if (repeat_times_data.size() < x_shape.size()) {
-      int diff = x_shape.size() - repeat_times_data.size();
-      repeat_times_data.insert(repeat_times_data.begin(), diff, 1);
-    } else {
-      int diff = repeat_times_data.size() - x_shape.size();
-      x_shape.insert(x_shape.begin(), diff, 1);
-    }
-    for (int i = 0; i < static_cast<int>(out_grad_shape.size()); i++) {
-      if (out_grad_shape[i] == -1) {
-        out_grad_shape[i] = x_shape[i] * repeat_times_data[i];
-      }
-    }
-    auto result = reshape<T>(out_grad, out_grad_shape);
-
+    auto result = out_grad;
     for (int i = 0; i < static_cast<int>(repeat_times_data.size()); i++) {
       int size = out_grad_shape[i] / repeat_times_data[i];
       std::vector<int> sections(repeat_times_data[i], size);
diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc
index ea27eba513051..08b2c830d195d 100644
--- a/paddle/phi/infermeta/unary.cc
+++ b/paddle/phi/infermeta/unary.cc
@@ -4220,7 +4220,7 @@ void TileInferMeta(const MetaTensor& x,
   auto repeat_times_data = repeat_times.GetData();
   auto x_dims = x.dims();
   if (repeat_times_data.size() == 0) {
-    repeat_times_data = std::vector<int64_t>(x_dims.size(), -1);
+    repeat_times_data = std::vector<int64_t>(x_dims.size(), 1);
   }
 
   PADDLE_ENFORCE_LE(
@@ -4253,10 +4253,10 @@ void TileInferMeta(const MetaTensor& x,
   auto x_dim_vec = phi::vectorize<int>(x_dims);
   if (x_dim_vec.size() > repeat_times_data.size()) {
     auto diff = x_dim_vec.size() - repeat_times_data.size();
-    repeat_times_data.insert(repeat_times_data.begin(), diff, -1);
+    repeat_times_data.insert(repeat_times_data.begin(), diff, 1);
   } else {
     auto diff = repeat_times_data.size() - x_dim_vec.size();
-    x_dim_vec.insert(x_dim_vec.begin(), diff, -1);
+    x_dim_vec.insert(x_dim_vec.begin(), diff, 1);
   }
   for (size_t i = 0; i < repeat_times_data.size(); ++i) {
     if (x_dim_vec[i] == -1 || repeat_times_data[i] == -1) {

From b1668b0e3126c17c906a0bb3972d89a8124788c4 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Tue, 25 Apr 2023 16:59:49 +0800
Subject: [PATCH 061/405] Fix calling for tensor.data.

---
 paddle/fluid/operators/fused/fused_gate_attention.h | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 3579caf1bc99d..84e687e4b9ccf 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -1303,7 +1303,7 @@ class FlashAttnWithGating {
         num_splits,
         softmax_lse->data(),
         softmax_d.data(),
-        bias_d.data(),
+        nonbatched_bias ? bias_d.data() : nullptr,
         nullptr,
         &workspace_size,
         stream,
@@ -1316,7 +1316,8 @@ class FlashAttnWithGating {
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
+              << "Get workspace_size=" << workspace_size;
 
     phi::DenseTensor workspace = CreateWorkspace(workspace_size);
     succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
@@ -1345,8 +1346,8 @@ class FlashAttnWithGating {
         num_splits,
         softmax_lse->data(),
         softmax_d.data(),
-        bias_d.data(),
-        workspace.data(),
+        nonbatched_bias ? bias_d.data() : nullptr,
+        (workspace_size > 0) ? static_cast<void*>(workspace.data()) : nullptr,
         &workspace_size,
         stream,
         seed,

From 6c15247291fcc2d80f8a1f440797abb9b26aa1c1 Mon Sep 17 00:00:00 2001
From: Zenghui Yuan <56808883+huizyuan@users.noreply.github.com>
Date: Tue, 25 Apr 2023 17:26:36 +0800
Subject: [PATCH 062/405] Fix some problems in Paddle english instruction doc
 files. (#53145)

* Fix some problems in Paddle english instruction doc files.

* fix some new questions

* fix cn doc problems, test=document_fix
---
 python/paddle/fluid/framework.py | 9 ++++++---
 python/paddle/tensor/math.py     | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py
index 63ab3a65bb6b9..cab57eae045b3 100644
--- a/python/paddle/fluid/framework.py
+++ b/python/paddle/fluid/framework.py
@@ -718,7 +718,8 @@ def is_compiled_with_cinn():
     """
     Whether this whl package can be used to run the model on CINN.
 
-    Returns (bool): `True` if CINN is currently available, otherwise `False`.
+    Returns:
+        Bool: `True` if CINN is currently available, otherwise `False`.
 
     Examples:
         .. code-block:: python
@@ -733,7 +734,8 @@ def is_compiled_with_cuda():
     """
     Whether this whl package can be used to run the model on GPU.
 
-    Returns (bool): `True` if CUDA is currently available, otherwise `False`.
+    Returns:
+        Bool: `True` if CUDA is currently available, otherwise `False`.
 
     Examples:
         .. code-block:: python
@@ -748,7 +750,8 @@ def is_compiled_with_rocm():
     """
     Whether this whl package can be used to run the model on AMD or Hygon GPU(ROCm).
 
-    Returns (bool): `True` if ROCm is currently available, otherwise `False`.
+    Returns:
+        Bool: `True` if ROCm is currently available, otherwise `False`.
 
     Examples:
         .. code-block:: python
diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py
index 9f5212abf9147..ece8463f35e63 100644
--- a/python/paddle/tensor/math.py
+++ b/python/paddle/tensor/math.py
@@ -3057,7 +3057,7 @@ def __check_input(x, offset, axis1, axis2):
 
 def diagonal(x, offset=0, axis1=0, axis2=1, name=None):
     """
-    This OP computes the diagonals of the input tensor x.
+    Computes the diagonals of the input tensor x.
 
     If ``x`` is 2D, returns the diagonal.
     If ``x`` has larger dimensions, diagonals be taken from the 2D planes specified by axis1 and axis2.

From 3b9730314ccbaa6523ec0a7057205df95e2eb6a8 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Tue, 25 Apr 2023 21:53:26 +0800
Subject: [PATCH 063/405] Change all LOG(INFO) to VLOG and fix the backward.

---
 .../operators/fused/fused_gate_attention.h    | 264 ++++++------------
 .../fused/fused_gate_attention_op.cu          |   5 -
 2 files changed, 93 insertions(+), 176 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 84e687e4b9ccf..cf4fbdd3a1739 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -57,10 +57,14 @@ void AllocWithDebugInfo(const phi::GPUContext& dev_ctx,
 }
 
 inline std::string TensorDebugString(const phi::DenseTensor* t) {
-  if (t && t->initialized()) {
-    std::stringstream ss;
-    ss << "shape=[" << t->dims() << "], ptr=" << t->data();
-    return ss.str();
+  if (t) {
+    if (t->initialized()) {
+      std::stringstream ss;
+      ss << "shape=[" << t->dims() << "], ptr=" << t->data();
+      return ss.str();
+    } else {
+      return "not initialized";
+    }
   } else {
     return "nullptr";
   }
@@ -366,34 +370,6 @@ struct GateAttentionGradConfig : public GateAttentionConfig<T> {
   phi::DenseTensor qk_out_grad;
 };
 
-#define DEBUG_HERE printf("[%s, %d]: Run here!\n", __func__, __LINE__);
-#define DEBUG_DATA_INT(name, x)                                                \
-  do {                                                                         \
-    printf(                                                                    \
-        "[%s, %d]: %s = %d\n", __func__, __LINE__, name, static_cast<int>(x)); \
-  }                                                                            \
-  whilie(0);
-
-#define DEBUG_DATA_FlOAT(name, x)  \
-  do {                             \
-    printf("[%s, %d]: %s = %f\n",  \
-           __func__,               \
-           __LINE__,               \
-           std::string(name),      \
-           static_cast<float>(x)); \
-  }                                \
-  whilie(0);
-
-#define DEBUG_DIMS(x)                                    \
-  do {                                                   \
-    printf("[%s, %d]: dims is : [", __func__, __LINE__); \
-    for (int i = 0; i < x.size(); ++i) {                 \
-      printf("%d, ", x[i]);                              \
-    }                                                    \
-    printf(" ]\n");                                      \
-  }                                                      \
-  whilie(0);
-
 template <typename T>
 __global__ void FlashAttRange(int start, int step, int size, T* out1, T* out2) {
   CUDA_KERNEL_LOOP(index, size) {
@@ -402,30 +378,6 @@ __global__ void FlashAttRange(int start, int step, int size, T* out1, T* out2) {
   }
 }
 
-static void GetFlashAttnDimsString(const std::string& prefix,
-                                   const phi::DDim dim_val) {
-  // if (VLOG_IS_ON(4)) {
-  std::ostringstream out_string;
-  out_string << "FlashAttn - " << prefix << ".dims() is [";
-  for (int i = 0; i < dim_val.size(); ++i) {
-    out_string << dim_val[i] << ", ";
-  }
-  out_string << "]\n";
-  VLOG(4) << out_string.str();
-  std::cout << out_string.str();
-  // }
-}
-
-#define DBGPTR(ptr, prefix)                                              \
-  do {                                                                   \
-    std::ostringstream out_string;                                       \
-    void* data = static_cast<void*>(ptr);                                \
-    out_string << "[" << __func__ << ", " << __LINE__ << "]: " << prefix \
-               << "`s addr is ";                                         \
-    out_string << ptr << std::endl;                                      \
-    std::cout << out_string.str();                                       \
-  } while (0);
-
 template <typename T>
 class FMHAGateRef {
  public:
@@ -943,9 +895,6 @@ class FlashAttnWithGating {
 
   void ComputeForward(const phi::DenseTensor* nonbatched_bias,
                       const phi::DenseTensor* src_mask,
-                      phi::DenseTensor* q_transpose_out,
-                      phi::DenseTensor* k_transpose_out,
-                      phi::DenseTensor* v_transpose_out,
                       phi::DenseTensor* qkv_transpose_out,
                       phi::DenseTensor* softmax_lse,
                       phi::DenseTensor* fmha_out,
@@ -955,15 +904,13 @@ class FlashAttnWithGating {
         qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
 
     if (std::is_same<T, phi::dtype::float16>::value) {
-      LOG(INFO) << "T is phi::dtype::float16.";
+      VLOG(4) << "T is phi::dtype::float16.";
     } else if (std::is_same<T, phi::dtype::bfloat16>::value) {
-      LOG(INFO) << "T is phi::dtype::bfloat16.";
+      VLOG(4) << "T is phi::dtype::bfloat16.";
     } else if (std::is_same<T, float>::value) {
-      LOG(INFO) << "T is float.";
+      VLOG(4) << "T is float.";
     }
 
-    LOG(INFO) << "Use flash attention: merge_qkv=" << merge_qkv_;
-
     PADDLE_ENFORCE_NOT_NULL(
         qkv_transpose_out,
         platform::errors::NotFound("The input qkv_transpose_out can not be "
@@ -981,10 +928,9 @@ class FlashAttnWithGating {
          seq_batch_size * static_cast<int>(config->seq_len_r),
          static_cast<int>(config->num_heads),
          static_cast<int>(config->head_dim)});
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
-              << "1: Reshape qkv_transpose_out: ["
-              << config->qkv_transpose_out_dims << "] -> ["
-              << qkv_transpose_out->dims() << "]";
+    VLOG(5) << WaitWithDebugInfo(dev_ctx_) << "Reshape qkv_transpose_out: ["
+            << config->qkv_transpose_out_dims << "] -> ["
+            << qkv_transpose_out->dims() << "]";
 
     // q_size == k_size
     int64_t q_size = config->GetQuerySize();
@@ -1014,52 +960,53 @@ class FlashAttnWithGating {
                             dims_[dims_rank - 3],
                             dims_[dims_rank - 2],
                             dims_[dims_rank - 1]});
-        GetFlashAttnDimsString(prefix, temp_mask.dims());
+        VLOG(6) << prefix << ": " << TensorDebugString(dst_tensor);
       }
     };
-    auto& qkv_dims = qkv_transpose_out->dims();
-    dims_merge_func(src_mask, &temp_mask, "mask_dim");
-    dims_merge_func(nonbatched_bias, &temp_bias, "bias_dim");
-    GetFlashAttnDimsString("qkv_transpose_out", qkv_dims);
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
-              << "3: Merge dimensions for mask and bias";
+    dims_merge_func(src_mask, &temp_mask, "temp_mask");
+    dims_merge_func(nonbatched_bias, &temp_bias, "temp_bias");
 
     // 4. flash_attn parameter setting.
+    auto& qkv_dims = qkv_transpose_out->dims();
     int batch_size_ = seq_batch_size;
-    int total_q_ = qkv_dims[1];    // q.dims()[0]
-    int total_k_ = qkv_dims[1];    // q.dims()[0]
-    int num_heads_ = qkv_dims[2];  // q.dims()[1]
-    int head_size_ = qkv_dims[3];  // q.dims()[2]
-    int max_seqlen_q_ = batch_size_;
-    int max_seqlen_k_ = batch_size_;
+    int total_q_ = qkv_dims[1];             // q.dims()[0]
+    int total_k_ = qkv_dims[1];             // q.dims()[0]
+    int num_heads_ = qkv_dims[2];           // q.dims()[1]
+    int head_size_ = qkv_dims[3];           // q.dims()[2]
+    int max_seqlen_q_ = config->seq_len_r;  // batch_size_;
+    int max_seqlen_k_ = config->m_size;     // batch_size_;
     int num_splits = 0;  // 0 for an internal heuristic, which is optimal
-    LOG(INFO) << "[Flash_attn Fwd] batch_size : " << batch_size_;
-    LOG(INFO) << "[Flash_attn Fwd] total_q   : " << total_q_;
-    LOG(INFO) << "[Flash_attn Fwd] total_k   : " << total_k_;
-    LOG(INFO) << "[Flash_attn Fwd] num_heads : " << num_heads_;
-    LOG(INFO) << "[Flash_attn Fwd] head_size : " << head_size_;
-    LOG(INFO) << "[Flash_attn Fwd] max_seqlen_q : " << max_seqlen_q_;
-    LOG(INFO) << "[Flash_attn Fwd] max_seqlen_k : " << max_seqlen_k_;
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
-              << "4: Init flash-attention parameters";
+    VLOG(6) << "batch_size : " << batch_size_;
+    VLOG(6) << "total_q   : " << total_q_;
+    VLOG(6) << "total_k   : " << total_k_;
+    VLOG(6) << "num_heads : " << num_heads_;
+    VLOG(6) << "head_size : " << head_size_;
+    VLOG(6) << "max_seqlen_q : " << max_seqlen_q_;
+    VLOG(6) << "max_seqlen_k : " << max_seqlen_k_;
 
     // 5. construct softmax_lse
     int softmax_lse_last_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
     softmax_lse->Resize({batch_size_, num_heads_, softmax_lse_last_dim});
     AllocWithDebugInfo<float>(dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
-              << "5: Allocate softmax_lse: shape=[" << softmax_lse->dims()
-              << "]";
+    VLOG(5) << WaitWithDebugInfo(dev_ctx_) << "Allocate softmax_lse: shape=["
+            << softmax_lse->dims() << "]";
 
     // 6. construct random seed
     auto seed_offset_pair = GenerateSeedOffsetPair(batch_size_, num_heads_);
     uint64_t seed = seed_offset_pair.first;
     uint64_t offset = seed_offset_pair.second;
-
-    GetFlashAttnDimsString("softmax_lse", softmax_lse->dims());
-    GetFlashAttnDimsString("cu_seq_q", cu_seq_q.dims());
-    GetFlashAttnDimsString("cu_seq_k", cu_seq_k.dims());
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_) << "6: Construct random seed";
+    VLOG(5) << WaitWithDebugInfo(dev_ctx_) << "Construct random seed";
+
+    VLOG(6) << "cu_seq_q: " << TensorDebugString(&cu_seq_q);
+    VLOG(6) << "cu_seq_k: " << TensorDebugString(&cu_seq_k);
+    VLOG(6) << "temp_bias: " << TensorDebugString(&temp_bias);
+    VLOG(6) << "temp_mask: " << TensorDebugString(&temp_mask);
+    VLOG(6) << "nonbatched_bias: " << TensorDebugString(nonbatched_bias);
+    VLOG(6) << "src_mask: " << TensorDebugString(src_mask);
+    VLOG(6) << "qkv_transpose_out: " << TensorDebugString(qkv_transpose_out);
+    VLOG(6) << "softmax_lse: " << TensorDebugString(softmax_lse);
+    VLOG(6) << "fmha_out: " << TensorDebugString(fmha_out);
+    VLOG(6) << "gate_out: " << TensorDebugString(gate_out);
 
     // 7. flas_attn part one, get temp worksapce size.
     float p_dropout = 0.f;
@@ -1070,7 +1017,7 @@ class FlashAttnWithGating {
         static_cast<const void*>(q_ptr),
         static_cast<const void*>(k_ptr),
         static_cast<const void*>(v_ptr),
-        nullptr,  // for calculation workspace size
+        nullptr,  // set out to nullptr to calculate workspace size
         cu_seq_q.data<int32_t>(),
         cu_seq_k.data<int32_t>(),
         total_q_,
@@ -1100,17 +1047,11 @@ class FlashAttnWithGating {
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
-              << "7: Get workspace_size=" << workspace_size;
+    VLOG(5) << WaitWithDebugInfo(dev_ctx_)
+            << "Get workspace_size=" << workspace_size;
 
+    // 8. Run flash-attention kernel.
     phi::DenseTensor workspace = CreateWorkspace(workspace_size);
-
-    LOG(INFO) << "qkv_transpose_out: " << TensorDebugString(qkv_transpose_out);
-    LOG(INFO) << "src_mask: " << TensorDebugString(src_mask);
-    LOG(INFO) << "fmha_out: " << TensorDebugString(fmha_out);
-    LOG(INFO) << "gate_out: " << TensorDebugString(gate_out);
-
-    // 8. flas_attn part two, run impl.
     succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
         static_cast<const void*>(q_ptr),
         static_cast<const void*>(k_ptr),
@@ -1145,7 +1086,8 @@ class FlashAttnWithGating {
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_) << "8: Run SUCCESS";
+    VLOG(5) << "[ComputeForward]" << WaitWithDebugInfo(dev_ctx_)
+            << "Run SUCCESS!!!";
 
     if (config->has_gating) {
       gate_out->Resize(config->gate_out_dims);
@@ -1165,11 +1107,11 @@ class FlashAttnWithGating {
         qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
 
     if (std::is_same<T, phi::dtype::float16>::value) {
-      std::cout << "[Grad]: T is phi::dtype::float16. \n";
+      VLOG(4) << "[Grad]: T is phi::dtype::float16.";
     } else if (std::is_same<T, phi::dtype::bfloat16>::value) {
-      std::cout << "[Grad]: T is phi::dtype::bfloat16. \n";
+      VLOG(4) << "[Grad]: T is phi::dtype::bfloat16.";
     } else if (std::is_same<T, float>::value) {
-      std::cout << "[Grad]: T is float. \n";
+      VLOG(4) << "[Grad]: T is float.";
     }
 
     PADDLE_ENFORCE_NOT_NULL(
@@ -1193,7 +1135,7 @@ class FlashAttnWithGating {
 
     int seq_batch_size = static_cast<int>(config->batch_size) *
                          static_cast<int>(config->seq_len_m);
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+    VLOG(5) << WaitWithDebugInfo(dev_ctx_);
 
     // 2. Init with cu_seq_q and cu_seq_k for flash_attn.
     phi::DenseTensor cu_seq_q, cu_seq_k;
@@ -1217,59 +1159,55 @@ class FlashAttnWithGating {
                             dims_[dims_rank - 3],
                             dims_[dims_rank - 2],
                             dims_[dims_rank - 1]});
-        GetFlashAttnDimsString(prefix, temp_mask.dims());
+        VLOG(6) << prefix << ": " << TensorDebugString(dst_tensor);
       }
     };
-    dims_merge_func(src_mask, &temp_mask, "[Grad] mask_dim");
-    dims_merge_func(nonbatched_bias, &temp_bias, "[Grad] bias_dim");
+    dims_merge_func(src_mask, &temp_mask, "temp_mask");
+    dims_merge_func(nonbatched_bias, &temp_bias, "temp_bias");
 
     phi::DDim qkv_dims({3,
                         seq_batch_size * static_cast<int>(config->seq_len_r),
                         static_cast<int>(config->num_heads),
                         static_cast<int>(config->head_dim)});
     int batch_size_ = seq_batch_size;
-    int total_q_ = qkv_dims[1];    // q.dims()[0]
-    int total_k_ = qkv_dims[1];    // q.dims()[0]
-    int num_heads_ = qkv_dims[2];  // q.dims()[1]
-    int head_size_ = qkv_dims[3];  // q.dims()[2]
-    int max_seqlen_q_ = batch_size_;
-    int max_seqlen_k_ = batch_size_;
-    VLOG(6) << "[Flash_attn Grad] batch_size : " << batch_size_;
-    VLOG(6) << "[Flash_attn Grad] total_q   : " << total_q_;
-    VLOG(6) << "[Flash_attn Grad] total_k   : " << total_k_;
-    VLOG(6) << "[Flash_attn Grad] num_heads : " << num_heads_;
-    VLOG(6) << "[Flash_attn Grad] head_size : " << head_size_;
-    VLOG(6) << "[Flash_attn Grad] max_seqlen_q : " << max_seqlen_q_;
-    VLOG(6) << "[Flash_attn Grad] max_seqlen_k : " << max_seqlen_k_;
+    int total_q_ = qkv_dims[1];             // q.dims()[0]
+    int total_k_ = qkv_dims[1];             // q.dims()[0]
+    int num_heads_ = qkv_dims[2];           // q.dims()[1]
+    int head_size_ = qkv_dims[3];           // q.dims()[2]
+    int max_seqlen_q_ = config->seq_len_r;  // batch_size_;
+    int max_seqlen_k_ = config->m_size;     // batch_size_;
+    VLOG(6) << "batch_size : " << batch_size_;
+    VLOG(6) << "total_q   : " << total_q_;
+    VLOG(6) << "total_k   : " << total_k_;
+    VLOG(6) << "num_heads : " << num_heads_;
+    VLOG(6) << "head_size : " << head_size_;
+    VLOG(6) << "max_seqlen_q : " << max_seqlen_q_;
+    VLOG(6) << "max_seqlen_k : " << max_seqlen_k_;
 
     // 5. construct softmax_lse
-    int last_q_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
-    // softmax_lse->Resize({batch_size_, num_heads_, last_q_dim});
-    // AllocWithDebugInfo<float>(
-    //     dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
-
-    phi::DenseTensor softmax_d = phi::Empty<float, phi::GPUContext>(
-        dev_ctx_, {batch_size_, num_heads_, last_q_dim});
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+    phi::DenseTensor softmax_d;
+    softmax_d.Resize(softmax_lse->dims());
+    AllocWithDebugInfo<float>(dev_ctx_, "d_softmax_lse", &softmax_d);
+    VLOG(5) << WaitWithDebugInfo(dev_ctx_);
 
     phi::DenseTensor bias_d;
     if (nonbatched_bias) {
       bias_d = phi::Empty<T, phi::GPUContext>(
           dev_ctx_, {batch_size_, num_heads_, max_seqlen_q_, max_seqlen_k_});
     }
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+    VLOG(5) << WaitWithDebugInfo(dev_ctx_);
 
     // 6. construct random seed
     auto seed_offset_pair = GenerateSeedOffsetPair(batch_size_, num_heads_);
     uint64_t seed = seed_offset_pair.first;
     uint64_t offset = seed_offset_pair.second;
 
-    LOG(INFO) << "fmha_out: " << TensorDebugString(fmha_out);
-    LOG(INFO) << "fmha_out_grad: " << TensorDebugString(fmha_out_grad);
-    LOG(INFO) << "softmax_lse: " << TensorDebugString(softmax_lse);
-    LOG(INFO) << "softmax_d: " << TensorDebugString(&softmax_d);
-    LOG(INFO) << "bias_d: " << TensorDebugString(&bias_d);
+    VLOG(6) << "fmha_out: " << TensorDebugString(fmha_out);
+    VLOG(6) << "fmha_out_grad: " << TensorDebugString(fmha_out_grad);
+    VLOG(6) << "softmax_lse: " << TensorDebugString(softmax_lse);
+    VLOG(6) << "softmax_d: " << TensorDebugString(&softmax_d);
+    VLOG(6) << "nonbatched_bias: " << TensorDebugString(nonbatched_bias);
+    VLOG(6) << "bias_d: " << TensorDebugString(&bias_d);
 
     // 7. flas_attn part one, get temp worksapce size.
     uint64_t workspace_size;
@@ -1284,7 +1222,7 @@ class FlashAttnWithGating {
         static_cast<void*>(q_grad_ptr),
         static_cast<void*>(k_grad_ptr),
         static_cast<void*>(v_grad_ptr),
-        static_cast<const void*>(fmha_out->data()),
+        nullptr,  // set out to nullptr to calculate workspace size
         static_cast<const void*>(fmha_out_grad->data()),
         cu_seq_q.data<int32_t>(),
         cu_seq_k.data<int32_t>(),
@@ -1303,7 +1241,7 @@ class FlashAttnWithGating {
         num_splits,
         softmax_lse->data(),
         softmax_d.data(),
-        nonbatched_bias ? bias_d.data() : nullptr,
+        nonbatched_bias_grad ? bias_d.data() : nullptr,
         nullptr,
         &workspace_size,
         stream,
@@ -1316,8 +1254,8 @@ class FlashAttnWithGating {
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
-              << "Get workspace_size=" << workspace_size;
+    VLOG(5) << WaitWithDebugInfo(dev_ctx_)
+            << "Get workspace_size=" << workspace_size;
 
     phi::DenseTensor workspace = CreateWorkspace(workspace_size);
     succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
@@ -1359,7 +1297,7 @@ class FlashAttnWithGating {
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_);
+    VLOG(5) << WaitWithDebugInfo(dev_ctx_);
 
     if (nonbatched_bias) {
       // compare block reduce
@@ -1407,9 +1345,9 @@ class FlashAttnWithGating {
     FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
         start, step, end, cu_seq_q->data<int32_t>(), cu_seq_k->data<int32_t>());
 
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
-              << "AllocAndInit cu_seq_q and cu_seq_k: start=" << start
-              << ", step=" << step << ", end=" << end;
+    VLOG(5) << WaitWithDebugInfo(dev_ctx_)
+            << "AllocAndInit cu_seq_q and cu_seq_k: start=" << start
+            << ", step=" << step << ", end=" << end;
   }
 
   phi::DenseTensor CreateWorkspace(uint64_t workspace_size) {
@@ -1417,10 +1355,9 @@ class FlashAttnWithGating {
     if (workspace_size > 0) {
       workspace = phi::Empty<float, phi::GPUContext>(
           dev_ctx_, {int64_t(workspace_size / sizeof(float))});
-      DBGPTR(workspace.data(), "workspace");
     }
-    LOG(INFO) << WaitWithDebugInfo(dev_ctx_)
-              << "Allocate workspace: workspace_size=" << workspace_size;
+    VLOG(5) << WaitWithDebugInfo(dev_ctx_)
+            << "Allocate workspace: workspace_size=" << workspace_size;
     return workspace;
   }
 
@@ -1440,25 +1377,10 @@ class FlashAttnWithGating {
         dev_ctx_, qkv_out, perm, qkv_transpose_out);
   }
 
-  void ComputeQKVTransposeBackward(const phi::DenseTensor& q_transpose_out_grad,
-                                   const phi::DenseTensor& k_transpose_out_grad,
-                                   const phi::DenseTensor& v_transpose_out_grad,
-                                   phi::DenseTensor* q_out_grad,
-                                   phi::DenseTensor* k_out_grad,
-                                   phi::DenseTensor* v_out_grad) {
-    std::vector<int> perm = {0, 1, 3, 2, 4};
-    phi::funcs::TransposeGPUKernelDriver<T>(
-        dev_ctx_, q_transpose_out_grad, perm, q_out_grad);
-    phi::funcs::TransposeGPUKernelDriver<T>(
-        dev_ctx_, k_transpose_out_grad, perm, k_out_grad);
-    phi::funcs::TransposeGPUKernelDriver<T>(
-        dev_ctx_, v_transpose_out_grad, perm, v_out_grad);
-  }
-
   void ComputeQKVTransposeBackward(
       const phi::DenseTensor& qkv_transpose_out_grad,
       phi::DenseTensor* qkv_out_grad) {
-    std::vector<int> perm = {1, 2, 4, 0, 3, 5};
+    std::vector<int> perm = {1, 2, 3, 0, 4, 5};
     phi::funcs::TransposeGPUKernelDriver<T>(
         dev_ctx_, qkv_transpose_out_grad, perm, qkv_out_grad);
   }
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cu b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
index 261e2e377b5f8..03ccc1282fa51 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cu
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
@@ -432,9 +432,6 @@ class FusedGateAttentionOpKernel : public framework::OpKernel<T> {
       auto fmha_compute = FlashAttnWithGating<T>(dev_ctx, merge_qkv);
       fmha_compute.ComputeForward(nonbatched_bias,
                                   src_mask,
-                                  q_transpose_out,
-                                  k_transpose_out,
-                                  v_transpose_out,
                                   qkv_transpose_out,
                                   softmax_lse,
                                   fmha_out,
@@ -474,8 +471,6 @@ template <typename T, typename DeviceContext>
 class FusedGateAttentionGradKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext &ctx) const override {
-    LOG(INFO) << "In FusedGateAttentionGradKernel";
-
     // forward input
     const auto *query = ctx.Input<phi::DenseTensor>("Query");
     const auto *key = ctx.Input<phi::DenseTensor>("Key");

From af986bd521f9e35b5cfe6307c41c785638532d74 Mon Sep 17 00:00:00 2001
From: lzydev <lizhiyu02@baidu.com>
Date: Tue, 25 Apr 2023 23:21:54 +0800
Subject: [PATCH 064/405] Add singlely compile gpu kernel camke function
 (#53305)

* support register single .cu file

* add register GPU kernel function
---
 cmake/operators.cmake                         |  52 +++++++
 paddle/fluid/operators/CMakeLists.txt         |   4 +
 .../fluid/operators/class_center_sample_op.cc | 131 ------------------
 paddle/phi/api/yaml/legacy_ops.yaml           |   8 --
 paddle/phi/api/yaml/op_compat.yaml            |   6 +
 paddle/phi/api/yaml/ops.yaml                  |   9 ++
 .../phi/ops/compat/class_center_sample_sig.cc |  36 -----
 7 files changed, 71 insertions(+), 175 deletions(-)
 delete mode 100644 paddle/fluid/operators/class_center_sample_op.cc
 delete mode 100644 paddle/phi/ops/compat/class_center_sample_sig.cc

diff --git a/cmake/operators.cmake b/cmake/operators.cmake
index e22a747688b76..826d0e773a8a5 100644
--- a/cmake/operators.cmake
+++ b/cmake/operators.cmake
@@ -51,6 +51,58 @@ function(find_phi_register FILENAME ADD_PATH PATTERN)
   endif()
 endfunction()
 
+# Just for those gpu kernels locating at "fluid/operators/", such as 'class_center_sample_op.cu'.
+# Add other file modes if need in the future.
+function(register_cu_kernel TARGET)
+  set(options "")
+  set(oneValueArgs "")
+  set(multiValueArgs SRCS DEPS)
+  cmake_parse_arguments(register_cu_kernel "${options}" "${oneValueArgs}"
+                        "${multiValueArgs}" ${ARGN})
+
+  set(cu_srcs)
+  set(op_common_deps operator op_registry math_function layer
+                     common_infer_shape_functions)
+  foreach(cu_src ${register_cu_kernel_SRCS})
+    if(${cu_src} MATCHES ".*\\.cu$")
+      list(APPEND cu_srcs ${cu_src})
+    endif()
+  endforeach()
+  list(LENGTH cu_srcs cu_srcs_len)
+  if(${cu_srcs_len} EQUAL 0)
+    message(
+      FATAL_ERROR
+        "The GPU kernel file of ${TARGET} should contains at least one .cu file"
+    )
+  endif()
+  if(WITH_GPU)
+    nv_library(
+      ${TARGET}
+      SRCS ${cu_srcs}
+      DEPS ${op_library_DEPS} ${op_common_deps})
+  elseif(WITH_ROCM)
+    hip_library(
+      ${TARGET}
+      SRCS ${cu_srcs}
+      DEPS ${op_library_DEPS} ${op_common_deps})
+  endif()
+  set(OP_LIBRARY
+      ${TARGET} ${OP_LIBRARY}
+      CACHE INTERNAL "op libs")
+  foreach(cu_src ${cu_srcs})
+    set(op_name "")
+    # Add PHI Kernel Registry Message
+    find_phi_register(${cu_src} ${pybind_file} "PD_REGISTER_KERNEL")
+    find_phi_register(${cu_src} ${pybind_file} "PD_REGISTER_STRUCT_KERNEL")
+    find_phi_register(${cu_src} ${pybind_file}
+                      "PD_REGISTER_KERNEL_FOR_ALL_DTYPE")
+    find_register(${cu_src} "REGISTER_OP_CUDA_KERNEL" op_name)
+    if(NOT ${op_name} EQUAL "")
+      file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${op_name}, CUDA);\n")
+    endif()
+  endforeach()
+endfunction()
+
 function(op_library TARGET)
   # op_library is a function to create op library. The interface is same as
   # cc_library. But it handle split GPU/CPU code and link some common library
diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt
index cb9686611dfd5..e4add9ae4bd3c 100644
--- a/paddle/fluid/operators/CMakeLists.txt
+++ b/paddle/fluid/operators/CMakeLists.txt
@@ -102,6 +102,10 @@ op_library(quantize_linear_op DEPS phi)
 op_library(save_combine_op DEPS string_array phi)
 op_library(load_combine_op DEPS string_array)
 
+if (WITH_GPU OR WITH_ROCM)
+    register_cu_kernel(class_center_sample_op SRCS class_center_sample_op.cu DEPS ${OP_HEADER_DEPS})
+endif()
+
 if (WITH_GPU OR WITH_ROCM)
     op_library(activation_op SRCS activation_op.cc activation_op.kps soft_relu_op.cu DEPS ${OP_HEADER_DEPS})
 elseif (WITH_XPU_KP)
diff --git a/paddle/fluid/operators/class_center_sample_op.cc b/paddle/fluid/operators/class_center_sample_op.cc
deleted file mode 100644
index 54f0e981ca078..0000000000000
--- a/paddle/fluid/operators/class_center_sample_op.cc
+++ /dev/null
@@ -1,131 +0,0 @@
-//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/framework/op_version_registry.h"
-#include "paddle/phi/core/infermeta_utils.h"
-#include "paddle/phi/infermeta/unary.h"
-
-namespace paddle {
-namespace operators {
-
-class ClassCenterSampleOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(ctx, "Label"),
-                          ctx.device_context().GetPlace());
-  }
-};
-
-class ClassCenterSampleOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput(
-        "Label",
-        "(Tensor<int|int64>) The input of ClassCenterSample op. Each value "
-        "of Label is an integer label.");
-    AddOutput("RemappedLabel",
-              "(Tensor<int|int64>) Output tensor with same shape as Label. "
-              "Each label is remap using sampled class.");
-    AddOutput("SampledLocalClassCenter",
-              "(Tensor<int|int64>) The sampled class center for local rank,"
-              "value in [0, num_classes).");
-    AddAttr<int>(
-        "num_classes",
-        "A positive integer to specify the number of classes at local rank. "
-        "Note that num_classes of each GPU can be different.");
-    AddAttr<int>(
-        "num_samples",
-        "A positive integer to specify the number of class center to sample.");
-    AddAttr<int>("ring_id", "(int default 0) nccl communication ring id.")
-        .SetDefault(0);
-    AddAttr<int>("nranks", "(int default 1) The total number of GPUs.")
-        .SetDefault(1);
-    AddAttr<int>("rank", "(int default 0) The rank id in nranks.")
-        .SetDefault(0);
-    AddAttr<bool>("fix_seed",
-                  "A flag indicating whether to use a fixed seed to generate "
-                  "random negative class center. NOTE: DO NOT set this flag to"
-                  "true in training. Setting this flag to true is only useful "
-                  "in unittest or for debug")
-        .SetDefault(false);
-    AddAttr<int>("seed",
-                 "Random seed used to generate random negative class center. "
-                 "[default 0].")
-        .SetDefault(0);
-    AddComment(R"DOC(
-    Class center sample method is proposed from the paper PartialFC that only sample a subset of the class centers.
-    The process of sampling subset class centers is straightforward: 1) First select the positive class centers;
-    2) Randomly sample negative class centers. Specifically, given a Label tensor, shape [batch_size], select all
-    the positive class centers and randomly sample negative class centers, then remap the input label tensor using
-    the sampled class centers. Note that if the number of the positive class centers is greater than the input
-    num_samples, it keeps all the positive class centers and the shape of SampledLocalClassCenter will be
-    [num_positive_class_centers]. The op supports CPU, single GPU and multi GPU.
-
-    For more information, Partial FC: Training 10 Million Identities on a Single Machine
-    arxiv: https://arxiv.org/abs/2010.05222
-
-    Examples:
-      For CPU or only one GPU
-      Given:
-        Label: [11, 5 , 1 , 3 , 12, 2 , 15, 19, 18, 19]
-        num_classes = 20
-        num_samples = 6
-      Then:
-        RemappedLabel: [4, 3, 0, 2, 5, 1, 6, 8, 7, 8]
-        SampledLocalClassCenter: [1 , 2 , 3 , 5 , 11, 12, 15, 18, 19]
-
-      For multi GPU
-      Given:
-        rank0:
-            Label: [10, 17, 15, 11, 9 , 12, 18, 18, 17, 18, 19, 2 , 8 , 13, 11, 13, 9 , 10, 0 , 4 ]
-            num_classes = 10
-            num_samples = 6
-            ring_id = 0
-            nranks = 2
-            rank = 0
-        rank1:
-            Label: [10, 17, 15, 11, 9 , 12, 18, 18, 17, 18, 19, 2 , 8 , 13, 11, 13, 9 , 10, 0 , 4 ]
-            num_classes = 10
-            num_samples = 6
-            ring_id = 0
-            nranks = 2
-            rank = 1
-      Then:
-        rank0:
-            RemappedLabel: [6 , 11, 10, 7 , 4 , 8 , 12, 12, 11, 12, 13, 1 , 3 , 9 , 7 , 9 , 4 , 6 , 0 , 2 ]
-            SampledLocalClassCenter: [0, 2, 4, 8, 9, 3]
-        rank1:
-            RemappedLabel: [6 , 11, 10, 7 , 4 , 8 , 12, 12, 11, 12, 13, 1 , 3 , 9 , 7 , 9 , 4 , 6 , 0 , 2 ]
-            SampledLocalClassCenter: [0, 1, 2, 3, 5, 7, 8]
-)DOC");
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-DECLARE_INFER_SHAPE_FUNCTOR(class_center_sample,
-                            ClassCenterSampleInferShapeFunctor,
-                            PD_INFER_META(phi::ClassCenterSampleInferMeta));
-REGISTER_OP_WITHOUT_GRADIENT(class_center_sample,
-                             ops::ClassCenterSampleOp,
-                             ops::ClassCenterSampleOpMaker,
-                             ClassCenterSampleInferShapeFunctor);
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index d23a20b18fcc4..61a8cb76e5c2e 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -180,14 +180,6 @@
     data_type : x
   inplace : (x -> out), (input_found_infinite -> output_found_infinite)
 
-- op : class_center_sample
-  args : (Tensor label, int num_classes, int num_samples, int ring_id, int rank, int nranks, bool fix_seed, int seed)
-  output : Tensor(remapped_label), Tensor(sampled_local_class_center)
-  infer_meta :
-    func : ClassCenterSampleInferMeta
-  kernel :
-    func : class_center_sample
-
 - op : coalesce_tensor
   args : (Tensor[] input, DataType dtype, bool copy_data = false, bool set_constant = false, bool persist_output = false, float constant = 0.0, bool use_align = true, int align_size = -1, int size_of_dtype = -1, int64_t[] concated_shapes = {}, int64_t[] concated_ranks = {})
   output : Tensor[](output){input.size()}, Tensor(fused_output)
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index caf3194b899c2..5585d0c2d3b97 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -379,6 +379,12 @@
   outputs :
     out : Out
 
+- op : class_center_sample
+  inputs :
+    label : Label
+  outputs :
+    {remapped_label : RemappedLabel, sampled_local_class_center : SampledLocalClassCenter}
+
 - op : clip
   backward : clip_grad, clip_double_grad
   inputs :
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index 1541d1890a07a..c228a0506e813 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -377,6 +377,15 @@
     func : cholesky_solve
   backward : cholesky_solve_grad
 
+- op : class_center_sample
+  args : (Tensor label, int num_classes, int num_samples, int ring_id = 0, int rank = 0, int nranks = 1, bool fix_seed = false, int seed = 0)
+  output : Tensor(remapped_label), Tensor(sampled_local_class_center)
+  infer_meta :
+    func : ClassCenterSampleInferMeta
+  kernel :
+    func : class_center_sample
+    data_type : label
+
 - op : clip
   args : (Tensor x, Scalar(float) min, Scalar(float) max)
   output : Tensor(out)
diff --git a/paddle/phi/ops/compat/class_center_sample_sig.cc b/paddle/phi/ops/compat/class_center_sample_sig.cc
deleted file mode 100644
index cfaf2b86436db..0000000000000
--- a/paddle/phi/ops/compat/class_center_sample_sig.cc
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/core/compat/op_utils.h"
-
-namespace phi {
-
-KernelSignature ClassCenterSampleOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature("class_center_sample",
-                         {"Label"},
-                         {"num_classes",
-                          "num_samples",
-                          "ring_id",
-                          "rank",
-                          "nranks",
-                          "fix_seed",
-                          "seed"},
-                         {"RemappedLabel", "SampledLocalClassCenter"});
-}
-
-}  // namespace phi
-
-PD_REGISTER_ARG_MAPPING_FN(class_center_sample,
-                           phi::ClassCenterSampleOpArgumentMapping);

From 37489df57f42f882c8fbc8bbe2f9edbf6ef3298c Mon Sep 17 00:00:00 2001
From: huangjiyi <43315610+huangjiyi@users.noreply.github.com>
Date: Wed, 26 Apr 2023 10:40:07 +0800
Subject: [PATCH 065/405] Register fluid xpu kerenls to phi [part 3] (#53189)

* update

* update
---
 .../fluid/operators/affine_channel_op_xpu.cc  | 16 ++++++++------
 .../fused/resnet_basic_block_op_xpu.cc        | 18 ++++++++++-----
 .../operators/fused/resnet_unit_op_xpu.cc     | 22 ++++++++++++-------
 paddle/fluid/operators/sampling_id_op_xpu.cc  |  6 ++---
 .../sequence_ops/sequence_conv_op_xpu.cc      | 18 +++++++--------
 .../sequence_ops/sequence_unpad_op_xpu.cc     |  4 ++--
 .../uniform_random_inplace_op_xpu.cc          | 19 ++++++++++------
 7 files changed, 60 insertions(+), 43 deletions(-)

diff --git a/paddle/fluid/operators/affine_channel_op_xpu.cc b/paddle/fluid/operators/affine_channel_op_xpu.cc
index 2649a9190b86f..7a4de54954d8b 100644
--- a/paddle/fluid/operators/affine_channel_op_xpu.cc
+++ b/paddle/fluid/operators/affine_channel_op_xpu.cc
@@ -25,7 +25,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename DeviceContext, typename T>
+template <typename T, typename DeviceContext>
 class AffineChannelXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
@@ -85,7 +85,7 @@ class AffineChannelXPUKernel : public framework::OpKernel<T> {
   }
 };
 
-template <typename DeviceContext, typename T>
+template <typename T, typename DeviceContext>
 class AffineChannelGradXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
@@ -189,10 +189,12 @@ class AffineChannelGradXPUKernel : public framework::OpKernel<T> {
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-using XPU = paddle::platform::XPUDeviceContext;
-
-REGISTER_OP_XPU_KERNEL(affine_channel, ops::AffineChannelXPUKernel<XPU, float>);
-REGISTER_OP_XPU_KERNEL(affine_channel_grad,
-                       ops::AffineChannelGradXPUKernel<XPU, float>);
 
+PD_REGISTER_STRUCT_KERNEL(
+    affine_channel, XPU, ALL_LAYOUT, ops::AffineChannelXPUKernel, float) {}
+PD_REGISTER_STRUCT_KERNEL(affine_channel_grad,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::AffineChannelGradXPUKernel,
+                          float) {}
 #endif
diff --git a/paddle/fluid/operators/fused/resnet_basic_block_op_xpu.cc b/paddle/fluid/operators/fused/resnet_basic_block_op_xpu.cc
index f6b2d30453f42..4d026f4b780a1 100644
--- a/paddle/fluid/operators/fused/resnet_basic_block_op_xpu.cc
+++ b/paddle/fluid/operators/fused/resnet_basic_block_op_xpu.cc
@@ -293,7 +293,7 @@ static inline void xpu_conv2d_grad(xpu::Context* ctx,
   PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d_grad");
 }
 
-template <typename T>
+template <typename T, typename DeviceContext>
 class ResNetBasicBlockXPUKernel : public framework::OpKernel<T> {
  public:
   using XPUT = typename XPUTypeTrait<T>::Type;
@@ -696,7 +696,7 @@ class ResNetBasicBlockXPUKernel : public framework::OpKernel<T> {
   }
 };
 
-template <typename T>
+template <typename T, typename DeviceContext>
 class ResNetBasicBlockGradXPUKernel : public framework::OpKernel<T> {
  public:
   using XPUT = typename XPUTypeTrait<T>::Type;
@@ -992,8 +992,14 @@ class ResNetBasicBlockGradXPUKernel : public framework::OpKernel<T> {
 
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
-REGISTER_OP_XPU_KERNEL(resnet_basic_block,
-                       ops::ResNetBasicBlockXPUKernel<float>);
-REGISTER_OP_XPU_KERNEL(resnet_basic_block_grad,
-                       ops::ResNetBasicBlockGradXPUKernel<float>);
+PD_REGISTER_STRUCT_KERNEL(resnet_basic_block,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::ResNetBasicBlockXPUKernel,
+                          float) {}
+PD_REGISTER_STRUCT_KERNEL(resnet_basic_block_grad,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::ResNetBasicBlockGradXPUKernel,
+                          float) {}
 #endif
diff --git a/paddle/fluid/operators/fused/resnet_unit_op_xpu.cc b/paddle/fluid/operators/fused/resnet_unit_op_xpu.cc
index 1e2741cde5d9e..1e4ed290f43a9 100644
--- a/paddle/fluid/operators/fused/resnet_unit_op_xpu.cc
+++ b/paddle/fluid/operators/fused/resnet_unit_op_xpu.cc
@@ -19,7 +19,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename T>
+template <typename T, typename DeviceContext>
 class ResNetUnitXPUKernel : public framework::OpKernel<T> {
   using XPUType = typename XPUTypeTrait<T>::Type;
 
@@ -181,7 +181,7 @@ class ResNetUnitXPUKernel : public framework::OpKernel<T> {
   }
 };
 
-template <typename T>
+template <typename T, typename DeviceContext>
 class ResNetUnitGradXPUKernel : public framework::OpKernel<T> {
   using XPUType = typename XPUTypeTrait<T>::Type;
 
@@ -361,9 +361,15 @@ class ResNetUnitGradXPUKernel : public framework::OpKernel<T> {
 
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
-REGISTER_OP_XPU_KERNEL(resnet_unit,
-                       ops::ResNetUnitXPUKernel<plat::float16>,
-                       ops::ResNetUnitXPUKernel<float>);
-REGISTER_OP_XPU_KERNEL(resnet_unit_grad,
-                       ops::ResNetUnitGradXPUKernel<plat::float16>,
-                       ops::ResNetUnitGradXPUKernel<float>);
+PD_REGISTER_STRUCT_KERNEL(resnet_unit,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::ResNetUnitXPUKernel,
+                          plat::float16,
+                          float) {}
+PD_REGISTER_STRUCT_KERNEL(resnet_unit_grad,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::ResNetUnitGradXPUKernel,
+                          plat::float16,
+                          float) {}
diff --git a/paddle/fluid/operators/sampling_id_op_xpu.cc b/paddle/fluid/operators/sampling_id_op_xpu.cc
index 0b720c21381ac..9fd0193733e6e 100644
--- a/paddle/fluid/operators/sampling_id_op_xpu.cc
+++ b/paddle/fluid/operators/sampling_id_op_xpu.cc
@@ -16,8 +16,6 @@
 #include "paddle/fluid/platform/device_context.h"
 
 namespace ops = paddle::operators;
-using XPUCtx = paddle::platform::XPUDeviceContext;
 
-REGISTER_OP_XPU_KERNEL(sampling_id,
-                       paddle::operators::SamplingIdKernel<float, XPUCtx>,
-                       paddle::operators::SamplingIdKernel<double, XPUCtx>);
+PD_REGISTER_STRUCT_KERNEL(
+    sampling_id, XPU, ALL_LAYOUT, ops::SamplingIdKernel, float, double) {}
diff --git a/paddle/fluid/operators/sequence_ops/sequence_conv_op_xpu.cc b/paddle/fluid/operators/sequence_ops/sequence_conv_op_xpu.cc
index f7b0b5c3b581a..53fb13180c36a 100644
--- a/paddle/fluid/operators/sequence_ops/sequence_conv_op_xpu.cc
+++ b/paddle/fluid/operators/sequence_ops/sequence_conv_op_xpu.cc
@@ -20,7 +20,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename DeviceContext, typename T>
+template <typename T, typename DeviceContext>
 class SequenceConvXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& context) const override {
@@ -154,7 +154,7 @@ class SequenceConvXPUKernel : public framework::OpKernel<T> {
   }
 };
 
-template <typename DeviceContext, typename T>
+template <typename T, typename DeviceContext>
 class SequenceConvGradXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& context) const override {
@@ -363,12 +363,12 @@ class SequenceConvGradXPUKernel : public framework::OpKernel<T> {
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OP_XPU_KERNEL(
-    sequence_conv,
-    ops::SequenceConvXPUKernel<paddle::platform::XPUDeviceContext, float>);
-
-REGISTER_OP_XPU_KERNEL(
-    sequence_conv_grad,
-    ops::SequenceConvGradXPUKernel<paddle::platform::XPUDeviceContext, float>);
+PD_REGISTER_STRUCT_KERNEL(
+    sequence_conv, XPU, ALL_LAYOUT, ops::SequenceConvXPUKernel, float) {}
+PD_REGISTER_STRUCT_KERNEL(sequence_conv_grad,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::SequenceConvGradXPUKernel,
+                          float) {}
 
 #endif
diff --git a/paddle/fluid/operators/sequence_ops/sequence_unpad_op_xpu.cc b/paddle/fluid/operators/sequence_ops/sequence_unpad_op_xpu.cc
index cc81ad20cacda..c875cdc37e80b 100644
--- a/paddle/fluid/operators/sequence_ops/sequence_unpad_op_xpu.cc
+++ b/paddle/fluid/operators/sequence_ops/sequence_unpad_op_xpu.cc
@@ -17,7 +17,7 @@ limitations under the License. */
 #include "paddle/fluid/operators/sequence_ops/sequence_unpad_op.h"
 
 namespace ops = paddle::operators;
-REGISTER_OP_XPU_KERNEL(sequence_unpad,
-                       ops::SequenceUnpadOpKernel<float, phi::XPUContext>);
+PD_REGISTER_STRUCT_KERNEL(
+    sequence_unpad, XPU, ALL_LAYOUT, ops::SequenceUnpadOpKernel, float) {}
 
 #endif
diff --git a/paddle/fluid/operators/uniform_random_inplace_op_xpu.cc b/paddle/fluid/operators/uniform_random_inplace_op_xpu.cc
index bf0360ace0b87..f1afd8ef3e213 100644
--- a/paddle/fluid/operators/uniform_random_inplace_op_xpu.cc
+++ b/paddle/fluid/operators/uniform_random_inplace_op_xpu.cc
@@ -22,7 +22,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename T>
+template <typename T, typename DeviceContext>
 class XPUUniformRandomInplaceKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext &ctx) const override {
@@ -71,7 +71,7 @@ class XPUUniformRandomInplaceKernel : public framework::OpKernel<T> {
   }
 };
 
-template <typename T>
+template <typename T, typename DeviceContext>
 class XPUUniformRandomInplaceGradKernel : public framework::OpKernel<T> {
  public:
   void Compute(const paddle::framework::ExecutionContext &ctx) const override {
@@ -95,10 +95,15 @@ class XPUUniformRandomInplaceGradKernel : public framework::OpKernel<T> {
 }  // namespace operators
 }  // namespace paddle
 
-REGISTER_OP_XPU_KERNEL(uniform_random_inplace,
-                       paddle::operators::XPUUniformRandomInplaceKernel<float>);
-REGISTER_OP_XPU_KERNEL(
-    uniform_random_inplace_grad,
-    paddle::operators::XPUUniformRandomInplaceGradKernel<float>);
+PD_REGISTER_STRUCT_KERNEL(uniform_random_inplace,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::XPUUniformRandomInplaceKernel,
+                          float) {}
+PD_REGISTER_STRUCT_KERNEL(uniform_random_inplace_grad,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::XPUUniformRandomInplaceGradKernel,
+                          float) {}
 
 #endif  // PADDLE_WITH_XPU

From aed9a75b05ee51276d92f23367a5147d9ff607e4 Mon Sep 17 00:00:00 2001
From: HongyuJia <jiahongyu@baidu.com>
Date: Wed, 26 Apr 2023 10:47:36 +0800
Subject: [PATCH 066/405] [Fix Patch Error] Fix patch error in cmake (#53321)

---
 cmake/external/eigen.cmake    |  5 +++--
 cmake/external/gloo.cmake     | 11 +++++++----
 cmake/external/gtest.cmake    |  6 ++++--
 cmake/external/pybind11.cmake |  9 +++++++--
 4 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/cmake/external/eigen.cmake b/cmake/external/eigen.cmake
index eda9ae8c4a2ac..8ae880773a4f4 100644
--- a/cmake/external/eigen.cmake
+++ b/cmake/external/eigen.cmake
@@ -51,9 +51,10 @@ if(CMAKE_COMPILER_IS_GNUCC)
   if(GCC_VERSION GREATER_EQUAL "12.0")
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Complex.h.patch
          complex_header)
+    # See: [Why calling some `git` commands before `patch`?]
     set(EIGEN_PATCH_COMMAND
-        patch -Nd ${EIGEN_SOURCE_DIR}/Eigen/src/Core/arch/SSE/ <
-        ${complex_header})
+        git checkout -- . && git checkout ${EIGEN_TAG} && patch -Nd
+        ${EIGEN_SOURCE_DIR}/Eigen/src/Core/arch/SSE/ < ${complex_header})
   endif()
 endif()
 
diff --git a/cmake/external/gloo.cmake b/cmake/external/gloo.cmake
index 63212c974e257..ebdcabd770cb0 100755
--- a/cmake/external/gloo.cmake
+++ b/cmake/external/gloo.cmake
@@ -37,8 +37,9 @@ if(WITH_GPU)
                                                   VERSION_GREATER 12.0)
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/gloo/device.cc.patch
          native_dst)
-    set(GLOO_PATCH_COMMAND patch -d ${GLOO_SOURCE_DIR}/gloo/transport/tcp <
-                           ${native_dst})
+    set(GLOO_PATCH_COMMAND
+        git checkout -- . && git checkout ${GLOO_TAG} &&patch -Nd
+        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst})
   endif()
 endif()
 
@@ -54,9 +55,11 @@ if(CMAKE_COMPILER_IS_GNUCC)
          native_dst)
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/gloo/types.h.patch
          types_header)
+    # See: [Why calling some `git` commands before `patch`?]
     set(GLOO_PATCH_COMMAND
-        patch -Nd ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst} &&
-        patch -Nd ${GLOO_SOURCE_DIR}/gloo/ < ${types_header})
+        git checkout -- . && git checkout ${GLOO_TAG} && patch -Nd
+        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch -Nd
+        ${GLOO_SOURCE_DIR}/gloo/ < ${types_header})
   endif()
 endif()
 include_directories(${GLOO_INCLUDE_DIR})
diff --git a/cmake/external/gtest.cmake b/cmake/external/gtest.cmake
index 315f6b5b752b2..777d37a023f05 100644
--- a/cmake/external/gtest.cmake
+++ b/cmake/external/gtest.cmake
@@ -66,8 +66,10 @@ endif()
 if(NOT WIN32 AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 12.0)
   file(TO_NATIVE_PATH
        ${PADDLE_SOURCE_DIR}/patches/gtest/gtest-death-test.cc.patch native_src)
-  set(GTEST_PATCH_COMMAND patch -Nd ${GTEST_SOURCE_DIR}/googletest/src <
-                          ${native_src})
+  # See: [Why calling some `git` commands before `patch`?]
+  set(GTEST_PATCH_COMMAND
+      git checkout -- . && git checkout ${GTEST_TAG} && patch -Nd
+      ${GTEST_SOURCE_DIR}/googletest/src < ${native_src})
 endif()
 if(WIN32)
   ExternalProject_Add(
diff --git a/cmake/external/pybind11.cmake b/cmake/external/pybind11.cmake
index db53e3511be44..c2782af255c59 100644
--- a/cmake/external/pybind11.cmake
+++ b/cmake/external/pybind11.cmake
@@ -25,8 +25,13 @@ set(PYBIND_PATCH_COMMAND "")
 if(NOT WIN32)
   file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/pybind/cast.h.patch
        native_dst)
-  set(PYBIND_PATCH_COMMAND patch -d ${PYBIND_INCLUDE_DIR}/pybind11 <
-                           ${native_dst})
+  # Note: [Why calling some `git` commands before `patch`?]
+  # Paddle's CI uses cache to accelarate the make process. However, error might raise when patch codes in two scenarios:
+  # 1. Patch to the wrong version: the tag version of CI's cache falls behind PYBIND_TAG, use `git checkout ${PYBIND_TAG}` to solve this.
+  # 2. Patch twice: the tag version of cache == PYBIND_TAG, but patch has already applied to cache.
+  set(PYBIND_PATCH_COMMAND
+      git checkout -- . && git checkout ${PYBIND_TAG} && patch -Nd
+      ${PYBIND_INCLUDE_DIR}/pybind11 < ${native_dst})
 endif()
 
 ExternalProject_Add(

From 652d100da33a629e49bb46281647ff5298f1cc70 Mon Sep 17 00:00:00 2001
From: risemeup1 <62429225+risemeup1@users.noreply.github.com>
Date: Wed, 26 Apr 2023 11:02:58 +0800
Subject: [PATCH 067/405] update cmake (#53345)

---
 tools/dockerfile/build_scripts/build.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/dockerfile/build_scripts/build.sh b/tools/dockerfile/build_scripts/build.sh
index 5e076e7696542..94014a4235713 100644
--- a/tools/dockerfile/build_scripts/build.sh
+++ b/tools/dockerfile/build_scripts/build.sh
@@ -61,9 +61,9 @@ yum -y install bzip2 make git patch unzip bison yasm diffutils \
 # /bin/sh cmake-3.8.1-Linux-x86_64.sh --prefix=/usr/local --skip-license
 # rm cmake-3.8.1-Linux-x86_64.sh
 
-wget -q https://cmake.org/files/v3.16/cmake-3.16.0.tar.gz && tar xzf cmake-3.16.0.tar.gz && \
-cd cmake-3.16.0 && ./bootstrap && \
-make -j8 && make install && cd .. && rm cmake-3.16.0.tar.gz && rm -rf cmake-3.16.0
+wget -q https://cmake.org/files/v3.18/cmake-3.18.0.tar.gz && tar xzf cmake-3.18.0.tar.gz && \
+cd cmake-3.18.0 && ./bootstrap && \
+make -j8 && make install && cd .. && rm cmake-3.18.0.tar.gz && rm -rf cmake-3.18.0
 
 # Install newest autoconf
 build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH

From ed040a17dba36ed12328102bef595fe8ae3bbf6d Mon Sep 17 00:00:00 2001
From: Wang Xin <xinwang614@gmail.com>
Date: Wed, 26 Apr 2023 11:03:22 +0800
Subject: [PATCH 068/405] add autogen code support for box_coder op (#53309)

---
 .../fluid/operators/detection/CMakeLists.txt  |   1 -
 .../fluid/operators/detection/box_coder_op.cc | 144 ------------------
 paddle/phi/api/yaml/legacy_ops.yaml           |   9 --
 paddle/phi/api/yaml/op_compat.yaml            |   6 +
 paddle/phi/api/yaml/ops.yaml                  |   9 ++
 paddle/phi/ops/compat/box_coder_sig.cc        |  28 ----
 6 files changed, 15 insertions(+), 182 deletions(-)
 delete mode 100644 paddle/fluid/operators/detection/box_coder_op.cc
 delete mode 100644 paddle/phi/ops/compat/box_coder_sig.cc

diff --git a/paddle/fluid/operators/detection/CMakeLists.txt b/paddle/fluid/operators/detection/CMakeLists.txt
index 8c5c1a5d8a2be..64f9e03d7e061 100644
--- a/paddle/fluid/operators/detection/CMakeLists.txt
+++ b/paddle/fluid/operators/detection/CMakeLists.txt
@@ -28,7 +28,6 @@ function(detection_library TARGET_NAME)
       PARENT_SCOPE)
 endfunction()
 
-detection_library(box_coder_op SRCS box_coder_op.cc)
 detection_library(density_prior_box_op SRCS density_prior_box_op.cc
                   density_prior_box_op.cu)
 
diff --git a/paddle/fluid/operators/detection/box_coder_op.cc b/paddle/fluid/operators/detection/box_coder_op.cc
deleted file mode 100644
index aafe040991ea0..0000000000000
--- a/paddle/fluid/operators/detection/box_coder_op.cc
+++ /dev/null
@@ -1,144 +0,0 @@
-/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-    http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include <vector>
-
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/phi/core/infermeta_utils.h"
-#include "paddle/phi/infermeta/ternary.h"
-
-namespace paddle {
-namespace operators {
-
-class BoxCoderOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-};
-
-class BoxCoderOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput(
-        "PriorBox",
-        "(Tensor, default Tensor<float>) "
-        "Box list PriorBox is a 2-D Tensor with shape [M, 4] holds M boxes, "
-        "each box is represented as [xmin, ymin, xmax, ymax], "
-        "[xmin, ymin] is the left top coordinate of the anchor box, "
-        "if the input is image feature map, they are close to the origin "
-        "of the coordinate system. [xmax, ymax] is the right bottom "
-        "coordinate of the anchor box.");
-    AddInput("PriorBoxVar",
-             "(Tensor, default Tensor<float>, optional) "
-             "PriorBoxVar is a 2-D Tensor with shape [M, 4] holds M group "
-             "of variance. PriorBoxVar will set all elements to 1 by "
-             "default.")
-        .AsDispensable();
-    AddInput(
-        "TargetBox",
-        "(phi::DenseTensor or Tensor) This input can be a 2-D phi::DenseTensor "
-        "with shape "
-        "[N, 4] when code_type is 'encode_center_size'. This input also can "
-        "be a 3-D Tensor with shape [N, M, 4] when code_type is "
-        "'decode_center_size'. [N, 4], each box is represented as "
-        "[xmin, ymin, xmax, ymax], [xmin, ymin] is the left top coordinate "
-        "of the box if the input is image feature map, they are close to "
-        "the origin of the coordinate system. [xmax, ymax] is the right "
-        "bottom coordinate of the box. This tensor can contain LoD "
-        "information to represent a batch of inputs. One instance of this "
-        "batch can contain different numbers of entities.");
-    AddAttr<std::string>("code_type",
-                         "(string, default encode_center_size) "
-                         "the code type used with the target box")
-        .SetDefault("encode_center_size")
-        .InEnum({"encode_center_size", "decode_center_size"});
-    AddAttr<bool>("box_normalized",
-                  "(bool, default true) "
-                  "whether treat the priorbox as a normalized box")
-        .SetDefault(true);
-    AddAttr<int>("axis",
-                 "(int, default 0)"
-                 "which axis in PriorBox to broadcast for box decode,"
-                 "for example, if axis is 0 and TargetBox has shape"
-                 "[N, M, 4] and PriorBox has shape [M, 4], then PriorBox "
-                 "will broadcast to [N, M, 4] for decoding. It is only valid"
-                 "when code type is decode_center_size")
-        .SetDefault(0)
-        .InEnum({0, 1});
-    AddAttr<std::vector<float>>(
-        "variance",
-        "(vector<float>, default {}),"
-        "variance of prior box with shape [4]. PriorBoxVar and variance can"
-        "not be provided at the same time.")
-        .SetDefault(std::vector<float>{});
-    AddOutput("OutputBox",
-              "(phi::DenseTensor or Tensor) "
-              "When code_type is 'encode_center_size', the output tensor of "
-              "box_coder_op with shape [N, M, 4] representing the result of N "
-              "target boxes encoded with M Prior boxes and variances. When "
-              "code_type is 'decode_center_size', N represents the batch size "
-              "and M represents the number of decoded boxes.");
-
-    AddComment(R"DOC(
-
-Bounding Box Coder.
-
-Encode/Decode the target bounding box with the priorbox information.
-
-The Encoding schema described below:
-
-    ox = (tx - px) / pw / pxv
-
-    oy = (ty - py) / ph / pyv
-
-    ow = log(abs(tw / pw)) / pwv
-
-    oh = log(abs(th / ph)) / phv
-
-The Decoding schema described below:
-
-    ox = (pw * pxv * tx * + px) - tw / 2
-
-    oy = (ph * pyv * ty * + py) - th / 2
-
-    ow = exp(pwv * tw) * pw + tw / 2
-
-    oh = exp(phv * th) * ph + th / 2
-
-where `tx`, `ty`, `tw`, `th` denote the target box's center coordinates, width
-and height respectively. Similarly, `px`, `py`, `pw`, `ph` denote the
-priorbox's (anchor) center coordinates, width and height. `pxv`, `pyv`, `pwv`,
-`phv` denote the variance of the priorbox and `ox`, `oy`, `ow`, `oh` denote the
-encoded/decoded coordinates, width and height.
-
-During Box Decoding, two modes for broadcast are supported. Say target box has
-shape [N, M, 4], and the shape of prior box can be [N, 4] or [M, 4]. Then prior
-box will broadcast to target box along the assigned axis.
-)DOC");
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-
-DECLARE_INFER_SHAPE_FUNCTOR(box_coder,
-                            BoxCoderInferShapeFunctor,
-                            PD_INFER_META(phi::BoxCoderInferMeta));
-
-REGISTER_OPERATOR(
-    box_coder,
-    ops::BoxCoderOp,
-    ops::BoxCoderOpMaker,
-    paddle::framework::EmptyGradOpMaker<paddle::framework::OpDesc>,
-    paddle::framework::EmptyGradOpMaker<paddle::imperative::OpBase>,
-    BoxCoderInferShapeFunctor);
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index 61a8cb76e5c2e..5bb3e8d0d73e7 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -139,15 +139,6 @@
     func: bincount
   optional: weights
 
-- op : box_coder
-  args : (Tensor prior_box, Tensor prior_box_var, Tensor target_box, str code_type, bool box_normalized, int axis, float[] variance)
-  output : Tensor(output_box)
-  infer_meta :
-    func : BoxCoderInferMeta
-  kernel :
-    func : box_coder
-  optional : prior_box_var
-
 - op : cast
   args : (Tensor x, DataType dtype)
   output : Tensor
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index 5585d0c2d3b97..8d29a77337320 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -343,6 +343,12 @@
   outputs :
     out : Out
 
+- op : box_coder
+  inputs :
+    {prior_box : PriorBox , prior_box_var : PriorBoxVar, target_box: TargetBox}
+  outputs :
+    output_box : OutputBox
+
 - op : broadcast_tensors
   backward : broadcast_tensors_grad
   inputs :
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index c228a0506e813..507fef3309660 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -329,6 +329,15 @@
     func : bmm
   backward : bmm_grad
 
+- op : box_coder
+  args : (Tensor prior_box, Tensor prior_box_var, Tensor target_box, str code_type = "encode_center_size", bool box_normalized = true, int axis = 0, float[] variance = {})
+  output : Tensor(output_box)
+  infer_meta :
+    func : BoxCoderInferMeta
+  kernel :
+    func : box_coder
+  optional : prior_box_var
+
 - op : broadcast_tensors
   args: (Tensor[] input)
   output: Tensor[]{input.size()}
diff --git a/paddle/phi/ops/compat/box_coder_sig.cc b/paddle/phi/ops/compat/box_coder_sig.cc
deleted file mode 100644
index 5b674f3dcd253..0000000000000
--- a/paddle/phi/ops/compat/box_coder_sig.cc
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/core/compat/op_utils.h"
-
-namespace phi {
-
-KernelSignature BoxCoderOpArgumentMapping(const ArgumentMappingContext& ctx) {
-  return KernelSignature("box_coder",
-                         {"PriorBox", "PriorBoxVar", "TargetBox"},
-                         {"code_type", "box_normalized", "axis", "variance"},
-                         {"OutputBox"});
-}
-
-}  // namespace phi
-
-PD_REGISTER_ARG_MAPPING_FN(box_coder, phi::BoxCoderOpArgumentMapping);

From 3ec12c2bfec19490669a8f475fbe0d6e3162d167 Mon Sep 17 00:00:00 2001
From: risemeup1 <62429225+risemeup1@users.noreply.github.com>
Date: Wed, 26 Apr 2023 11:04:46 +0800
Subject: [PATCH 069/405] Optimize prompt information (#53291)

* Optimize prompt information

* add_information

* add_information
---
 paddle/phi/backends/gpu/gpu_resources.cc | 39 +++++++++++++++++++++---
 1 file changed, 34 insertions(+), 5 deletions(-)

diff --git a/paddle/phi/backends/gpu/gpu_resources.cc b/paddle/phi/backends/gpu/gpu_resources.cc
index 06dc74f2d27f0..a447df94cb4dc 100644
--- a/paddle/phi/backends/gpu/gpu_resources.cc
+++ b/paddle/phi/backends/gpu/gpu_resources.cc
@@ -16,12 +16,12 @@
 
 #include <set>
 
+#include <map>
 #include "paddle/phi/api/include/tensor.h"
 #include "paddle/phi/backends/gpu/gpu_decls.h"
 #include "paddle/phi/backends/gpu/gpu_info.h"
 #include "paddle/phi/common/place.h"
 #include "paddle/phi/core/allocator.h"
-
 #ifdef PADDLE_WITH_CUDA
 #include "paddle/phi/backends/dynload/cublas.h"
 #include "paddle/phi/backends/dynload/cublasLt.h"
@@ -78,10 +78,39 @@ void InitGpuProperties(Place place,
       for (const int32_t& arch : compiled_archs) {
         compile_arch_str += std::to_string(arch) + " ";
       }
-      LOG(WARNING) << "Paddle with runtime capability " << *compute_capability
-                   << " is not compatible with Paddle installation with arch: "
-                   << compile_arch_str
-                   << ". Please check compiled version of Paddle. ";
+      std::map<int, std::string> arch_computing_mapping_table = {
+          {20, "Fermi"},
+          {30, "Kepler"},
+          {35, "Kapler"},
+          {37, "Kepler"},
+          {50, "Maxwell"},
+          {52, "Maxwell"},
+          {60, "Pascal"},
+          {61, "Pascal"},
+          {70, "Volta"},
+          {75, "Turing"},
+          {80, "Ampere"},
+          {86, "Ampere"},
+          {89, "Ampere"}};
+      if (arch_computing_mapping_table.count(*compute_capability)) {
+        LOG(WARNING)
+            << "The GPU architecture in your current machine is "
+            << arch_computing_mapping_table[*compute_capability]
+            << ", which is not compatible with Paddle installation with arch: "
+            << compile_arch_str
+            << ", it is recommended to install the corresponding wheel package "
+               "according to the installation information on the official "
+               "Paddle "
+               "website.";
+      } else {
+        LOG(WARNING)
+            << "The GPU compute capability in your current machine is "
+            << *compute_capability << ", which is not supported by Paddle"
+            << ", it is recommended to install the corresponding wheel package "
+               "according to the installation information on the official "
+               "Paddle "
+               "website.";
+      }
     }
   }
 #endif

From 1d549400dc4ae3b5da32a6a54e471257df027dd4 Mon Sep 17 00:00:00 2001
From: Lucas <33367939+cqulilujia@users.noreply.github.com>
Date: Wed, 26 Apr 2023 11:06:54 +0800
Subject: [PATCH 070/405] [Bug Fixs] fix bugs when using cast<int64_t, int32_t>
 in xpu/cross_entropy kernels, *test=kunlun (#53325)

---
 .../kernels/xpu/cross_entropy_grad_kernel.cc  | 65 +++++++++++++------
 .../phi/kernels/xpu/cross_entropy_kernel.cc   | 32 ++++++---
 2 files changed, 66 insertions(+), 31 deletions(-)

diff --git a/paddle/phi/kernels/xpu/cross_entropy_grad_kernel.cc b/paddle/phi/kernels/xpu/cross_entropy_grad_kernel.cc
index edb7157a3440a..a88abbb5e4c81 100644
--- a/paddle/phi/kernels/xpu/cross_entropy_grad_kernel.cc
+++ b/paddle/phi/kernels/xpu/cross_entropy_grad_kernel.cc
@@ -54,21 +54,32 @@ void CrossEntropyWithSoftmaxGradKernel(const Context& dev_ctx,
           d);
       PADDLE_ENFORCE_XDNN_SUCCESS(r, "soft_softmax_with_cross_entropy_grad");
     } else {
-      xpu::ctx_guard RAII_GUARD(dev_ctx.x_context());
-      int* labels_int_ptr_l3 =
-          RAII_GUARD.alloc_l3_or_gm<int32_t>(labels.numel());
-      PADDLE_ENFORCE_XDNN_NOT_NULL(labels_int_ptr_l3);
-
-      r = xpu::cast<int64_t, int32_t>(dev_ctx.x_context(),
-                                      labels.data<int64_t>(),
-                                      labels_int_ptr_l3,
-                                      labels.numel());
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast");
+      const int* labels_int_ptr = nullptr;
+      if (labels.dtype() == DataType::INT32) {
+        labels_int_ptr = labels.data<int32_t>();
+      } else if (labels.dtype() == DataType::INT64) {
+        xpu::ctx_guard RAII_GUARD(dev_ctx.x_context());
+        int* labels_int_ptr_l3 =
+            RAII_GUARD.alloc_l3_or_gm<int32_t>(labels.numel());
+        PADDLE_ENFORCE_XDNN_NOT_NULL(labels_int_ptr_l3);
+
+        r = xpu::cast<int64_t, int32_t>(dev_ctx.x_context(),
+                                        labels.data<int64_t>(),
+                                        labels_int_ptr_l3,
+                                        labels.numel());
+        PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast");
+        labels_int_ptr = labels_int_ptr_l3;
+      } else {
+        // TODO(lilujia): other data types should be handled
+        errors::Unimplemented(
+            ("cross_entropy does not support data types other than int32 and "
+             "int64"));
+      }
 
       r = xpu::hard_softmax_with_cross_entropy_grad<XPUType, int>(
           dev_ctx.x_context(),
           reinterpret_cast<const XPUType*>(loss_grad.data<T>()),
-          labels_int_ptr_l3,
+          labels_int_ptr,
           reinterpret_cast<const XPUType*>(softmax.data<T>()),
           reinterpret_cast<XPUType*>(logit_grad->data<T>()),
           ignore_index,
@@ -113,19 +124,31 @@ void CrossEntropyWithSoftmaxGradKernel(const Context& dev_ctx,
           t);
       PADDLE_ENFORCE_XDNN_SUCCESS(r, "soft_softmax_with_cross_entropy_grad");
     } else {
-      int* labels_int_ptr_l3 =
-          RAII_GUARD.alloc_l3_or_gm<int32_t>(labels.numel());
-      PADDLE_ENFORCE_XDNN_NOT_NULL(labels_int_ptr_l3);
-
-      r = xpu::cast<int64_t, int32_t>(dev_ctx.x_context(),
-                                      labels.data<int64_t>(),
-                                      labels_int_ptr_l3,
-                                      labels.numel());
-      PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast");
+      const int* labels_int_ptr = nullptr;
+      if (labels.dtype() == DataType::INT32) {
+        labels_int_ptr = labels.data<int32_t>();
+      } else if (labels.dtype() == DataType::INT64) {
+        int* labels_int_ptr_l3 =
+            RAII_GUARD.alloc_l3_or_gm<int32_t>(labels.numel());
+        PADDLE_ENFORCE_XDNN_NOT_NULL(labels_int_ptr_l3);
+
+        r = xpu::cast<int64_t, int32_t>(dev_ctx.x_context(),
+                                        labels.data<int64_t>(),
+                                        labels_int_ptr_l3,
+                                        labels.numel());
+        PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast");
+        labels_int_ptr = labels_int_ptr_l3;
+      } else {
+        // TODO(lilujia): other data types should be handled
+        errors::Unimplemented(
+            ("cross_entropy does not support data types other than int32 and "
+             "int64"));
+      }
+
       r = xpu::hard_softmax_with_cross_entropy_grad<XPUType, int>(
           dev_ctx.x_context(),
           reinterpret_cast<const XPUType*>(loss_grad.data<T>()),
-          labels_int_ptr_l3,
+          labels_int_ptr,
           trans_softmax,
           trans_logit,
           ignore_index,
diff --git a/paddle/phi/kernels/xpu/cross_entropy_kernel.cc b/paddle/phi/kernels/xpu/cross_entropy_kernel.cc
index f1b2257427f67..b678fde9a882b 100644
--- a/paddle/phi/kernels/xpu/cross_entropy_kernel.cc
+++ b/paddle/phi/kernels/xpu/cross_entropy_kernel.cc
@@ -133,20 +133,32 @@ void CrossEntropyWithSoftmaxKernel(const Context& dev_ctx,
                                          axis == rank - 1 ? d : t);
     PADDLE_ENFORCE_XDNN_SUCCESS(r, "soft_cross_entropy");
   } else {
-    DenseTensor labels_int32;
-    int* labels_int_ptr_l3 = RAII_GUARD.alloc_l3_or_gm<int32_t>(labels.numel());
-    PADDLE_ENFORCE_XDNN_NOT_NULL(labels_int_ptr_l3);
-
-    r = xpu::cast<int64_t, int32_t>(dev_ctx.x_context(),
-                                    labels.data<int64_t>(),
-                                    labels_int_ptr_l3,
-                                    labels.numel());
-    PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast");
+    const int* labels_int_ptr = nullptr;
+    if (labels.dtype() == DataType::INT32) {
+      labels_int_ptr = labels.data<int32_t>();
+    } else if (labels.dtype() == DataType::INT64) {
+      xpu::ctx_guard RAII_GUARD(dev_ctx.x_context());
+      int* labels_int_ptr_l3 =
+          RAII_GUARD.alloc_l3_or_gm<int32_t>(labels.numel());
+      PADDLE_ENFORCE_XDNN_NOT_NULL(labels_int_ptr_l3);
+
+      r = xpu::cast<int64_t, int32_t>(dev_ctx.x_context(),
+                                      labels.data<int64_t>(),
+                                      labels_int_ptr_l3,
+                                      labels.numel());
+      PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast");
+      labels_int_ptr = labels_int_ptr_l3;
+    } else {
+      // TODO(lilujia): other data types should be handled
+      errors::Unimplemented(
+          ("cross_entropy does not support data types other than int32 and "
+           "int64"));
+    }
 
     r = xpu::hard_cross_entropy<XPUType, int32_t>(
         dev_ctx.x_context(),
         softmax_data,
-        labels_int_ptr_l3,
+        labels_int_ptr,
         loss_data,
         nullptr,
         axis == rank - 1 ? n : n * d / t,

From 9127cc3c969cd3c94c91cac90a0fb6c164c29d9a Mon Sep 17 00:00:00 2001
From: denglianbin <112610123+denglianbin@users.noreply.github.com>
Date: Wed, 26 Apr 2023 11:15:24 +0800
Subject: [PATCH 071/405] =?UTF-8?q?=E3=80=90Hackathon=20No.48=E3=80=91?=
 =?UTF-8?q?=E4=B8=BA=20Paddle=20meshgrid=20=E7=AE=97=E5=AD=90=E5=AE=9E?=
 =?UTF-8?q?=E7=8E=B0=20float16=20=E6=95=B0=E6=8D=AE=E7=B1=BB=E5=9E=8B?=
 =?UTF-8?q?=E6=94=AF=E6=8C=81=20(#53284)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 paddle/phi/kernels/gpu/meshgrid_grad_kernel.cu.cc       | 1 +
 paddle/phi/kernels/gpu/meshgrid_kernel.cu.cc            | 1 +
 python/paddle/fluid/tests/unittests/test_meshgrid_op.py | 8 ++++++++
 python/paddle/tensor/creation.py                        | 2 +-
 4 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/paddle/phi/kernels/gpu/meshgrid_grad_kernel.cu.cc b/paddle/phi/kernels/gpu/meshgrid_grad_kernel.cu.cc
index 80cf88b3ceb7f..17f74cd3743bd 100644
--- a/paddle/phi/kernels/gpu/meshgrid_grad_kernel.cu.cc
+++ b/paddle/phi/kernels/gpu/meshgrid_grad_kernel.cu.cc
@@ -22,6 +22,7 @@ PD_REGISTER_KERNEL(meshgrid_grad,
                    GPU,
                    ALL_LAYOUT,
                    phi::MeshgridGradKernel,
+                   phi::dtype::float16,
                    float,
                    double,
                    int,
diff --git a/paddle/phi/kernels/gpu/meshgrid_kernel.cu.cc b/paddle/phi/kernels/gpu/meshgrid_kernel.cu.cc
index c863550979444..73120c1391642 100644
--- a/paddle/phi/kernels/gpu/meshgrid_kernel.cu.cc
+++ b/paddle/phi/kernels/gpu/meshgrid_kernel.cu.cc
@@ -22,6 +22,7 @@ PD_REGISTER_KERNEL(meshgrid,
                    GPU,
                    ALL_LAYOUT,
                    phi::MeshgridKernel,
+                   phi::dtype::float16,
                    float,
                    double,
                    int,
diff --git a/python/paddle/fluid/tests/unittests/test_meshgrid_op.py b/python/paddle/fluid/tests/unittests/test_meshgrid_op.py
index 60af417ebc545..0039d4ee422e8 100644
--- a/python/paddle/fluid/tests/unittests/test_meshgrid_op.py
+++ b/python/paddle/fluid/tests/unittests/test_meshgrid_op.py
@@ -76,6 +76,14 @@ def get_x_shape(self):
         return [100, 300]
 
 
+class TestMeshgridOp2Fp16(TestMeshgridOp):
+    def get_x_shape(self):
+        return [100, 300]
+
+    def get_dtype(self):
+        return np.float16
+
+
 class TestMeshgridOp3(unittest.TestCase):
     def test_api(self):
         x = paddle.static.data(shape=[100], dtype='int32', name='x')
diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py
index 186eda03e74d8..c57fceeeb8525 100644
--- a/python/paddle/tensor/creation.py
+++ b/python/paddle/tensor/creation.py
@@ -1512,7 +1512,7 @@ def meshgrid(*args, **kwargs):
 
     Args:
         *args(Tensor|list of Tensor) : tensors (tuple(list) of tensor): the shapes of input k tensors are (N1,),
-            (N2,),..., (Nk,). Support data types: ``float64``, ``float32``, ``int32``, ``int64``.
+            (N2,),..., (Nk,). Support data types: ``float64``, ``float16``, ``float32``, ``int32``, ``int64``.
         **kwargs (optional): Currently, only accept name in **kwargs
             The default value is None. Normally there is no need for
             user to set this property. For more information, please refer to :ref:`api_guide_Name`.

From 2a705b74a02cb8072d5a8510276e6a5154ab9ee1 Mon Sep 17 00:00:00 2001
From: denglianbin <112610123+denglianbin@users.noreply.github.com>
Date: Wed, 26 Apr 2023 11:15:49 +0800
Subject: [PATCH 072/405] =?UTF-8?q?=E3=80=90Hackathon=20No.48=E3=80=91?=
 =?UTF-8?q?=E4=B8=BA=20Paddle=20determinant=20=E7=AE=97=E5=AD=90=E5=AE=9E?=
 =?UTF-8?q?=E7=8E=B0=20float16=20=E6=95=B0=E6=8D=AE=E7=B1=BB=E5=9E=8B?=
 =?UTF-8?q?=E6=94=AF=E6=8C=81=20(#53286)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../kernels/gpu/determinant_grad_kernel.cu    |  1 +
 paddle/phi/kernels/gpu/determinant_kernel.cu  |  9 ++++--
 .../impl/determinant_grad_kernel_impl.h       | 32 +++++++++++++++----
 .../kernels/impl/determinant_kernel_impl.h    | 12 ++++++-
 .../tests/unittests/test_determinant_op.py    | 19 +++++++++++
 python/paddle/tensor/linalg.py                |  2 +-
 6 files changed, 65 insertions(+), 10 deletions(-)

diff --git a/paddle/phi/kernels/gpu/determinant_grad_kernel.cu b/paddle/phi/kernels/gpu/determinant_grad_kernel.cu
index cce12a87fac72..f3187d5fefb51 100644
--- a/paddle/phi/kernels/gpu/determinant_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/determinant_grad_kernel.cu
@@ -21,5 +21,6 @@ PD_REGISTER_KERNEL(determinant_grad,
                    GPU,
                    ALL_LAYOUT,
                    phi::DeterminantGradKernel,
+                   phi::dtype::float16,
                    float,
                    double) {}
diff --git a/paddle/phi/kernels/gpu/determinant_kernel.cu b/paddle/phi/kernels/gpu/determinant_kernel.cu
index 2518408387395..58e27e3ce4abd 100644
--- a/paddle/phi/kernels/gpu/determinant_kernel.cu
+++ b/paddle/phi/kernels/gpu/determinant_kernel.cu
@@ -17,5 +17,10 @@
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/impl/determinant_kernel_impl.h"
 
-PD_REGISTER_KERNEL(
-    determinant, GPU, ALL_LAYOUT, phi::DeterminantKernel, float, double) {}
+PD_REGISTER_KERNEL(determinant,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::DeterminantKernel,
+                   phi::dtype::float16,
+                   float,
+                   double) {}
diff --git a/paddle/phi/kernels/impl/determinant_grad_kernel_impl.h b/paddle/phi/kernels/impl/determinant_grad_kernel_impl.h
index 3f463e1d9e064..4d58698c64d22 100644
--- a/paddle/phi/kernels/impl/determinant_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/determinant_grad_kernel_impl.h
@@ -15,8 +15,10 @@
 #pragma once
 
 #include "glog/logging.h"
+#include "paddle/phi/common/amp_type_traits.h"
 
 #include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/kernels/cast_kernel.h"
 #include "paddle/phi/kernels/determinant_grad_kernel.h"
 #include "paddle/phi/kernels/elementwise_multiply_kernel.h"
 #include "paddle/phi/kernels/empty_kernel.h"
@@ -26,7 +28,6 @@
 #include "paddle/phi/kernels/funcs/matrix_inverse.h"
 #include "paddle/phi/kernels/funcs/unsqueeze.h"
 #include "paddle/phi/kernels/transpose_kernel.h"
-
 namespace phi {
 namespace detail {
 
@@ -113,6 +114,11 @@ void DeterminantGradKernel(const Context& dev_ctx,
     return;
   }
 
+  using MPType = typename phi::dtype::MPTypeTrait<T>::Type;
+  auto origin_dt = std::is_same<phi::dtype::float16, T>::value
+                       ? DataType::FLOAT16
+                       : DataType::BFLOAT16;
+
   // The matrix is invertible
   // let |A| = Determinant(A)
   // Ref to https://people.maths.ox.ac.uk/gilesm/files/NA-08-01.pdf
@@ -123,16 +129,22 @@ void DeterminantGradKernel(const Context& dev_ctx,
   DenseTensor inverse_A;
   // A must be square matrices!
   inverse_A.Resize(x.dims());
-  dev_ctx.template Alloc<T>(&inverse_A);
+  dev_ctx.template Alloc<MPType>(&inverse_A);
 
-  phi::funcs::MatrixInverseFunctor<Context, T> mat_inv;
-  mat_inv(dev_ctx, x, &inverse_A);
+  phi::funcs::MatrixInverseFunctor<Context, MPType> mat_inv;
+  if (!std::is_same<MPType, T>::value) {
+    mat_inv(dev_ctx,
+            phi::Cast<T, Context>(dev_ctx, x, DataType::FLOAT32),
+            &inverse_A);
+  } else {
+    mat_inv(dev_ctx, x, &inverse_A);
+  }
 
   VLOG(3) << "inverse(A) dims: " << inverse_A.dims();
 
   // Second: inverse(A).transpose(-2, -1)
   DenseTensor transpose_inverse_A =
-      phi::TransposeLast2Dim<T>(dev_ctx, inverse_A);
+      phi::TransposeLast2Dim<MPType>(dev_ctx, inverse_A);
 
   VLOG(3) << "(dA * |A|).transpose(-2, -1) dims: "
           << transpose_inverse_A.dims();
@@ -147,7 +159,15 @@ void DeterminantGradKernel(const Context& dev_ctx,
   VLOG(3) << "unsqueezed(dA * |A|) dims: " << unsqueeze2.dims();
 
   // Finally: unsqueeze(dA * |A|) * inverse(A)
-  auto res = phi::Multiply<T>(dev_ctx, unsqueeze2, transpose_inverse_A);
+  DenseTensor res;
+  if (!std::is_same<MPType, T>::value) {
+    res = phi::Multiply<T>(
+        dev_ctx,
+        unsqueeze2,
+        phi::Cast<MPType, Context>(dev_ctx, transpose_inverse_A, origin_dt));
+  } else {
+    res = phi::Multiply<T>(dev_ctx, unsqueeze2, transpose_inverse_A);
+  }
 
   VLOG(3) << "unsqueeze(dA * |A|) * inverse(A) dims: " << res.dims();
 
diff --git a/paddle/phi/kernels/impl/determinant_kernel_impl.h b/paddle/phi/kernels/impl/determinant_kernel_impl.h
index 36e47c78c832c..3c437ad659c43 100644
--- a/paddle/phi/kernels/impl/determinant_kernel_impl.h
+++ b/paddle/phi/kernels/impl/determinant_kernel_impl.h
@@ -21,6 +21,7 @@
 #include <vector>
 
 #include "glog/logging.h"
+#include "paddle/phi/common/amp_type_traits.h"
 
 #include "paddle/phi/core/enforce.h"
 #include "paddle/phi/core/tensor_utils.h"
@@ -31,6 +32,13 @@ namespace detail {
 template <typename T>
 class EigenMatrix {};
 
+template <>
+class EigenMatrix<phi::dtype::float16> {
+ public:
+  using MatrixType =
+      Eigen::Matrix<phi::dtype::float16, Eigen::Dynamic, Eigen::Dynamic>;
+};
+
 template <>
 class EigenMatrix<float> {
  public:
@@ -74,6 +82,7 @@ struct DeterminantFunctor {
     std::vector<T> input_vec;
     std::vector<T> output_vec;
     phi::TensorToVector(input, dev_ctx, &input_vec);
+    using MPType = typename phi::dtype::MPTypeTrait<T>::Type;
     for (int64_t i = 0; i < batch_count; ++i) {  // maybe can be parallel
       auto begin_iter = input_vec.begin() + i * rank * rank;
       auto end_iter = input_vec.begin() + (i + 1) * rank * rank;
@@ -85,7 +94,8 @@ struct DeterminantFunctor {
           matrix(i, j) = sub_vec[rank * i + j];
         }
       }
-      output_vec.push_back(matrix.determinant());
+      output_vec.push_back(
+          static_cast<T>(matrix.template cast<MPType>().determinant()));
     }
     phi::TensorFromVector(output_vec, dev_ctx, output);
   }
diff --git a/python/paddle/fluid/tests/unittests/test_determinant_op.py b/python/paddle/fluid/tests/unittests/test_determinant_op.py
index ade000cda8712..8e50f0c5552ec 100644
--- a/python/paddle/fluid/tests/unittests/test_determinant_op.py
+++ b/python/paddle/fluid/tests/unittests/test_determinant_op.py
@@ -50,6 +50,14 @@ def init_data(self):
         self.target = np.linalg.det(self.case)
 
 
+class TestDeterminantOpCase1FP16(TestDeterminantOp):
+    def init_data(self):
+        np.random.seed(0)
+        self.case = np.random.rand(10, 10).astype(np.float16)
+        self.inputs = {'Input': self.case}
+        self.target = np.linalg.det(self.case.astype(np.float32))
+
+
 class TestDeterminantOpCase2(TestDeterminantOp):
     def init_data(self):
         np.random.seed(0)
@@ -59,6 +67,17 @@ def init_data(self):
         self.target = np.linalg.det(self.case)
 
 
+class TestDeterminantOpCase2FP16(TestDeterminantOp):
+    def init_data(self):
+        np.random.seed(0)
+        # not invertible matrix
+        self.case = np.ones([4, 2, 4, 4]).astype(np.float16)
+        self.inputs = {'Input': self.case}
+        self.target = np.linalg.det(self.case.astype(np.float32)).astype(
+            np.float16
+        )
+
+
 class TestDeterminantAPI(unittest.TestCase):
     def setUp(self):
         np.random.seed(0)
diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py
index 3dcbc7c6ac63b..2235cf93cfb60 100644
--- a/python/paddle/tensor/linalg.py
+++ b/python/paddle/tensor/linalg.py
@@ -1809,7 +1809,7 @@ def det(x, name=None):
     if in_dygraph_mode():
         return _C_ops.det(x)
     else:
-        check_dtype(x.dtype, 'Input', ['float32', 'float64'], 'det')
+        check_dtype(x.dtype, 'Input', ['float16', 'float32', 'float64'], 'det')
 
         input_shape = list(x.shape)
         assert len(input_shape) >= 2, (

From cf6ed7cb004831384421f62bab236f6e4f2505d7 Mon Sep 17 00:00:00 2001
From: denglianbin <112610123+denglianbin@users.noreply.github.com>
Date: Wed, 26 Apr 2023 11:16:22 +0800
Subject: [PATCH 073/405] =?UTF-8?q?=20=E3=80=90Hackathon=20No.48=E3=80=91?=
 =?UTF-8?q?=E4=B8=BA=20Paddle=20kthvalue=20=E7=AE=97=E5=AD=90=E5=AE=9E?=
 =?UTF-8?q?=E7=8E=B0=20float16=20=E6=95=B0=E6=8D=AE=E7=B1=BB=E5=9E=8B?=
 =?UTF-8?q?=E6=94=AF=E6=8C=81=20(#53285)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../fluid/tests/unittests/test_kthvalue_op.py | 24 +++++++++++++++----
 python/paddle/tensor/search.py                |  2 +-
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/test_kthvalue_op.py b/python/paddle/fluid/tests/unittests/test_kthvalue_op.py
index 0bf3d8e948097..e2fa225fd8f7e 100644
--- a/python/paddle/fluid/tests/unittests/test_kthvalue_op.py
+++ b/python/paddle/fluid/tests/unittests/test_kthvalue_op.py
@@ -40,11 +40,14 @@ def init_args(self):
         self.k = 5
         self.axis = -1
 
+    def init_dtype(self):
+        self.dtype = np.float64
+
     def setUp(self):
         self.op_type = "kthvalue"
         self.python_api = paddle.kthvalue
-        self.dtype = np.float64
-        self.input_data = np.random.random((2, 1, 2, 4, 10))
+        self.init_dtype()
+        self.input_data = np.random.random((2, 1, 2, 4, 10)).astype(self.dtype)
         self.init_args()
         self.inputs = {'X': self.input_data}
         self.attrs = {'k': self.k, 'axis': self.axis}
@@ -62,17 +65,25 @@ def test_check_grad(self):
         self.check_grad({'X'}, 'Out')
 
 
+class TestKthvalueOpFp16(TestKthvalueOp):
+    def init_dtype(self):
+        self.dtype = np.float16
+
+
 class TestKthvalueOpWithKeepdim(OpTest):
     def init_args(self):
         self.k = 2
         self.axis = 1
 
+    def init_dtype(self):
+        self.dtype = np.float64
+
     def setUp(self):
         self.init_args()
+        self.init_dtype()
         self.op_type = "kthvalue"
         self.python_api = paddle.kthvalue
-        self.dtype = np.float64
-        self.input_data = np.random.random((1, 3, 2, 4, 10))
+        self.input_data = np.random.random((1, 3, 2, 4, 10)).astype(self.dtype)
         self.inputs = {'X': self.input_data}
         self.attrs = {'k': self.k, 'axis': self.axis, 'keepdim': True}
         output, indices = cal_kthvalue(
@@ -89,6 +100,11 @@ def test_check_grad(self):
         self.check_grad({'X'}, 'Out')
 
 
+class TestKthvalueOpWithKeepdimFp16(TestKthvalueOpWithKeepdim):
+    def init_dtype(self):
+        self.dtype = np.float16
+
+
 class TestKthvalueOpKernels(unittest.TestCase):
     def setUp(self):
         self.axises = [2, -1]
diff --git a/python/paddle/tensor/search.py b/python/paddle/tensor/search.py
index c9c1d4c35c165..9fc8e39a9ed82 100755
--- a/python/paddle/tensor/search.py
+++ b/python/paddle/tensor/search.py
@@ -1074,7 +1074,7 @@ def kthvalue(x, k, axis=None, keepdim=False, name=None):
     Find values and indices of the k-th smallest at the axis.
 
     Args:
-        x(Tensor): A N-D Tensor with type float32, float64, int32, int64.
+        x(Tensor): A N-D Tensor with type float16, float32, float64, int32, int64.
         k(int): The k for the k-th smallest number to look for along the axis.
         axis(int, optional): Axis to compute indices along. The effective range
             is [-R, R), where R is x.ndim. when axis < 0, it works the same way

From b305629c992bf98f5d13722d1f6b9a8f7b3de6f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=99=88=E6=B2=A7=E5=A4=9C?= <chenloong@88.com>
Date: Wed, 26 Apr 2023 11:42:23 +0800
Subject: [PATCH 074/405] remove *npu.cc (#53342)

---
 .../operators/detection/box_coder_op_npu.cc   | 448 ------------------
 .../detection/density_prior_box_op_npu.cc     | 396 ----------------
 .../detection/iou_similarity_op_npu.cc        | 204 --------
 .../operators/detection/prior_box_op_npu.cc   | 106 -----
 4 files changed, 1154 deletions(-)
 delete mode 100644 paddle/fluid/operators/detection/box_coder_op_npu.cc
 delete mode 100644 paddle/fluid/operators/detection/density_prior_box_op_npu.cc
 delete mode 100644 paddle/fluid/operators/detection/iou_similarity_op_npu.cc
 delete mode 100644 paddle/fluid/operators/detection/prior_box_op_npu.cc

diff --git a/paddle/fluid/operators/detection/box_coder_op_npu.cc b/paddle/fluid/operators/detection/box_coder_op_npu.cc
deleted file mode 100644
index 4170088faff18..0000000000000
--- a/paddle/fluid/operators/detection/box_coder_op_npu.cc
+++ /dev/null
@@ -1,448 +0,0 @@
-/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-    http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include <string>
-#include <vector>
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/phi/kernels/impl/box_coder.h"
-
-namespace paddle {
-namespace operators {
-
-template <typename T>
-struct BoxCoderFunction {
- public:
-  explicit BoxCoderFunction(const framework::ExecutionContext& ctx) : ctx(ctx) {
-    place = ctx.GetPlace();
-    stream = ctx.template device_context<paddle::platform::NPUDeviceContext>()
-                 .stream();
-  }
-  phi::DenseTensor Adds(const phi::DenseTensor& x, float scalar) {
-    phi::DenseTensor y;
-    y.mutable_data<T>(x.dims(), place);
-    const auto& runner = NpuOpRunner("Adds", {x}, {y}, {{"value", scalar}});
-    runner.Run(stream);
-    return y;
-  }
-  phi::DenseTensor Muls(const phi::DenseTensor& x, float scalar) {
-    phi::DenseTensor y;
-    y.mutable_data<T>(x.dims(), place);
-    const auto& runner = NpuOpRunner("Muls", {x}, {y}, {{"value", scalar}});
-    runner.Run(stream);
-    return y;
-  }
-  phi::DenseTensor Mul(const phi::DenseTensor& x, const phi::DenseTensor& y) {
-    phi::DenseTensor z;
-    z.mutable_data<T>(x.dims(), place);
-    const auto& runner = NpuOpRunner("Mul", {x, y}, {z}, {});
-    runner.Run(stream);
-    return z;
-  }
-  phi::DenseTensor SubWithBroadCast(const phi::DenseTensor& x,
-                                    const phi::DenseTensor& y,
-                                    const framework::DDim& shape) {
-    phi::DenseTensor z;
-    z.mutable_data<T>(shape, place);
-    const auto& runner = NpuOpRunner("Sub", {x, y}, {z}, {});
-    runner.Run(stream);
-    return z;
-  }
-  void DivWithBroadCastVoid(const phi::DenseTensor& x,
-                            const phi::DenseTensor& y,
-                            const framework::DDim& shape,
-                            phi::DenseTensor* z) {
-    z->mutable_data<T>(shape, place);
-    const auto& runner = NpuOpRunner("Div", {x, y}, {*z}, {});
-    runner.Run(stream);
-  }
-  phi::DenseTensor DivWithBroadCast(const phi::DenseTensor& x,
-                                    const phi::DenseTensor& y,
-                                    const framework::DDim& shape) {
-    phi::DenseTensor z;
-    DivWithBroadCastVoid(x, y, shape, &z);
-    return z;
-  }
-  void MulWithBroadCastVoid(const phi::DenseTensor& x,
-                            const phi::DenseTensor& y,
-                            const framework::DDim& shape,
-                            phi::DenseTensor* z) {
-    z->mutable_data<T>(shape, place);
-    const auto& runner = NpuOpRunner("Mul", {x, y}, {*z}, {});
-    runner.Run(stream);
-  }
-  phi::DenseTensor MulWithBroadCast(const phi::DenseTensor& x,
-                                    const phi::DenseTensor& y,
-                                    const framework::DDim& shape) {
-    phi::DenseTensor z;
-    MulWithBroadCastVoid(x, y, shape, &z);
-    return z;
-  }
-  void AddWithBroadCastVoid(const phi::DenseTensor& x,
-                            const phi::DenseTensor& y,
-                            const framework::DDim& shape,
-                            phi::DenseTensor* z) {
-    z->mutable_data<T>(shape, place);
-    const auto& runner = NpuOpRunner("AddV2", {x, y}, {*z}, {});
-    runner.Run(stream);
-  }
-  phi::DenseTensor AddWithBroadCast(const phi::DenseTensor& x,
-                                    const phi::DenseTensor& y,
-                                    const framework::DDim& shape) {
-    phi::DenseTensor z;
-    AddWithBroadCastVoid(x, y, shape, &z);
-    return z;
-  }
-  phi::DenseTensor Abs(const phi::DenseTensor& x) {
-    phi::DenseTensor y;
-    y.mutable_data<T>(x.dims(), place);
-    const auto& runner = NpuOpRunner("Abs", {x}, {y}, {});
-    runner.Run(stream);
-    return y;
-  }
-  phi::DenseTensor Log(const phi::DenseTensor& x) {
-    phi::DenseTensor t_x_m1 = Adds(x, -1);
-    phi::DenseTensor y;
-    y.mutable_data<T>(x.dims(), place);
-    const auto& runner = NpuOpRunner("Log1p", {t_x_m1}, {y}, {});
-    runner.Run(stream);
-    return y;
-  }
-  phi::DenseTensor Exp(const phi::DenseTensor& x) {
-    phi::DenseTensor y;
-    y.mutable_data<T>(x.dims(), place);
-    const auto& runner = NpuOpRunner("Exp", {x}, {y}, {});
-    runner.Run(stream);
-    return y;
-  }
-  phi::DenseTensor Dot(const phi::DenseTensor& x, const phi::DenseTensor& y) {
-    auto dim_x = x.dims();
-    auto dim_y = y.dims();
-    PADDLE_ENFORCE_EQ(
-        dim_x.size(),
-        2,
-        platform::errors::InvalidArgument(
-            "x should be a 2-dim tensor, but got %d-dim.", dim_x.size()));
-    PADDLE_ENFORCE_EQ(
-        dim_y.size(),
-        2,
-        platform::errors::InvalidArgument(
-            "y should be a 2-dim tensor, but got %d-dim.", dim_y.size()));
-    PADDLE_ENFORCE_EQ(
-        dim_x[1],
-        dim_y[0],
-        platform::errors::InvalidArgument("Expect dim_x[1] == dim_y[0], but "
-                                          "got dim_x[1] = %d, dim_y[0] = %d.",
-                                          dim_x[1],
-                                          dim_y[0]));
-    phi::DenseTensor z;
-    z.mutable_data<T>({dim_x[0], dim_y[1]}, place);
-    const auto& runner =
-        NpuOpRunner("MatMul",
-                    {x, y},
-                    {z},
-                    {{"transpose_x1", false}, {"transpose_x2", false}});
-    runner.Run(stream);
-    return z;
-  }
-  void ConcatVoid(const std::vector<phi::DenseTensor>& inputs,
-                  const framework::DDim& shape_out,
-                  int axis,
-                  phi::DenseTensor* output) {
-    output->mutable_data<T>(shape_out, place);
-    std::vector<std::string> names;
-    for (size_t i = 0; i < inputs.size(); i++) {
-      names.push_back("x" + std::to_string(i));
-    }
-    NpuOpRunner runner{
-        "ConcatD",
-        {inputs},
-        {*output},
-        {{"concat_dim", axis}, {"N", static_cast<int>(inputs.size())}}};
-    runner.AddInputNames(names);
-    runner.Run(stream);
-  }
-  phi::DenseTensor Concat(const std::vector<phi::DenseTensor>& inputs,
-                          const framework::DDim& shape_out,
-                          int axis) {
-    phi::DenseTensor output;
-    ConcatVoid(inputs, shape_out, axis, &output);
-    return output;
-  }
-  phi::DenseTensor Slice(const phi::DenseTensor& x,
-                         const std::vector<int>& offsets,
-                         const std::vector<int>& size,
-                         const framework::DDim& shape) {
-    phi::DenseTensor y;
-    y.mutable_data<T>(shape, place);
-    const auto& runner =
-        NpuOpRunner("SliceD", {x}, {y}, {{"offsets", offsets}, {"size", size}});
-    runner.Run(stream);
-    return y;
-  }
-
- private:
-  platform::Place place;
-  aclrtStream stream;
-  const framework::ExecutionContext& ctx;
-};
-
-template <typename T>
-void Vector2Tensor(const framework::ExecutionContext& ctx,
-                   const std::vector<T>& vec,
-                   const framework::DDim& ddim,
-                   phi::DenseTensor* tsr) {
-  framework::TensorFromVector<T>(vec, ctx.device_context(), tsr);
-  ctx.template device_context<paddle::platform::NPUDeviceContext>().Wait();
-  tsr->Resize(ddim);
-}
-
-template <typename T>
-void BoxCoderEnc(const framework::ExecutionContext& ctx,
-                 const phi::DenseTensor* tb,
-                 const phi::DenseTensor* pb,
-                 const phi::DenseTensor* pbv,
-                 const bool norm,
-                 const std::vector<float>& variance,
-                 phi::DenseTensor* out) {
-  auto M = pb->dims()[0];
-  auto N = tb->dims()[0];
-  auto shape_0 = phi::make_ddim({4, 2});
-  phi::DenseTensor m_diff;
-  phi::DenseTensor m_aver;
-  std::vector<T> vec_diff = {static_cast<T>(-1),
-                             static_cast<T>(0),
-                             static_cast<T>(0),
-                             static_cast<T>(-1),
-                             static_cast<T>(1),
-                             static_cast<T>(0),
-                             static_cast<T>(0),
-                             static_cast<T>(1)};
-  std::vector<T> vec_aver = {static_cast<T>(0.5),
-                             static_cast<T>(0),
-                             static_cast<T>(0),
-                             static_cast<T>(0.5),
-                             static_cast<T>(0.5),
-                             static_cast<T>(0),
-                             static_cast<T>(0),
-                             static_cast<T>(0.5)};
-  Vector2Tensor<T>(ctx, vec_diff, shape_0, &m_diff);
-  Vector2Tensor<T>(ctx, vec_aver, shape_0, &m_aver);
-
-  BoxCoderFunction<T> F(ctx);
-  phi::DenseTensor pb_xy = F.Adds(F.Dot(*pb, m_aver), (norm ? 0 : 0.5));
-  phi::DenseTensor pb_wh = F.Adds(F.Dot(*pb, m_diff), (norm ? 0 : 1));
-  phi::DenseTensor tb_xy = F.Dot(*tb, m_aver);
-  phi::DenseTensor tb_wh = F.Adds(F.Dot(*tb, m_diff), (norm ? 0 : 1));
-
-  pb_xy.Resize({1, M, 2});
-  pb_wh.Resize({1, M, 2});
-  tb_xy.Resize({N, 1, 2});
-  tb_wh.Resize({N, 1, 2});
-
-  auto shape_half = phi::make_ddim({N, M, 2});
-  auto shape_full = phi::make_ddim({N, M, 4});
-
-  phi::DenseTensor out_xy_0 = F.DivWithBroadCast(
-      F.SubWithBroadCast(tb_xy, pb_xy, shape_half), pb_wh, shape_half);
-  phi::DenseTensor out_wh_0 =
-      F.Log(F.Abs(F.DivWithBroadCast(tb_wh, pb_wh, shape_half)));
-  phi::DenseTensor out_0 = F.Concat({out_xy_0, out_wh_0}, shape_full, 2);
-
-  if (pbv) {
-    F.DivWithBroadCastVoid(out_0, *pbv, shape_full, out);
-  } else {
-    phi::DenseTensor t_var;
-    std::vector<T> vec_var(4);
-    for (auto i = 0; i < 4; i++) {
-      vec_var[i] = static_cast<T>(variance[i]);
-    }
-    Vector2Tensor(ctx, vec_var, phi::make_ddim({1, 1, 4}), &t_var);
-    F.DivWithBroadCastVoid(out_0, t_var, shape_full, out);
-  }
-}
-
-template <typename T>
-void BoxCoderDec(const framework::ExecutionContext& ctx,
-                 const phi::DenseTensor* tb,
-                 const phi::DenseTensor* pb,
-                 const phi::DenseTensor* pbv,
-                 const bool norm,
-                 const std::vector<float>& variance,
-                 int axis,
-                 phi::DenseTensor* out) {
-  auto shape_0 = phi::make_ddim({4, 2});
-  phi::DenseTensor m_diff;
-  phi::DenseTensor m_aver;
-  std::vector<T> vec_diff = {static_cast<T>(-1),
-                             static_cast<T>(0),
-                             static_cast<T>(0),
-                             static_cast<T>(-1),
-                             static_cast<T>(1),
-                             static_cast<T>(0),
-                             static_cast<T>(0),
-                             static_cast<T>(1)};
-  std::vector<T> vec_aver = {static_cast<T>(0.5),
-                             static_cast<T>(0),
-                             static_cast<T>(0),
-                             static_cast<T>(0.5),
-                             static_cast<T>(0.5),
-                             static_cast<T>(0),
-                             static_cast<T>(0),
-                             static_cast<T>(0.5)};
-  Vector2Tensor<T>(ctx, vec_diff, shape_0, &m_diff);
-  Vector2Tensor<T>(ctx, vec_aver, shape_0, &m_aver);
-
-  BoxCoderFunction<T> F(ctx);
-  phi::DenseTensor pb_xy = F.Adds(F.Dot(*pb, m_aver), (norm ? 0 : 0.5));
-  phi::DenseTensor pb_wh = F.Adds(F.Dot(*pb, m_diff), (norm ? 0 : 1));
-  auto pb_resize_shape = axis == 0 ? phi::make_ddim({1, pb->dims()[0], 2})
-                                   : phi::make_ddim({pb->dims()[0], 1, 2});
-  pb_xy.Resize(pb_resize_shape);
-  pb_wh.Resize(pb_resize_shape);
-
-  auto tbox_slice_shape = phi::make_ddim({tb->dims()[0], tb->dims()[1], 2});
-  std::vector<int> tbox_slice_size = {
-      static_cast<int>(tb->dims()[0]), static_cast<int>(tb->dims()[1]), 2};
-  phi::DenseTensor tbox01 =
-      F.Slice(*tb, {0, 0, 0}, tbox_slice_size, tbox_slice_shape);
-  phi::DenseTensor tbox23 =
-      F.Slice(*tb, {0, 0, 2}, tbox_slice_size, tbox_slice_shape);
-
-  phi::DenseTensor tb_xy;
-  phi::DenseTensor tb_wh;
-  if (pbv) {
-    auto pbvt_slice_shape = phi::make_ddim({pbv->dims()[0], 2});
-    auto pbvt_resize_shape = axis == 0 ? phi::make_ddim({1, pbv->dims()[0], 2})
-                                       : phi::make_ddim({pbv->dims()[0], 1, 2});
-    std::vector<int> pbvt_slice_size = {static_cast<int>(pbv->dims()[0]), 2};
-    phi::DenseTensor pbv_t01 =
-        F.Slice(*pbv, {0, 0}, pbvt_slice_size, pbvt_slice_shape);
-    phi::DenseTensor pbv_t23 =
-        F.Slice(*pbv, {0, 2}, pbvt_slice_size, pbvt_slice_shape);
-    pbv_t01.Resize(pbvt_resize_shape);
-    pbv_t23.Resize(pbvt_resize_shape);
-
-    F.AddWithBroadCastVoid(
-        F.MulWithBroadCast(tbox01, F.Mul(pb_wh, pbv_t01), tbox_slice_shape),
-        pb_xy,
-        tbox_slice_shape,
-        &tb_xy);
-    F.MulWithBroadCastVoid(
-        F.Exp(F.MulWithBroadCast(pbv_t23, tbox23, tbox_slice_shape)),
-        pb_wh,
-        tbox_slice_shape,
-        &tb_wh);
-  } else if (variance.empty()) {
-    F.AddWithBroadCastVoid(F.MulWithBroadCast(tbox01, pb_wh, tbox_slice_shape),
-                           pb_xy,
-                           tbox_slice_shape,
-                           &tb_xy);
-    F.MulWithBroadCastVoid(F.Exp(tbox23), pb_wh, tbox_slice_shape, &tb_wh);
-  } else {
-    phi::DenseTensor t_var01, t_var23;
-    auto t_var_shape = phi::make_ddim({1, 1, 2});
-    std::vector<T> vec_var01 = {static_cast<T>(variance[0]),
-                                static_cast<T>(variance[1])};
-    std::vector<T> vec_var23 = {static_cast<T>(variance[2]),
-                                static_cast<T>(variance[3])};
-    Vector2Tensor(ctx, vec_var01, t_var_shape, &t_var01);
-    Vector2Tensor(ctx, vec_var23, t_var_shape, &t_var23);
-    F.AddWithBroadCastVoid(
-        F.MulWithBroadCast(tbox01,
-                           F.MulWithBroadCast(pb_wh, t_var01, pb_resize_shape),
-                           tbox_slice_shape),
-        pb_xy,
-        tbox_slice_shape,
-        &tb_xy);
-    F.MulWithBroadCastVoid(
-        F.Exp(F.MulWithBroadCast(t_var23, tbox23, tbox_slice_shape)),
-        pb_wh,
-        tbox_slice_shape,
-        &tb_wh);
-  }
-  phi::DenseTensor obox01 =
-      F.AddWithBroadCast(tb_xy, F.Muls(tb_wh, -0.5), tbox_slice_shape);
-  phi::DenseTensor obox23 =
-      F.Adds(F.AddWithBroadCast(tb_xy, F.Muls(tb_wh, 0.5), tbox_slice_shape),
-             (norm ? 0 : -1));
-  F.ConcatVoid({obox01, obox23}, out->dims(), 2, out);
-}
-
-template <typename T>
-class BoxCoderNPUKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
-    auto* prior_box = ctx.Input<phi::DenseTensor>("PriorBox");
-    auto* prior_box_var = ctx.Input<phi::DenseTensor>("PriorBoxVar");
-    auto* target_box = ctx.Input<phi::DenseTensor>("TargetBox");
-    auto* output_box = ctx.Output<phi::DenseTensor>("OutputBox");
-    std::vector<float> variance = ctx.Attr<std::vector<float>>("variance");
-    const int axis = ctx.Attr<int>("axis");
-
-    if (prior_box_var) {
-      PADDLE_ENFORCE_EQ(variance.empty(),
-                        true,
-                        platform::errors::InvalidArgument(
-                            "Input 'PriorBoxVar' and attribute 'variance'"
-                            " of BoxCoder operator should not be used at the "
-                            "same time."));
-    }
-    if (!(variance.empty())) {
-      PADDLE_ENFORCE_EQ(static_cast<int>(variance.size()),
-                        4,
-                        platform::errors::InvalidArgument(
-                            "Size of attribute 'variance' in BoxCoder operator"
-                            " should be 4. But received size is %d",
-                            variance.size()));
-    }
-
-    if (target_box->lod().size()) {
-      PADDLE_ENFORCE_EQ(target_box->lod().size(),
-                        1,
-                        platform::errors::InvalidArgument(
-                            "Input 'TargetBox' of BoxCoder operator only"
-                            " supports LoD with one level."));
-    }
-
-    auto code_type =
-        phi::funcs::GetBoxCodeType(ctx.Attr<std::string>("code_type"));
-    bool normalized = ctx.Attr<bool>("box_normalized");
-
-    if (code_type == phi::funcs::BoxCodeType::kEncodeCenterSize) {
-      BoxCoderEnc<T>(ctx,
-                     target_box,
-                     prior_box,
-                     prior_box_var,
-                     normalized,
-                     variance,
-                     output_box);
-    } else {
-      BoxCoderDec<T>(ctx,
-                     target_box,
-                     prior_box,
-                     prior_box_var,
-                     normalized,
-                     variance,
-                     axis,
-                     output_box);
-    }
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-namespace plat = paddle::platform;
-
-REGISTER_OP_NPU_KERNEL(box_coder,
-                       ops::BoxCoderNPUKernel<float>,
-                       ops::BoxCoderNPUKernel<plat::float16>);
diff --git a/paddle/fluid/operators/detection/density_prior_box_op_npu.cc b/paddle/fluid/operators/detection/density_prior_box_op_npu.cc
deleted file mode 100644
index c9935e54d82ef..0000000000000
--- a/paddle/fluid/operators/detection/density_prior_box_op_npu.cc
+++ /dev/null
@@ -1,396 +0,0 @@
-/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-    http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include "paddle/fluid/operators/detection/density_prior_box_op.h"
-
-namespace paddle {
-namespace operators {
-
-using fp16 = paddle::platform::float16;
-
-template <typename T>
-struct DensityPriorBoxFunction {
- public:
-  explicit DensityPriorBoxFunction(const framework::ExecutionContext& ctx)
-      : ctx(ctx) {
-    place = ctx.GetPlace();
-    stream = ctx.template device_context<platform::NPUDeviceContext>().stream();
-    t0.mutable_data<float>({1}, place);
-    t1.mutable_data<float>({1}, place);
-    tn.mutable_data<float>({1}, place);
-    FillNpuTensorWithConstant<float>(&t0, static_cast<float>(0));
-    FillNpuTensorWithConstant<float>(&t1, static_cast<float>(1));
-  }
-  void Arange(int n, phi::DenseTensor* x) {
-    //  x should be init first
-    FillNpuTensorWithConstant<float>(&tn, static_cast<float>(n));
-    const auto& runner = NpuOpRunner("Range", {t0, tn, t1}, {*x}, {});
-    runner.Run(stream);
-  }
-  void Add(const phi::DenseTensor* x,
-           const phi::DenseTensor* y,
-           phi::DenseTensor* z) {
-    //  z should be init first
-    const auto& runner = NpuOpRunner("AddV2", {*x, *y}, {*z}, {});
-    runner.Run(stream);
-  }
-  void Cast(const phi::DenseTensor* x, phi::DenseTensor* y) {
-    auto dst_dtype =
-        ConvertToNpuDtype(framework::TransToProtoVarType(y->type()));
-    const auto& runner = NpuOpRunner(
-        "Cast", {*x}, {*y}, {{"dst_type", static_cast<int>(dst_dtype)}});
-    runner.Run(stream);
-  }
-  void Sub(const phi::DenseTensor* x,
-           const phi::DenseTensor* y,
-           phi::DenseTensor* z) {
-    //  z should be init first
-    const auto& runner = NpuOpRunner("Sub", {*x, *y}, {*z}, {});
-    runner.Run(stream);
-  }
-  void Mul(const phi::DenseTensor* x,
-           const phi::DenseTensor* y,
-           phi::DenseTensor* z) {
-    //  y should be init first
-    const auto& runner = NpuOpRunner("Mul", {*x, *y}, {*z}, {});
-    runner.Run(stream);
-  }
-  void Adds(const phi::DenseTensor* x, float scalar, phi::DenseTensor* y) {
-    //  y should be init first
-    const auto& runner = NpuOpRunner("Adds", {*x}, {*y}, {{"value", scalar}});
-    runner.Run(stream);
-  }
-  void Muls(const phi::DenseTensor* x, float scalar, phi::DenseTensor* y) {
-    //  y should be init first
-    const auto& runner = NpuOpRunner("Muls", {*x}, {*y}, {{"value", scalar}});
-    runner.Run(stream);
-  }
-  void Maximum(const phi::DenseTensor* x,
-               const phi::DenseTensor* y,
-               phi::DenseTensor* z) {
-    //  y should be init first
-    const auto& runner = NpuOpRunner("Maximum", {*x, *y}, {*z}, {});
-    runner.Run(stream);
-  }
-  void Minimum(const phi::DenseTensor* x,
-               const phi::DenseTensor* y,
-               phi::DenseTensor* z) {
-    //  y should be init first
-    const auto& runner = NpuOpRunner("Minimum", {*x, *y}, {*z}, {});
-    runner.Run(stream);
-  }
-  void Concat(const std::vector<phi::DenseTensor>& inputs,
-              int axis,
-              phi::DenseTensor* output) {
-    //  output should be init first
-    std::vector<std::string> names;
-    for (size_t i = 0; i < inputs.size(); i++) {
-      names.push_back("x" + std::to_string(i));
-    }
-    NpuOpRunner runner{
-        "ConcatD",
-        {inputs},
-        {*output},
-        {{"concat_dim", axis}, {"N", static_cast<int>(inputs.size())}}};
-    runner.AddInputNames(names);
-    runner.Run(stream);
-  }
-  void Tile(const phi::DenseTensor* x,
-            phi::DenseTensor* y,
-            const std::vector<int>& multiples) {
-    //  y should be init first
-    if (x->dims() == y->dims()) {
-      framework::TensorCopy(
-          *x,
-          place,
-          ctx.template device_context<platform::NPUDeviceContext>(),
-          y);
-      return;
-    }
-    const auto& runner =
-        NpuOpRunner("TileD", {*x}, {*y}, {{"multiples", multiples}});
-    runner.Run(stream);
-  }
-  void FloatVec2Tsr(const std::vector<float>& vec, phi::DenseTensor* tsr_dst) {
-    //
-    framework::TensorFromVector<T>(vec, ctx.device_context(), tsr_dst);
-    ctx.template device_context<platform::NPUDeviceContext>().Wait();
-  }
-
- private:
-  platform::Place place;
-  aclrtStream stream;
-  const framework::ExecutionContext& ctx;
-  phi::DenseTensor t0;
-  phi::DenseTensor t1;
-  phi::DenseTensor tn;
-};
-
-template <>
-void DensityPriorBoxFunction<fp16>::Arange(int n, phi::DenseTensor* x) {
-  phi::DenseTensor x_fp32(phi::DataType::FLOAT32);
-  x_fp32.mutable_data<float>(x->dims(), place);
-  FillNpuTensorWithConstant<float>(&tn, static_cast<float>(n));
-  const auto& runner = NpuOpRunner("Range", {t0, tn, t1}, {x_fp32}, {});
-  runner.Run(stream);
-  Cast(&x_fp32, x);
-}
-
-template <>
-void DensityPriorBoxFunction<fp16>::FloatVec2Tsr(const std::vector<float>& vec,
-                                                 phi::DenseTensor* tsr_dst) {
-  phi::DenseTensor tsr_fp32(phi::DataType::FLOAT32);
-  tsr_fp32.mutable_data<float>(tsr_dst->dims(), place);
-  framework::TensorFromVector<float>(vec, ctx.device_context(), &tsr_fp32);
-  ctx.template device_context<paddle::platform::NPUDeviceContext>().Wait();
-  Cast(&tsr_fp32, tsr_dst);
-}
-
-template <typename T>
-class DensityPriorBoxOpNPUKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
-    auto* input = ctx.Input<phi::DenseTensor>("Input");
-    auto* image = ctx.Input<phi::DenseTensor>("Image");
-    auto* boxes = ctx.Output<phi::DenseTensor>("Boxes");
-    auto* vars = ctx.Output<phi::DenseTensor>("Variances");
-
-    auto variances = ctx.Attr<std::vector<float>>("variances");
-    auto clip = ctx.Attr<bool>("clip");
-
-    auto fixed_sizes = ctx.Attr<std::vector<float>>("fixed_sizes");
-    auto fixed_ratios = ctx.Attr<std::vector<float>>("fixed_ratios");
-    auto densities = ctx.Attr<std::vector<int>>("densities");
-
-    float step_w = ctx.Attr<float>("step_w");
-    float step_h = ctx.Attr<float>("step_h");
-    float offset = ctx.Attr<float>("offset");
-
-    int image_w = image->dims()[3];
-    int image_h = image->dims()[2];
-    int layer_w = input->dims()[3];
-    int layer_h = input->dims()[2];
-
-    auto _type = input->dtype();
-    auto place = ctx.GetPlace();
-    DensityPriorBoxFunction<T> F(ctx);
-
-    phi::DenseTensor h(_type);
-    h.mutable_data<T>({layer_h}, place);
-    phi::DenseTensor w(_type);
-    w.mutable_data<T>({layer_w}, place);
-    F.Arange(layer_h, &h);
-    F.Arange(layer_w, &w);
-    h.Resize({layer_h, 1, 1, 1});
-    w.Resize({1, layer_w, 1, 1});
-
-    step_w = step_w > 0 ? step_w : static_cast<float>(image_w) / layer_w;
-    step_h = step_h > 0 ? step_h : static_cast<float>(image_h) / layer_h;
-    int step_average = static_cast<int>((step_w + step_h) * 0.5);
-
-    int ratios_size = fixed_ratios.size();
-    int num_priors_per_ratio = 0;
-    for (size_t i = 0; i < densities.size(); ++i) {
-      num_priors_per_ratio += densities[i] * densities[i];
-    }
-    phi::DenseTensor di(_type);
-    phi::DenseTensor dj(_type);
-    phi::DenseTensor shifts(_type);
-    phi::DenseTensor box_w_ratio(_type);
-    phi::DenseTensor box_h_ratio(_type);
-    di.mutable_data<T>({ratios_size * num_priors_per_ratio}, place);
-    dj.mutable_data<T>({ratios_size * num_priors_per_ratio}, place);
-    shifts.mutable_data<T>({ratios_size * num_priors_per_ratio}, place);
-    box_w_ratio.mutable_data<T>({ratios_size * num_priors_per_ratio}, place);
-    box_h_ratio.mutable_data<T>({ratios_size * num_priors_per_ratio}, place);
-
-    int64_t start = 0;
-    std::vector<int> vec_tile = {0, 0, 0};
-    for (size_t i = 0; i < densities.size(); ++i) {
-      //  Range = start:start+ratios_size*density_sqr, density = densities[i]
-      int density_sqr = densities[i] * densities[i];
-      //  shifts[Range] = [step_average/density]*ratios_size*density_sqr
-      phi::DenseTensor shifts_part =
-          shifts.Slice(start, start + ratios_size * density_sqr);
-      FillNpuTensorWithConstant<T>(&shifts_part,
-                                   static_cast<T>(step_average / densities[i]));
-
-      //  di[Range] = [ i // density for i in range(density_sqr) ] * ratios_size
-      //  dj[Range] = [ i % density for i in range(density_sqr) ] * ratios_size
-      phi::DenseTensor di_part =
-          di.Slice(start, start + ratios_size * density_sqr);
-      phi::DenseTensor dj_part =
-          dj.Slice(start, start + ratios_size * density_sqr);
-      if (densities[i] > 1) {
-        di_part.Resize({ratios_size, densities[i], densities[i]});
-        dj_part.Resize({ratios_size, densities[i], densities[i]});
-        phi::DenseTensor range_n(_type);
-        range_n.mutable_data<T>({densities[i]}, place);
-        F.Arange(densities[i], &range_n);
-        range_n.Resize({1, densities[i], 1});
-        vec_tile[0] = ratios_size;
-        vec_tile[1] = 1;
-        vec_tile[2] = densities[i];
-        F.Tile(&range_n, &di_part, vec_tile);
-        range_n.Resize({1, 1, densities[i]});
-        vec_tile[1] = densities[i];
-        vec_tile[2] = 1;
-        F.Tile(&range_n, &dj_part, vec_tile);
-      } else {
-        FillNpuTensorWithConstant<T>(&di_part, static_cast<T>(0));
-        FillNpuTensorWithConstant<T>(&dj_part, static_cast<T>(0));
-      }
-
-      int start_box_ratio = start;
-      for (float ar : fixed_ratios) {
-        //  Range_mini = start_box_ratio:start_box_ratio+density_sqr
-        //  box_h_ratio[Range_mini] = [fixed_sizes[i] * sqrt(ar)]  * density_sqr
-        //  box_w_ratio[Range_mini] = [fixed_sizes[i] / sqrt(ar)]  * density_sqr
-        phi::DenseTensor box_h_ratio_part =
-            box_h_ratio.Slice(start_box_ratio, start_box_ratio + density_sqr);
-        phi::DenseTensor box_w_ratio_part =
-            box_w_ratio.Slice(start_box_ratio, start_box_ratio + density_sqr);
-        FillNpuTensorWithConstant<T>(&box_w_ratio_part,
-                                     static_cast<T>(fixed_sizes[i] * sqrt(ar)));
-        FillNpuTensorWithConstant<T>(&box_h_ratio_part,
-                                     static_cast<T>(fixed_sizes[i] / sqrt(ar)));
-        start_box_ratio += density_sqr;
-      }
-      start = start_box_ratio;
-    }
-    di.Resize({1, 1, ratios_size * num_priors_per_ratio, 1});
-    dj.Resize({1, 1, ratios_size * num_priors_per_ratio, 1});
-    shifts.Resize({1, 1, ratios_size * num_priors_per_ratio, 1});
-    box_w_ratio.Resize({1, 1, ratios_size * num_priors_per_ratio, 1});
-    box_h_ratio.Resize({1, 1, ratios_size * num_priors_per_ratio, 1});
-
-    //  c_x = (w+offset)*step_w - 0.5*step_average + 0.5*shifts + dj*shifts
-    //  c_y = (h+offset)*step_h - 0.5*step_average + 0.5*shifts + di*shifts
-    phi::DenseTensor c_x(_type);
-    phi::DenseTensor c_y(_type);
-    auto dim0 =
-        phi::make_ddim({1, layer_w, ratios_size * num_priors_per_ratio, 1});
-    auto dim1 =
-        phi::make_ddim({layer_h, 1, ratios_size * num_priors_per_ratio, 1});
-    c_x.mutable_data<T>(dim0, place);
-    c_y.mutable_data<T>(dim1, place);
-    F.Adds(&w, offset, &w);
-    F.Muls(&w, step_w, &w);
-    F.Adds(&w, static_cast<float>(-step_average) * static_cast<float>(0.5), &w);
-    F.Adds(&h, offset, &h);
-    F.Muls(&h, step_h, &h);
-    F.Adds(&h, static_cast<float>(-step_average) * static_cast<float>(0.5), &h);
-    F.Mul(&di, &shifts, &di);
-    F.Mul(&dj, &shifts, &dj);
-    F.Muls(&shifts, static_cast<float>(0.5), &shifts);
-    F.Add(&di, &shifts, &di);
-    F.Add(&dj, &shifts, &dj);
-    F.Add(&dj, &w, &c_x);
-    F.Add(&di, &h, &c_y);
-
-    //  box_w_ratio = box_w_ratio / 2
-    //  box_h_ratio = box_h_ratio / 2
-    F.Muls(&box_w_ratio, static_cast<float>(0.5), &box_w_ratio);
-    F.Muls(&box_h_ratio, static_cast<float>(0.5), &box_h_ratio);
-
-    phi::DenseTensor zero_t(_type);
-    phi::DenseTensor one_t(_type);
-    zero_t.mutable_data<T>({1}, place);
-    one_t.mutable_data<T>({1}, place);
-    FillNpuTensorWithConstant<T>(&zero_t, static_cast<T>(0));
-    FillNpuTensorWithConstant<T>(&one_t, static_cast<T>(1));
-
-    phi::DenseTensor outbox0(_type);
-    phi::DenseTensor outbox1(_type);
-    phi::DenseTensor outbox2(_type);
-    phi::DenseTensor outbox3(_type);
-    outbox0.mutable_data<T>(dim0, place);
-    outbox1.mutable_data<T>(dim1, place);
-    outbox2.mutable_data<T>(dim0, place);
-    outbox3.mutable_data<T>(dim1, place);
-
-    //  outbox0 = max ( (c_x - box_w_ratio)/image_w, 0 )
-    //  outbox1 = max ( (c_y - box_h_ratio)/image_h, 0 )
-    //  outbox2 = min ( (c_x + box_w_ratio)/image_w, 1 )
-    //  outbox3 = min ( (c_y + box_h_ratio)/image_h, 1 )
-    F.Sub(&c_x, &box_w_ratio, &outbox0);
-    F.Sub(&c_y, &box_h_ratio, &outbox1);
-    F.Add(&c_x, &box_w_ratio, &outbox2);
-    F.Add(&c_y, &box_h_ratio, &outbox3);
-    F.Muls(&outbox0, static_cast<float>(1.0 / image_w), &outbox0);
-    F.Muls(&outbox1, static_cast<float>(1.0 / image_h), &outbox1);
-    F.Muls(&outbox2, static_cast<float>(1.0 / image_w), &outbox2);
-    F.Muls(&outbox3, static_cast<float>(1.0 / image_h), &outbox3);
-
-    F.Maximum(&outbox0, &zero_t, &outbox0);
-    F.Maximum(&outbox1, &zero_t, &outbox1);
-    F.Minimum(&outbox2, &one_t, &outbox2);
-    F.Minimum(&outbox3, &one_t, &outbox3);
-    if (clip) {
-      //  outbox0 = min ( outbox0, 1 )
-      //  outbox1 = min ( outbox1, 1 )
-      //  outbox2 = max ( outbox2, 0 )
-      //  outbox3 = max ( outbox3, 0 )
-      F.Minimum(&outbox0, &one_t, &outbox0);
-      F.Minimum(&outbox1, &one_t, &outbox1);
-      F.Maximum(&outbox2, &zero_t, &outbox2);
-      F.Maximum(&outbox3, &zero_t, &outbox3);
-    }
-
-    auto out_dim = phi::make_ddim(
-        {layer_h, layer_w, ratios_size * num_priors_per_ratio, 4});
-    boxes->mutable_data<T>(place);
-    vars->mutable_data<T>(place);
-    phi::DenseTensor boxes_share(_type);
-    phi::DenseTensor vars_share(_type);
-    boxes_share.ShareDataWith(*boxes);
-    boxes_share.Resize(out_dim);
-    vars_share.ShareDataWith(*vars);
-    vars_share.Resize(out_dim);
-
-    phi::DenseTensor box0(_type);
-    phi::DenseTensor box1(_type);
-    phi::DenseTensor box2(_type);
-    phi::DenseTensor box3(_type);
-    // out_dim = {layer_h, layer_w, ratios_size*num_priors_per_ratio, 1}
-    out_dim[3] = 1;
-    box0.mutable_data<T>(out_dim, place);
-    box1.mutable_data<T>(out_dim, place);
-    box2.mutable_data<T>(out_dim, place);
-    box3.mutable_data<T>(out_dim, place);
-
-    std::vector<int> vec_exp_out02 = {layer_h, 1, 1, 1};
-    std::vector<int> vec_exp_out13 = {1, layer_w, 1, 1};
-    F.Tile(&outbox0, &box0, vec_exp_out02);
-    F.Tile(&outbox1, &box1, vec_exp_out13);
-    F.Tile(&outbox2, &box2, vec_exp_out02);
-    F.Tile(&outbox3, &box3, vec_exp_out13);
-    F.Concat({box0, box1, box2, box3}, 3, &boxes_share);
-
-    std::vector<int> multiples = {
-        layer_h, layer_w, ratios_size * num_priors_per_ratio, 1};
-    phi::DenseTensor variances_t(_type);
-    //  variances.size() == 4
-    variances_t.mutable_data<T>({4}, place);
-    F.FloatVec2Tsr(variances, &variances_t);
-    F.Tile(&variances_t, &vars_share, multiples);
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-namespace plat = paddle::platform;
-
-REGISTER_OP_NPU_KERNEL(density_prior_box,
-                       ops::DensityPriorBoxOpNPUKernel<plat::float16>,
-                       ops::DensityPriorBoxOpNPUKernel<float>);
diff --git a/paddle/fluid/operators/detection/iou_similarity_op_npu.cc b/paddle/fluid/operators/detection/iou_similarity_op_npu.cc
deleted file mode 100644
index 8395e25d46251..0000000000000
--- a/paddle/fluid/operators/detection/iou_similarity_op_npu.cc
+++ /dev/null
@@ -1,204 +0,0 @@
-/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include "paddle/fluid/operators/detection/iou_similarity_op.h"
-
-namespace paddle {
-namespace operators {
-
-template <typename T>
-struct IouFunction {
- public:
-  explicit IouFunction(const framework::ExecutionContext& ctx) : ctx(ctx) {
-    place = ctx.GetPlace();
-    stream = ctx.template device_context<paddle::platform::NPUDeviceContext>()
-                 .stream();
-  }
-  void Transpose(const phi::DenseTensor* x,
-                 phi::DenseTensor* y,
-                 const std::vector<int>& axis) {
-    //  y should be init first
-    const auto& runner =
-        NpuOpRunner("TransposeD", {*x}, {*y}, {{"perm", axis}});
-    runner.Run(stream);
-  }
-  void Add(const phi::DenseTensor* x,
-           const phi::DenseTensor* y,
-           phi::DenseTensor* z) {
-    //  y should be init first
-    const auto& runner = NpuOpRunner("AddV2", {*x, *y}, {*z}, {});
-    runner.Run(stream);
-  }
-  void Sub(const phi::DenseTensor* x,
-           const phi::DenseTensor* y,
-           phi::DenseTensor* z) {
-    //  y should be init first
-    const auto& runner = NpuOpRunner("Sub", {*x, *y}, {*z}, {});
-    runner.Run(stream);
-  }
-  void Mul(const phi::DenseTensor* x,
-           const phi::DenseTensor* y,
-           phi::DenseTensor* z) {
-    //  y should be init first
-    const auto& runner = NpuOpRunner("Mul", {*x, *y}, {*z}, {});
-    runner.Run(stream);
-  }
-  void DivNoNan(const phi::DenseTensor* x,
-                const phi::DenseTensor* y,
-                phi::DenseTensor* z) {
-    //  y should be init first
-    const auto& runner = NpuOpRunner("DivNoNan", {*x, *y}, {*z}, {});
-    runner.Run(stream);
-  }
-  void Adds(const phi::DenseTensor* x, float scalar, phi::DenseTensor* y) {
-    //  y should be init first
-    const auto& runner = NpuOpRunner("Adds", {*x}, {*y}, {{"value", scalar}});
-    runner.Run(stream);
-  }
-  void Maximum(const phi::DenseTensor* x,
-               const phi::DenseTensor* y,
-               phi::DenseTensor* z) {
-    //  z should be init first
-    const auto& runner = NpuOpRunner("Maximum", {*x, *y}, {*z}, {});
-    runner.Run(stream);
-  }
-  void Minimum(const phi::DenseTensor* x,
-               const phi::DenseTensor* y,
-               phi::DenseTensor* z) {
-    //  z should be init first
-    const auto& runner = NpuOpRunner("Minimum", {*x, *y}, {*z}, {});
-    runner.Run(stream);
-  }
-
- private:
-  platform::Place place;
-  aclrtStream stream;
-  const framework::ExecutionContext& ctx;
-};
-
-template <typename T>
-class IouSimilarityNPUKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
-    auto* x = ctx.Input<phi::DenseTensor>("X");
-    auto* y = ctx.Input<phi::DenseTensor>("Y");
-    bool normalized = ctx.Attr<bool>("box_normalized");
-    auto* out = ctx.Output<phi::DenseTensor>("Out");
-
-    auto _type = x->dtype();
-    auto place = ctx.GetPlace();
-
-    IouFunction<T> F(ctx);
-
-    auto N = x->dims()[0];
-    auto M = y->dims()[0];
-
-    out->mutable_data<T>({N, M}, place);
-    phi::DenseTensor xt(_type);
-    phi::DenseTensor yt(_type);
-    xt.mutable_data<T>({4, N}, place);
-    yt.mutable_data<T>({4, M}, place);
-    std::vector<int> vec_trans = {1, 0};
-    F.Transpose(x, &xt, vec_trans);
-    F.Transpose(y, &yt, vec_trans);
-    phi::DenseTensor xmin1 = xt.Slice(0, 1);
-    phi::DenseTensor ymin1 = xt.Slice(1, 2);
-    phi::DenseTensor xmax1 = xt.Slice(2, 3);
-    phi::DenseTensor ymax1 = xt.Slice(3, 4);
-    phi::DenseTensor xmin2 = yt.Slice(0, 1);
-    phi::DenseTensor ymin2 = yt.Slice(1, 2);
-    phi::DenseTensor xmax2 = yt.Slice(2, 3);
-    phi::DenseTensor ymax2 = yt.Slice(3, 4);
-    xmin1.Resize({N, 1});
-    ymin1.Resize({N, 1});
-    xmax1.Resize({N, 1});
-    ymax1.Resize({N, 1});
-    xmin2.Resize({1, M});
-    ymin2.Resize({1, M});
-    xmax2.Resize({1, M});
-    ymax2.Resize({1, M});
-
-    phi::DenseTensor w1(_type);
-    phi::DenseTensor h1(_type);
-    phi::DenseTensor w2(_type);
-    phi::DenseTensor h2(_type);
-    phi::DenseTensor area1(_type);
-    phi::DenseTensor area2(_type);
-    w1.mutable_data<T>({N, 1}, place);
-    h1.mutable_data<T>({N, 1}, place);
-    w2.mutable_data<T>({1, M}, place);
-    h2.mutable_data<T>({1, M}, place);
-    area1.mutable_data<T>({N, 1}, place);
-    area2.mutable_data<T>({1, M}, place);
-    F.Sub(&xmax1, &xmin1, &w1);
-    F.Sub(&ymax1, &ymin1, &h1);
-    F.Sub(&xmax2, &xmin2, &w2);
-    F.Sub(&ymax2, &ymin2, &h2);
-    if (!normalized) {
-      F.Adds(&w1, 1.0f, &w1);
-      F.Adds(&h1, 1.0f, &h1);
-      F.Adds(&w2, 1.0f, &w2);
-      F.Adds(&h2, 1.0f, &h2);
-    }
-    F.Mul(&w1, &h1, &area1);
-    F.Mul(&w2, &h2, &area2);
-
-    phi::DenseTensor inter_xmax(_type);
-    phi::DenseTensor inter_ymax(_type);
-    phi::DenseTensor inter_xmin(_type);
-    phi::DenseTensor inter_ymin(_type);
-    inter_xmax.mutable_data<T>({N, M}, place);
-    inter_ymax.mutable_data<T>({N, M}, place);
-    inter_xmin.mutable_data<T>({N, M}, place);
-    inter_ymin.mutable_data<T>({N, M}, place);
-    F.Minimum(&xmax1, &xmax2, &inter_xmax);
-    F.Minimum(&ymax1, &ymax2, &inter_ymax);
-    F.Maximum(&xmin1, &xmin2, &inter_xmin);
-    F.Maximum(&ymin1, &ymin2, &inter_ymin);
-
-    phi::DenseTensor inter_w(_type);
-    phi::DenseTensor inter_h(_type);
-    inter_w.mutable_data<T>({N, M}, place);
-    inter_h.mutable_data<T>({N, M}, place);
-    F.Sub(&inter_xmax, &inter_xmin, &inter_w);
-    F.Sub(&inter_ymax, &inter_ymin, &inter_h);
-
-    if (!normalized) {
-      F.Adds(&inter_w, 1.0f, &inter_w);
-      F.Adds(&inter_h, 1.0f, &inter_h);
-    }
-    phi::DenseTensor zeros(_type);
-    zeros.mutable_data<T>({1}, place);
-    FillNpuTensorWithConstant<T>(&zeros, static_cast<T>(0));
-    F.Maximum(&inter_w, &zeros, &inter_w);
-    F.Maximum(&inter_h, &zeros, &inter_h);
-
-    F.Mul(&inter_w, &inter_h, out);
-    phi::DenseTensor union_area(_type);
-    union_area.mutable_data<T>({N, M}, place);
-    F.Add(&area1, &area2, &union_area);
-    F.Sub(&union_area, out, &union_area);
-    F.DivNoNan(out, &union_area, out);
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-namespace plat = paddle::platform;
-
-REGISTER_OP_NPU_KERNEL(iou_similarity,
-                       ops::IouSimilarityNPUKernel<float>,
-                       ops::IouSimilarityNPUKernel<plat::float16>);
diff --git a/paddle/fluid/operators/detection/prior_box_op_npu.cc b/paddle/fluid/operators/detection/prior_box_op_npu.cc
deleted file mode 100644
index 7df68d2bbb1bb..0000000000000
--- a/paddle/fluid/operators/detection/prior_box_op_npu.cc
+++ /dev/null
@@ -1,106 +0,0 @@
-/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include "paddle/fluid/operators/detection/prior_box_op.h"
-
-namespace paddle {
-namespace operators {
-
-template <typename DeviceContext, typename T>
-class PriorBoxNPUKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
-    auto* input = ctx.Input<phi::DenseTensor>("Input");
-    auto* image = ctx.Input<phi::DenseTensor>("Image");
-    auto* boxes = ctx.Output<phi::DenseTensor>("Boxes");
-    auto* variances = ctx.Output<phi::DenseTensor>("Variances");
-
-    PADDLE_ENFORCE_EQ(boxes->dims(),
-                      variances->dims(),
-                      platform::errors::Unimplemented(
-                          "the shape of boxes and variances must be same in "
-                          "the npu kernel of prior_box, but got boxes->dims() "
-                          "= [%s], variances->dims() = [%s]",
-                          boxes->dims(),
-                          variances->dims()));
-
-    auto min_sizes = ctx.Attr<std::vector<float>>("min_sizes");
-    auto max_sizes = ctx.Attr<std::vector<float>>("max_sizes");
-    auto aspect_ratios = ctx.Attr<std::vector<float>>("aspect_ratios");
-    auto variances_attr = ctx.Attr<std::vector<float>>("variances");
-    bool flip = ctx.Attr<bool>("flip");
-    bool clip = ctx.Attr<bool>("clip");
-    float step_w = ctx.Attr<float>("step_w");
-    float step_h = ctx.Attr<float>("step_h");
-    float offset = ctx.Attr<float>("offset");
-
-    auto place = ctx.GetPlace();
-
-    phi::DenseTensor out(input->type());
-    auto out_dims = phi::vectorize(boxes->dims());
-    out_dims.insert(out_dims.begin(), 2);
-    out.Resize(phi::make_ddim(out_dims));
-    out.mutable_data<T>(place);
-
-    framework::NPUAttributeMap attr_input = {{"min_size", min_sizes},
-                                             {"max_size", max_sizes},
-                                             {"aspect_ratio", aspect_ratios},
-                                             {"step_h", step_h},
-                                             {"step_w", step_w},
-                                             {"flip", flip},
-                                             {"clip", clip},
-                                             {"offset", offset},
-                                             {"variance", variances_attr}};
-
-    auto stream =
-        ctx.template device_context<paddle::platform::NPUDeviceContext>()
-            .stream();
-
-    const auto& runner =
-        NpuOpRunner("PriorBox", {*input, *image}, {out}, attr_input);
-    runner.Run(stream);
-
-    out.Resize(phi::make_ddim({out.numel()}));
-    phi::DenseTensor out_boxes = out.Slice(0, boxes->numel());
-    phi::DenseTensor out_variances = out.Slice(boxes->numel(), out.numel());
-
-    out_boxes.Resize(boxes->dims());
-    out_variances.Resize(variances->dims());
-
-    boxes->mutable_data<T>(place);
-    variances->mutable_data<T>(place);
-
-    framework::TensorCopy(
-        out_boxes,
-        place,
-        ctx.template device_context<platform::NPUDeviceContext>(),
-        boxes);
-    framework::TensorCopy(
-        out_variances,
-        place,
-        ctx.template device_context<platform::NPUDeviceContext>(),
-        variances);
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-namespace plat = paddle::platform;
-
-REGISTER_OP_NPU_KERNEL(
-    prior_box,
-    ops::PriorBoxNPUKernel<plat::NPUDeviceContext, float>,
-    ops::PriorBoxNPUKernel<plat::NPUDeviceContext, plat::float16>);

From 23e96bdec0484a34a5b1b325932bac0a9b733360 Mon Sep 17 00:00:00 2001
From: engineer1109 <jialiang.wang@xdxct.com>
Date: Wed, 26 Apr 2023 11:46:37 +0800
Subject: [PATCH 075/405] [Debug][Werror]error: control reaches end of non-void
 function [-Werror=return-type](#53326)

---
 .../fluid/prim/utils/static/composite_grad_desc_maker.h  | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h b/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h
index eb13902be2068..1ef5161dc0714 100644
--- a/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h
+++ b/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h
@@ -405,15 +405,6 @@ class CompositeGradOpMakerBase {
       }
     }
     return input_grads;
-    PADDLE_ENFORCE_LE(
-        var_names.size(),
-        1UL,
-        platform::errors::Unavailable(
-            "BUG from operator developer:"
-            " for input argument with a list of variables, "
-            " drop_empty_grad is not allowed because it makes"
-            " the correspondence bewteen a variable and its gradient"
-            " ambiguous."));
   }
 
   std::vector<framework::VarDesc*> MultiOutputGrad(

From 66fbfba8ed5c4ac96fbf9634461a7c5a10017645 Mon Sep 17 00:00:00 2001
From: zhuyipin <yipinzhu@outlook.com>
Date: Wed, 26 Apr 2023 13:13:56 +0800
Subject: [PATCH 076/405] add leaky relu composite rule (#52909)

* add leaky relu composite rule

* add public python api

* unset default negative slope

* fix unittest case
---
 .../tests/unittests/test_activation_op.py     | 57 +++++++++++++++----
 .../incubate/autograd/composite_rules.py      |  9 +++
 2 files changed, 55 insertions(+), 11 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/test_activation_op.py b/python/paddle/fluid/tests/unittests/test_activation_op.py
index dfa95f760ce6a..c273cd4954941 100644
--- a/python/paddle/fluid/tests/unittests/test_activation_op.py
+++ b/python/paddle/fluid/tests/unittests/test_activation_op.py
@@ -1933,6 +1933,8 @@ def get_alpha(self):
     def setUp(self):
         self.op_type = "leaky_relu"
         self.python_api = paddle.nn.functional.leaky_relu
+        self.public_python_api = paddle.nn.functional.leaky_relu
+        self.prim_op_type = "comp"
         self.init_dtype()
         self.init_shape()
         alpha = self.get_alpha()
@@ -1948,10 +1950,13 @@ def setUp(self):
         self.attrs = {'alpha': alpha}
         self.convert_input_output()
 
+    def test_check_output(self):
+        self.check_output(check_prim=True)
+
     def test_check_grad(self):
         if self.dtype == np.float16:
             return
-        self.check_grad(['X'], 'Out')
+        self.check_grad(['X'], 'Out', check_prim=True)
 
 
 class TestLeakyReluAlpha1(TestLeakyRelu):
@@ -1973,6 +1978,26 @@ class TestLeakyRelu_ZeroDim(TestLeakyRelu):
     def init_shape(self):
         self.shape = []
 
+    def setUp(self):
+        self.op_type = "leaky_relu"
+        self.prim_op_type = "comp"
+        self.enable_cinn = False
+        self.python_api = paddle.nn.functional.leaky_relu
+        self.public_python_api = paddle.nn.functional.relu
+        self.init_dtype()
+        self.init_shape()
+        alpha = self.get_alpha()
+
+        np.random.seed(1024)
+        x = np.random.uniform(-1, 1, self.shape).astype(self.dtype)
+        # The same reason with TestAbs
+        x[np.abs(x) < 0.005] = 0.05
+        out = ref_leaky_relu(x, alpha)
+
+        self.inputs = {'X': x}
+        self.outputs = {'Out': out}
+        self.attrs = {'alpha': alpha}
+
 
 class TestLeakyReluAPI(unittest.TestCase):
     # test paddle.nn.LeakyReLU, paddle.nn.functional.leaky_relu,
@@ -4031,11 +4056,13 @@ def test_check_grad(self):
 create_test_act_fp16_class(TestSwish)
 create_test_act_fp16_class(TestHardSwish, check_prim=True)
 create_test_act_fp16_class(TestMish)
-create_test_act_fp16_class(TestLeakyRelu)
-create_test_act_fp16_class(TestLeakyReluAlpha1)
-create_test_act_fp16_class(TestLeakyReluAlpha2)
-create_test_act_fp16_class(TestLeakyReluAlpha3)
-create_test_act_fp16_class(TestLeakyRelu_ZeroDim)
+create_test_act_fp16_class(TestLeakyRelu, check_prim=True)
+create_test_act_fp16_class(TestLeakyReluAlpha1, check_prim=True)
+create_test_act_fp16_class(TestLeakyReluAlpha2, check_prim=True)
+create_test_act_fp16_class(TestLeakyReluAlpha3, check_prim=True)
+create_test_act_fp16_class(
+    TestLeakyRelu_ZeroDim, check_prim=True, enable_cinn=False
+)
 create_test_act_fp16_class(TestRsqrt)
 
 
@@ -4142,11 +4169,19 @@ def test_check_grad(self):
 create_test_act_bf16_class(TestSwish)
 create_test_act_bf16_class(TestHardSwish, check_prim=True)
 create_test_act_bf16_class(TestMish)
-create_test_act_bf16_class(TestLeakyRelu)
-create_test_act_bf16_class(TestLeakyReluAlpha1)
-create_test_act_bf16_class(TestLeakyReluAlpha2)
-create_test_act_bf16_class(TestLeakyReluAlpha3)
-create_test_act_bf16_class(TestLeakyRelu_ZeroDim)
+create_test_act_bf16_class(TestLeakyRelu, check_prim=True, enable_cinn=False)
+create_test_act_bf16_class(
+    TestLeakyReluAlpha1, check_prim=True, enable_cinn=False
+)
+create_test_act_bf16_class(
+    TestLeakyReluAlpha2, check_prim=True, enable_cinn=False
+)
+create_test_act_bf16_class(
+    TestLeakyReluAlpha3, check_prim=True, enable_cinn=False
+)
+create_test_act_bf16_class(
+    TestLeakyRelu_ZeroDim, check_prim=True, enable_cinn=False
+)
 create_test_act_bf16_class(TestRsqrt)
 
 if __name__ == "__main__":
diff --git a/python/paddle/incubate/autograd/composite_rules.py b/python/paddle/incubate/autograd/composite_rules.py
index 72bc1601bfacc..3a1a3ea7d6751 100644
--- a/python/paddle/incubate/autograd/composite_rules.py
+++ b/python/paddle/incubate/autograd/composite_rules.py
@@ -677,3 +677,12 @@ def group_norm_composite(x, scale, bias, epsilon, groups, data_layout):
     if is_amp:
         out = cast(out, "float16")
     return out, ret_mean_, ret_var_
+
+
+@REGISTER_COMPOSITE('leaky_relu')
+def leaky_relu_composite(x, negative_slope):
+    """define composite rule of op leaky_relu."""
+    if negative_slope < 1.0:
+        return maximum(x, negative_slope * x)
+    else:
+        return minimum(x, negative_slope * x)

From 35f5c245654f9e995e3d60dd6e603f696cbdae79 Mon Sep 17 00:00:00 2001
From: sneaxiy <32832641+sneaxiy@users.noreply.github.com>
Date: Wed, 26 Apr 2023 13:25:34 +0800
Subject: [PATCH 077/405] Optimize c_embedding op in deterministic mode
 (#53197)

* optimize embedding deterministic mode

* fix compile error

* change FLAGS_cudnn_deterministic to int64

* fix 700 error

* add ut

* fix ut

* fix ut

* fix win32 ci

* fix flags with PHI_DEFINE_EXPORTED_int64
---
 .../operators/collective/c_embedding_op.cu    |  87 ++++---
 paddle/phi/core/flags.cc                      |  12 +-
 paddle/phi/kernels/funcs/embedding_grad.h     | 167 ++++++++++++++
 .../phi/kernels/gpu/embedding_grad_kernel.cu  | 116 +---------
 .../unittests/test_embedding_deterministic.py | 213 ++++++++++++++++++
 5 files changed, 456 insertions(+), 139 deletions(-)
 create mode 100644 paddle/phi/kernels/funcs/embedding_grad.h
 create mode 100644 python/paddle/fluid/tests/unittests/test_embedding_deterministic.py

diff --git a/paddle/fluid/operators/collective/c_embedding_op.cu b/paddle/fluid/operators/collective/c_embedding_op.cu
index 8b521580c5cd5..4861b5d26ab0f 100644
--- a/paddle/fluid/operators/collective/c_embedding_op.cu
+++ b/paddle/fluid/operators/collective/c_embedding_op.cu
@@ -18,8 +18,9 @@ limitations under the License. */
 #include "paddle/fluid/framework/op_registry.h"
 #include "paddle/fluid/platform/float16.h"
 #include "paddle/phi/backends/gpu/gpu_primitives.h"
+#include "paddle/phi/kernels/funcs/embedding_grad.h"
 
-DECLARE_bool(cudnn_deterministic);
+DECLARE_int64(embedding_deterministic);
 
 namespace paddle {
 namespace operators {
@@ -154,7 +155,6 @@ class CEmbeddingGradCUDAKernel : public framework::OpKernel<T> {
     int D = d_table_t->dims()[1];
     int K = ids_t->numel();
 
-    const int64_t end_idx = start_idx + N;
     auto limit = K * D;
     int blocks = NumBlocks(limit);
     int threads = kNumCUDAThreads;
@@ -166,33 +166,64 @@ class CEmbeddingGradCUDAKernel : public framework::OpKernel<T> {
     t.device(*dev_ctx.eigen_device()) = t.constant(static_cast<T>(0));
 
     const auto &index_type = framework::TransToProtoVarType(ids_t->dtype());
-    if (FLAGS_cudnn_deterministic) {
-      VLOG(2) << "Run grad kernel of embedding with single thread.";
-      blocks = 1;
-    }
-    if (index_type == framework::proto::VarType::INT32) {
-      CEmbeddingGrad<T, int32_t>
-          <<<blocks, threads, 0, dev_ctx.stream()>>>(d_table,
-                                                     d_output,
-                                                     ids_t->data<int32_t>(),
-                                                     K,
-                                                     D,
-                                                     N,
-                                                     start_idx,
-                                                     end_idx,
-                                                     limit);
-    } else if (index_type == framework::proto::VarType::INT64) {
-      CEmbeddingGrad<T, int64_t>
-          <<<blocks, threads, 0, dev_ctx.stream()>>>(d_table,
-                                                     d_output,
-                                                     ids_t->data<int64_t>(),
-                                                     K,
-                                                     D,
-                                                     N,
-                                                     start_idx,
-                                                     end_idx,
-                                                     limit);
+    if (FLAGS_embedding_deterministic == 1) {
+      if (index_type == framework::proto::VarType::INT32) {
+        phi::funcs::LaunchEmbeddingGradDeterministicKernel<T, int32_t>(
+            dev_ctx,
+            ids_t->data<int32_t>(),
+            d_output,
+            d_table,
+            N,
+            D,
+            K,
+            start_idx);
+        return;
+      } else if (index_type == framework::proto::VarType::INT64) {
+        phi::funcs::LaunchEmbeddingGradDeterministicKernel<T, int64_t>(
+            dev_ctx,
+            ids_t->data<int64_t>(),
+            d_output,
+            d_table,
+            N,
+            D,
+            K,
+            start_idx);
+        return;
+      }
+    } else {
+      if (FLAGS_embedding_deterministic > 1) {
+        VLOG(2) << "Run grad kernel of embedding with single thread.";
+        blocks = 1;
+      }
+      const int64_t end_idx = start_idx + N;
+      if (index_type == framework::proto::VarType::INT32) {
+        CEmbeddingGrad<T, int32_t>
+            <<<blocks, threads, 0, dev_ctx.stream()>>>(d_table,
+                                                       d_output,
+                                                       ids_t->data<int32_t>(),
+                                                       K,
+                                                       D,
+                                                       N,
+                                                       start_idx,
+                                                       end_idx,
+                                                       limit);
+        return;
+      } else if (index_type == framework::proto::VarType::INT64) {
+        CEmbeddingGrad<T, int64_t>
+            <<<blocks, threads, 0, dev_ctx.stream()>>>(d_table,
+                                                       d_output,
+                                                       ids_t->data<int64_t>(),
+                                                       K,
+                                                       D,
+                                                       N,
+                                                       start_idx,
+                                                       end_idx,
+                                                       limit);
+        return;
+      }
     }
+    PADDLE_THROW(phi::errors::InvalidArgument(
+        "The data type of Input(Ids) must be int32 or int64."));
   }
 };
 
diff --git a/paddle/phi/core/flags.cc b/paddle/phi/core/flags.cc
index 058ab7159e5ee..d2428e28b0dec 100644
--- a/paddle/phi/core/flags.cc
+++ b/paddle/phi/core/flags.cc
@@ -236,17 +236,19 @@ PHI_DEFINE_EXPORTED_bool(
  * CUDA related FLAG
  * Name: FLAGS_embedding_deterministic
  * Since Version: 2.5
- * Value Range: bool, default=false
+ * Value Range: int64, default=0
  * Example:
  * Note: whether to use deterministic algorithm in embedding op.
- *       If true, it will use deterministic CUDA kernel in embedding op.
+ *       If it is 1, it will use the optimized deterministic CUDA kernel in
+ *       embedding op. If it is 2, it will use the legacy deterministic
+ *       CUDA kernel in embedding op.
  */
-PHI_DEFINE_EXPORTED_bool(
+PHI_DEFINE_EXPORTED_int64(
     embedding_deterministic,
-    false,
+    0,
     "Whether allow using an deterministic algorithm for embedding "
     "operator. The deterministic algorithm may be slower. If "
-    "true, the algorithm is deterministic.");
+    "it is larger than 0, the algorithm is deterministic.");
 
 /**
  * CUDNN related FLAG
diff --git a/paddle/phi/kernels/funcs/embedding_grad.h b/paddle/phi/kernels/funcs/embedding_grad.h
new file mode 100644
index 0000000000000..3ad0f22c8e912
--- /dev/null
+++ b/paddle/phi/kernels/funcs/embedding_grad.h
@@ -0,0 +1,167 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/common/amp_type_traits.h"
+
+namespace phi {
+namespace funcs {
+
+template <typename T, typename IdT, int WarpSize, int BlockDimY, bool UseLimit>
+__global__ void EmbeddingGradDeterministicKernel(T* table,
+                                                 const T* output,
+                                                 const IdT* ids,
+                                                 const int64_t K,
+                                                 const int64_t D,
+                                                 const int64_t start_idx,
+                                                 const int64_t end_idx) {
+  using MT = typename dtype::MPTypeTrait<T>::Type;
+  constexpr int64_t kInvalidId = -1;
+  extern __shared__ char buf[];
+  MT* smem = reinterpret_cast<MT*>(buf);
+  MT* my_s = smem + WarpSize * threadIdx.y;
+  IdT* indices_batch =
+      reinterpret_cast<IdT*>(buf + sizeof(MT) * WarpSize * BlockDimY);
+
+  const int stride = static_cast<int>(D);
+
+  const int feature = threadIdx.x + blockIdx.x * WarpSize;
+
+  // To ensure determinism. If any other warps pulled grad data targeting
+  // dst_row, we elect the first warp in each matching group as the leader.
+  // Each leader warp serializes the accumulates targeting dst_row in shared
+  // memory, then adding the accumulated buffer to dst_row in table.
+  for (int batch_start = 0; batch_start < K;
+       batch_start += WarpSize * BlockDimY) {
+    int tid = threadIdx.x + threadIdx.y * WarpSize;
+    if (batch_start + tid < K) {
+      int64_t cur_id = static_cast<int64_t>(ids[batch_start + tid]);
+      if (UseLimit) {
+        if (cur_id >= start_idx && cur_id < end_idx) {
+          cur_id -= start_idx;
+        } else {
+          cur_id = kInvalidId;
+        }
+      }
+      indices_batch[tid] = cur_id;
+    }
+
+    int batch_end =
+        min(static_cast<int64_t>(batch_start + WarpSize * BlockDimY), K);
+
+    // Loop over the batch of <= 1024 loaded indices in chunks of BLOCKDIMY
+    for (int chunk_start = batch_start; chunk_start < batch_end;
+         chunk_start += BlockDimY) {
+      // This sync makes sure that indices_batch is ready and match-group
+      // leaders are done with their accumulates before other warps start
+      // loading again.
+      __syncthreads();
+
+      int n_this_chunk = min(batch_end - chunk_start, BlockDimY);
+
+      int64_t src_row = static_cast<int64_t>(chunk_start + threadIdx.y);
+      int64_t dst_row = indices_batch[src_row - batch_start];
+      if (src_row < K && feature < stride) {
+        if (UseLimit && dst_row == kInvalidId) {
+          my_s[threadIdx.x] = static_cast<MT>(0);
+        } else {
+          my_s[threadIdx.x] = static_cast<MT>(output[src_row * D + feature]);
+        }
+      }
+
+      __syncthreads();
+
+      if (src_row < K) {
+        int match_found_this_thread = 0;
+        if (threadIdx.x < n_this_chunk &&
+            (!UseLimit || dst_row != kInvalidId)) {
+          match_found_this_thread =
+              (dst_row ==
+               indices_batch[chunk_start - batch_start + threadIdx.x]);
+        }
+#ifdef PADDLE_WITH_HIP
+        unsigned long long int matchmask =      // NOLINT
+            __ballot(match_found_this_thread);  // NOLINT
+        int first_remaining_peer = __ffsll(matchmask) - 1;
+#else
+        // If and only if match_found_this_thread of the Nth thread is non-zero,
+        // set the Nth bit of matchmask to 1.
+        unsigned int matchmask =
+            __ballot_sync(0xffffffff, match_found_this_thread);
+        // Find the position of the first bit set to 1 in matchmask.
+        int first_remaining_peer = __ffs(matchmask) - 1;
+#endif
+
+        // select lowest-indexed warp as the leader
+        if (threadIdx.y == first_remaining_peer) {
+          // Set the first bit 1 in matchmask to 0.
+          matchmask ^= (1 << first_remaining_peer);
+          while (matchmask) {
+#ifdef PADDLE_WITH_HIP
+            first_remaining_peer = __ffsll(matchmask) - 1;
+#else
+            first_remaining_peer = __ffs(matchmask) - 1;
+#endif
+            my_s[threadIdx.x] +=
+                smem[threadIdx.x + WarpSize * first_remaining_peer];
+            matchmask ^= (1 << first_remaining_peer);
+          }
+          if (feature < stride && (!UseLimit || dst_row != kInvalidId)) {
+            auto table_idx = dst_row * D + feature;
+            table[table_idx] = static_cast<T>(
+                static_cast<MT>(table[table_idx]) + my_s[threadIdx.x]);
+          }
+        }
+      }
+    }
+  }
+}
+
+template <typename T, typename IdT>
+void LaunchEmbeddingGradDeterministicKernel(const GPUContext& ctx,
+                                            const IdT* ids,
+                                            const T* d_out,
+                                            T* d_table,
+                                            int64_t N,
+                                            int64_t D,
+                                            int64_t K,
+                                            int64_t start_idx = -1) {
+#ifdef PADDLE_WITH_HIP
+  constexpr int kWarpSize = 64;
+  constexpr int kBlockDimY = 16;
+#else
+  constexpr int kWarpSize = 32;
+  constexpr int kBlockDimY = 32;
+#endif
+  dim3 threads(kWarpSize, kBlockDimY);
+  dim3 grids(static_cast<int>((D + kWarpSize - 1) / kWarpSize));
+  using MT = typename dtype::MPTypeTrait<T>::Type;
+  constexpr auto kSharedMemSize = sizeof(MT) * kWarpSize * kBlockDimY +
+                                  sizeof(IdT) * kWarpSize * kBlockDimY;
+  if (start_idx < 0) {
+    EmbeddingGradDeterministicKernel<T, IdT, kWarpSize, kBlockDimY, false>
+        <<<grids, threads, kSharedMemSize, ctx.stream()>>>(
+            d_table, d_out, ids, K, D, -1, -1);
+  } else {
+    int64_t end_idx = start_idx + N;
+    EmbeddingGradDeterministicKernel<T, IdT, kWarpSize, kBlockDimY, true>
+        <<<grids, threads, kSharedMemSize, ctx.stream()>>>(
+            d_table, d_out, ids, K, D, start_idx, end_idx);
+  }
+}
+
+}  // namespace funcs
+}  // namespace phi
diff --git a/paddle/phi/kernels/gpu/embedding_grad_kernel.cu b/paddle/phi/kernels/gpu/embedding_grad_kernel.cu
index 4771dd15dd296..99ba12b1d6213 100644
--- a/paddle/phi/kernels/gpu/embedding_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/embedding_grad_kernel.cu
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "paddle/phi/kernels/embedding_grad_kernel.h"
+#include "paddle/phi/kernels/funcs/embedding_grad.h"
 
 #include "gflags/gflags.h"
 #include "glog/logging.h"
@@ -26,20 +27,10 @@
 #include "paddle/phi/kernels/funcs/eigen/common.h"
 #include "paddle/phi/kernels/funcs/embedding_util.h"
 
-DECLARE_bool(embedding_deterministic);
+DECLARE_int64(embedding_deterministic);
 
 namespace phi {
 
-#ifdef PADDLE_WITH_HIP
-#define WARP_SIZE 64
-#define BLOCKDIMY 16
-#else
-#define WARP_SIZE 32
-#define BLOCKDIMY 32
-#endif
-
-#define MASK 0xffffffff
-
 template <typename InT, typename OutT>
 __global__ void InputTypeConvert(const InT* in_ids,
                                  const int64_t K,
@@ -74,91 +65,6 @@ __global__ void EmbeddingGrad(T* table,
   }
 }
 
-template <typename T, typename IdT>
-__global__ void EmbeddingGradDeterministic(
-    T* table, const T* output, const IdT* ids, const IdT K, const IdT D) {
-  using MT = typename dtype::MPTypeTrait<T>::Type;
-  extern __shared__ char buf[];
-  MT* smem = reinterpret_cast<MT*>(buf);
-  MT* my_s = smem + WARP_SIZE * threadIdx.y;
-  IdT* indices_batch =
-      reinterpret_cast<IdT*>(buf + sizeof(MT) * WARP_SIZE * BLOCKDIMY);
-
-  const int stride = static_cast<int>(D);
-
-  const int feature = threadIdx.x + blockIdx.x * WARP_SIZE;
-
-  // To ensure determinism. If any other warps pulled grad data targeting
-  // dst_row, we elect the first warp in each matching group as the leader.
-  // Each leader warp serializes the accumulates targeting dst_row in shared
-  // memory, then adding the accumulated buffer to dst_row in table.
-  for (int batch_start = 0; batch_start < K;
-       batch_start += WARP_SIZE * BLOCKDIMY) {
-    int tid = threadIdx.x + threadIdx.y * WARP_SIZE;
-    if (batch_start + tid < K)
-      indices_batch[tid] = static_cast<IdT>(ids[batch_start + tid]);
-
-    int batch_end =
-        min(static_cast<IdT>(batch_start + WARP_SIZE * BLOCKDIMY), K);
-
-    // Loop over the batch of <= 1024 loaded indices in chunks of BLOCKDIMY
-    for (int chunk_start = batch_start; chunk_start < batch_end;
-         chunk_start += BLOCKDIMY) {
-      // This sync makes sure that indices_batch is ready and match-group
-      // leaders are done with their accumulates before other warps start
-      // loading again.
-      __syncthreads();
-
-      int n_this_chunk = min(batch_end - chunk_start, BLOCKDIMY);
-
-      IdT src_row = static_cast<IdT>(chunk_start + threadIdx.y);
-      IdT dst_row = indices_batch[src_row - batch_start];
-      if (src_row < K && feature < stride)
-        my_s[threadIdx.x] = static_cast<MT>(output[src_row * D + feature]);
-
-      __syncthreads();
-
-      if (src_row < K) {
-        int match_found_this_thread = 0;
-        if (threadIdx.x < n_this_chunk) {
-          match_found_this_thread =
-              (dst_row ==
-               indices_batch[chunk_start - batch_start + threadIdx.x]);
-        }
-#ifdef PADDLE_WITH_HIP
-        unsigned long long int matchmask =      // NOLINT
-            __ballot(match_found_this_thread);  // NOLINT
-        int first_remaining_peer = __ffsll(matchmask) - 1;
-#else
-        // If and only if match_found_this_thread of the Nth thread is non-zero,
-        // set the Nth bit of matchmask to 1.
-        unsigned int matchmask = __ballot_sync(MASK, match_found_this_thread);
-        // Find the position of the first bit set to 1 in matchmask.
-        int first_remaining_peer = __ffs(matchmask) - 1;
-#endif
-
-        // select lowest-indexed warp as the leader
-        if (threadIdx.y == first_remaining_peer) {
-          // Set the first bit 1 in matchmask to 0.
-          matchmask ^= (1 << first_remaining_peer);
-          while (matchmask) {
-#ifdef PADDLE_WITH_HIP
-            first_remaining_peer = __ffsll(matchmask) - 1;
-#else
-            first_remaining_peer = __ffs(matchmask) - 1;
-#endif
-            my_s[threadIdx.x] +=
-                smem[threadIdx.x + WARP_SIZE * first_remaining_peer];
-            matchmask ^= (1 << first_remaining_peer);
-          }
-          if (feature < stride)
-            table[dst_row * D + feature] += static_cast<T>(my_s[threadIdx.x]);
-        }
-      }
-    }
-  }
-}
-
 template <typename T, typename Context>
 struct EmbeddingGradCUDAFunctor {
   EmbeddingGradCUDAFunctor(const Context& dev_ctx,
@@ -198,20 +104,18 @@ struct EmbeddingGradCUDAFunctor {
           cudaMemsetAsync(d_table, 0, N * D * sizeof(T), dev_ctx_.stream()));
 #endif
 
-      if (FLAGS_embedding_deterministic) {
-        dim3 threads(WARP_SIZE, BLOCKDIMY);
-        dim3 grids(static_cast<int>((D + WARP_SIZE - 1) / WARP_SIZE));
-        using MT = typename dtype::MPTypeTrait<T>::Type;
-        EmbeddingGradDeterministic<T, IdT>
-            <<<grids,
-               threads,
-               sizeof(MT) * WARP_SIZE * BLOCKDIMY +
-                   sizeof(IdT) * WARP_SIZE * BLOCKDIMY,
-               dev_ctx_.stream()>>>(d_table, d_output, ids, K, D);
+      if (FLAGS_embedding_deterministic == 1) {
+        phi::funcs::LaunchEmbeddingGradDeterministicKernel<T, IdT>(
+            dev_ctx_, ids, d_output, d_table, N, D, K);
       } else {
         const int gridx = 2 * dev_ctx_.GetSMCount();
         dim3 threads(128, 8);
         dim3 grids(gridx, 1);
+        if (FLAGS_embedding_deterministic > 1) {
+          VLOG(2) << "Run grad kernel of embedding with single thread.";
+          grids.x = 1;
+          threads.y = 1;
+        }
         EmbeddingGrad<T, IdT><<<grids, threads, 0, dev_ctx_.stream()>>>(
             d_table, d_output, ids, N, K, D);
       }
diff --git a/python/paddle/fluid/tests/unittests/test_embedding_deterministic.py b/python/paddle/fluid/tests/unittests/test_embedding_deterministic.py
new file mode 100644
index 0000000000000..e64b4aa07ef9c
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_embedding_deterministic.py
@@ -0,0 +1,213 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import contextlib
+import random
+import sys
+import unittest
+
+import numpy as np
+
+import paddle
+from paddle.distributed.fleet.layers.mpu.mp_ops import _c_lookup_table
+
+
+@contextlib.contextmanager
+def deterministic_guard(value):
+    flag_name = 'FLAGS_embedding_deterministic'
+    old_value = paddle.get_flags(flag_name)[flag_name]
+    paddle.set_flags({flag_name: value})
+    assert paddle.get_flags(flag_name)[flag_name] == value
+    yield
+    paddle.set_flags({flag_name: old_value})
+    assert paddle.get_flags(flag_name)[flag_name] == old_value
+
+
+def to_numpy(tensor):
+    if tensor.dtype in [paddle.float16, paddle.bfloat16]:
+        tensor = tensor.astype(paddle.float32)
+    return tensor.numpy()
+
+
+def clone_weight(weight):
+    if weight.dtype == paddle.bfloat16:
+        weight = weight.astype(paddle.float32).numpy()
+        weight = paddle.to_tensor(weight, dtype=paddle.float32).astype(
+            paddle.bfloat16
+        )
+    else:
+        weight = paddle.to_tensor(weight.numpy())
+    weight.stop_gradient = False
+    return weight
+
+
+def embedding(ids, weight, out_grad, deterministic_level=0, rank=None):
+    weight = clone_weight(weight)
+    with deterministic_guard(deterministic_level):
+        if rank is not None:
+            vocab_size, _ = weight.shape
+            start_idx = vocab_size * rank
+            out = _c_lookup_table(weight, ids, start_index=start_idx)
+        else:
+            out = paddle.nn.functional.embedding(ids, weight)
+        out.backward(out_grad.clone())
+        return to_numpy(out), to_numpy(weight.grad)
+
+
+def embedding_ground_truth(ids, weight, out_grad, rank=None):
+    weight = clone_weight(weight.astype(paddle.float32))
+    out_grad = out_grad.astype(paddle.float32)
+    return embedding(ids, weight, out_grad, deterministic_level=2, rank=rank)
+
+
+def generate_input_data(
+    ids_shape,
+    vocab_size,
+    hidden_size,
+    weight_dtype,
+    ids_dtype,
+    allow_duplicate_id=True,
+    rank=None,
+    nranks=None,
+    allow_pure_random=False,
+):
+    max_id = vocab_size if rank is None else vocab_size * nranks
+    if allow_duplicate_id:
+        ids = np.random.randint(low=0, high=max_id, size=ids_shape)
+    else:
+        sequence = list(range(max_id))
+        numel = int(np.prod(ids_shape))
+        if len(sequence) < numel:
+            return None, None, None
+        ids = np.array(random.sample(sequence, numel)).reshape(ids_shape)
+
+    ids = paddle.to_tensor(ids).astype(ids_dtype)
+    ids.stop_gradient = True
+
+    weight = paddle.randn([vocab_size, hidden_size]).astype(weight_dtype)
+    weight.stop_gradient = False
+
+    out_grad_shape = list(ids_shape) + [hidden_size]
+    if allow_duplicate_id and not allow_pure_random:
+        out_grad = paddle.randint(low=-10, high=10, shape=out_grad_shape)
+    else:
+        out_grad = paddle.randn(out_grad_shape)
+    out_grad = out_grad.astype(weight.dtype)
+    return ids, weight, out_grad
+
+
+def get_all_dtypes():
+    if not paddle.is_compiled_with_cuda() or paddle.is_compiled_with_rocm():
+        return []
+
+    dtypes = [paddle.float32, paddle.float16]
+    if 'A100' in paddle.device.cuda.get_device_properties().name:
+        dtypes.append(paddle.bfloat16)
+    return dtypes
+
+
+class TestEmbeddingBase(unittest.TestCase):
+    def setUp(self):
+        self.ids_shape = [32, 3]
+        self.vocab_size = 128
+        self.hidden_size = 1024
+        self.nranks = 8
+
+    def check_main(
+        self,
+        weight_dtype,
+        ids_dtype,
+        deterministic_level=0,
+        rank=None,
+        allow_duplicate_id=True,
+        allow_pure_random=False,
+    ):
+        if sys.platform == 'win32' and rank is not None:
+            return
+
+        ids, weight, out_grad = generate_input_data(
+            ids_shape=self.ids_shape,
+            vocab_size=self.vocab_size,
+            hidden_size=self.hidden_size,
+            weight_dtype=weight_dtype,
+            ids_dtype=ids_dtype,
+            allow_duplicate_id=allow_duplicate_id,
+            rank=rank,
+            nranks=self.nranks,
+            allow_pure_random=allow_pure_random,
+        )
+        if ids is None:
+            return
+
+        if allow_pure_random:
+            out_1, weight_grad_1 = embedding_ground_truth(
+                ids, weight, out_grad, rank
+            )
+            out_2, weight_grad_2 = embedding_ground_truth(
+                ids, weight, out_grad, rank
+            )
+        else:
+            out_1, weight_grad_1 = embedding_ground_truth(
+                ids, weight, out_grad, rank
+            )
+            out_2, weight_grad_2 = embedding(
+                ids,
+                weight,
+                out_grad,
+                deterministic_level=deterministic_level,
+                rank=rank,
+            )
+        np.testing.assert_equal(out_1, out_2)
+        np.testing.assert_equal(weight_grad_1, weight_grad_2)
+
+    def test_main(self):
+        weight_dtypes = get_all_dtypes()
+        ids_dtypes = [paddle.int64, paddle.int32]
+        deterministic_levels = [0, 1]
+        ranks = [None, 0, 2, 4, 8]
+        allow_duplicate_ids = [False, True]
+        allow_pure_randoms = [False, True]
+        for weight_dtype in weight_dtypes:
+            for ids_dtype in ids_dtypes:
+                for deterministic_level in deterministic_levels:
+                    for rank in ranks:
+                        for allow_duplicate_id in allow_duplicate_ids:
+                            for allow_pure_random in allow_pure_randoms:
+                                self.check_main(
+                                    weight_dtype,
+                                    ids_dtype,
+                                    deterministic_level,
+                                    rank,
+                                    allow_duplicate_id,
+                                    allow_pure_random,
+                                )
+
+
+class TestEmbedding2(TestEmbeddingBase):
+    def setUp(self):
+        self.ids_shape = [32, 16]
+        self.vocab_size = 128
+        self.hidden_size = 1024
+        self.nranks = 8
+
+
+class TestEmbeddingDeterministic(unittest.TestCase):
+    def setUp(self):
+        self.ids_shape = [32, 16]
+        self.vocab_size = 128
+        self.hidden_size = 1024
+
+
+if __name__ == "__main__":
+    unittest.main()

From c59debe22926fb97d99ab3007e1b5f092e40f6c3 Mon Sep 17 00:00:00 2001
From: ShenLiang <1422485404@qq.com>
Date: Wed, 26 Apr 2023 14:35:18 +0800
Subject: [PATCH 078/405] [HybridParallel]Add segment methods for
 pipelineparallel (#53344)

---
 .../parallel_layers/pp_layers.py              | 34 ++++++++++++++++++-
 .../unittests/hybrid_parallel_pp_layer.py     | 14 ++++++++
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py
index 2f5c42a69e362..f3be9894a9cfe 100755
--- a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py
+++ b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py
@@ -109,7 +109,37 @@ def __init__(
         ), "layer number should be greater than number of segments"
 
     def do_segment(self):
-        if self.method == "uniform":
+
+        if isinstance(self.method, list):
+            seg_method = self.method[:]
+            source_num_parts = len(seg_method) - 1
+
+            def check_sanity():
+                assert seg_method[0] == 0, "seg_method[0] should be 0"
+                for part in seg_method:
+                    assert isinstance(part, int), "part should be int"
+                    assert part >= 0, f"part[{part}] should be greater than 0"
+                    assert (
+                        part <= self.num_items
+                    ), "part[{}] should be less than num_items[{}]".format(
+                        part, self.num_items
+                    )
+
+            check_sanity()
+
+            if self.num_parts == source_num_parts + 1:
+                seg_method.append(self.num_items)
+                return seg_method
+            elif self.num_parts == source_num_parts:
+                return seg_method
+            else:
+                raise ValueError(
+                    "We set seg_method as {}, this length is {}, but the number of stages is {}".format(
+                        seg_method, len(seg_method), self.num_parts
+                    )
+                )
+
+        elif self.method == "uniform":
             return self.uniform(self.num_items, self.num_parts)
 
         elif self.method.startswith('layer:'):
@@ -144,6 +174,8 @@ def do_segment(self):
                     memory_counter = 0
             result[actual_num_parts] = len(weights)
             return result
+        else:
+            raise ValueError(f"method {self.method} is not supported")
 
     def _gen_layer_weight(self, layername):
         weight_idxs = []
diff --git a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py
index 5d15e79d64b75..cf4c20e550ab3 100644
--- a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py
+++ b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py
@@ -136,6 +136,20 @@ def test_pipelayer_sequential(self):
                 np.testing.assert_array_equal(param_a.name, param_b.name)
                 np.testing.assert_allclose(param_a.numpy(), param_b.numpy())
 
+    def test_pipelayer_segment_method(self):
+        init_net = AlexNetPipe()
+        pipe_model = PipelineLayer(
+            layers=init_net.to_layers(),
+            num_stages=self.pipeline_parallel_size,
+            seg_method=[0, 4],
+            loss_fn=nn.CrossEntropyLoss(),
+        )
+        stage_id = self.hcg.get_stage_id()
+        if stage_id == 0:
+            np.testing.assert_array_equal(len(pipe_model.parameters()), 4)
+        elif stage_id == 1:
+            np.testing.assert_array_equal(len(pipe_model.parameters()), 8)
+
 
 if __name__ == '__main__':
     unittest.main()

From cd88156a369bbfb83d6306f89e0ae6ebd78b8040 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=AA=91=E9=A9=AC=E5=B0=8F=E7=8C=AB?= <1435130236@qq.com>
Date: Wed, 26 Apr 2023 14:47:29 +0800
Subject: [PATCH 079/405] [Bug fixes] enable two ops to support bf16 in llama
 model (#53026)

---
 python/paddle/nn/functional/activation.py | 2 +-
 python/paddle/tensor/manipulation.py      | 6 ++++--
 python/paddle/tensor/math.py              | 1 +
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py
index d89ce9cb4537a..04fa9ebc6dd09 100644
--- a/python/paddle/nn/functional/activation.py
+++ b/python/paddle/nn/functional/activation.py
@@ -965,7 +965,7 @@ def silu(x, name=None):
     Where :math:`x` is the input Tensor.
 
     Parameters:
-        x (Tensor): The input Tensor with data type float32, float64.
+        x (Tensor): The input Tensor with data type bfloat16, float16, float32, float64.
         name (str, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
 
     Returns:
diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py
index 4d053e7c90c6d..27e2a4b812de5 100644
--- a/python/paddle/tensor/manipulation.py
+++ b/python/paddle/tensor/manipulation.py
@@ -1899,7 +1899,7 @@ def split(x, num_or_sections, axis=0, name=None):
     Split the input tensor into multiple sub-Tensors.
 
     Args:
-        x (Tensor): A N-D Tensor. The data type is bool, float16, float32, float64, uint8, int8, int32 or int64.
+        x (Tensor): A N-D Tensor. The data type is bool, bfloat16, float16, float32, float64, uint8, int8, int32 or int64.
         num_or_sections (int|list|tuple): If ``num_or_sections`` is an int, then ``num_or_sections``
             indicates the number of equal sized sub-Tensors that the ``x`` will be divided into.
             If ``num_or_sections`` is a list or tuple, the length of it indicates the number of
@@ -1970,6 +1970,7 @@ def split(x, num_or_sections, axis=0, name=None):
             'input',
             [
                 'bool',
+                'bfloat16',
                 'float16',
                 'uint16',
                 'float32',
@@ -2546,7 +2547,7 @@ def unsqueeze(x, axis, name=None):
     please use `Tensor.clone` like ``unsqueeze_clone_x = x.unsqueeze(-1).clone()``.
 
     Args:
-        x (Tensor): The input Tensor to be unsqueezed. Supported data type: float32, float64, bool, int8, int32, int64.
+        x (Tensor): The input Tensor to be unsqueezed. Supported data type: bfloat16, float16, float32, float64, bool, int8, int32, int64.
         axis (int|list|tuple|Tensor): Indicates the dimensions to be inserted. The data type is ``int32`` .
                                     If ``axis`` is a list or tuple, the elements of it should be integers or Tensors with shape [1].
                                     If ``axis`` is a Tensor, it should be an 1-D Tensor .
@@ -2600,6 +2601,7 @@ def unsqueeze(x, axis, name=None):
             input,
             'input',
             [
+                'uint16',
                 'float16',
                 'uint16',
                 'float32',
diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py
index ece8463f35e63..2f94f0a7e2013 100644
--- a/python/paddle/tensor/math.py
+++ b/python/paddle/tensor/math.py
@@ -499,6 +499,7 @@ def _elementwise_op(helper):
         "elementwise_sub",
         "elementwise_mul",
         "elementwise_div",
+        "elementwise_max",
     ]
     if original_op_type in bf16_and_complex_supported_ops:
         data_type = [

From f9e5072b2ba1a36f4fde23763b37d0fbf599fb1d Mon Sep 17 00:00:00 2001
From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com>
Date: Wed, 26 Apr 2023 15:59:40 +0800
Subject: [PATCH 080/405] remove some [-Wunused-parameter] waring (#53319)

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop
---
 paddle/fluid/distributed/ps/service/server.h  | 11 +++---
 .../fluid/distributed/ps/table/common_table.h | 14 ++++----
 paddle/fluid/distributed/ps/table/table.h     | 33 ++++++++---------
 paddle/fluid/framework/data_feed.h            | 23 ++++++------
 paddle/fluid/framework/data_set.h             | 36 ++++++++++---------
 paddle/fluid/framework/device_worker.h        | 19 +++++-----
 .../framework/no_need_buffer_vars_inference.h | 27 +++++++-------
 paddle/fluid/framework/op_registry.h          |  2 +-
 paddle/phi/core/kernel_registry.h             |  2 +-
 paddle/phi/kernels/empty_kernel.cc            |  6 ++--
 paddle/phi/kernels/funcs/compound_functors.h  | 18 +++++-----
 .../phi/kernels/funcs/elementwise_functor.h   | 25 ++++++-------
 paddle/phi/kernels/funcs/for_range.h          |  5 +--
 paddle/phi/kernels/funcs/reduce_functor.h     | 10 +++---
 .../kernels/impl/meshgrid_grad_kernel_impl.h  |  4 +--
 paddle/phi/kernels/impl/reduce_grad.h         |  6 ++--
 .../kernels/impl/set_value_grad_kernel_impl.h |  2 +-
 paddle/phi/kernels/onednn/conv_handler.h      |  4 +--
 18 files changed, 131 insertions(+), 116 deletions(-)

diff --git a/paddle/fluid/distributed/ps/service/server.h b/paddle/fluid/distributed/ps/service/server.h
index 7aab3abe71f91..48b32d22cac79 100644
--- a/paddle/fluid/distributed/ps/service/server.h
+++ b/paddle/fluid/distributed/ps/service/server.h
@@ -31,6 +31,7 @@
 #include "paddle/fluid/framework/scope.h"
 #include "paddle/fluid/platform/device_context.h"
 #include "paddle/fluid/platform/place.h"
+#include "paddle/phi/core/macros.h"
 
 namespace google {
 namespace protobuf {
@@ -96,7 +97,9 @@ class PSServer {
   virtual int32_t StartS2S() { return 0; }
 
   virtual ::std::future<int32_t> SendPServer2PServerMsg(
-      int msg_type, int to_pserver_id, const std::string &msg) {
+      int msg_type UNUSED,
+      int to_pserver_id UNUSED,
+      const std::string &msg UNUSED) {
     LOG(FATAL) << "NotImplementError: PSServer::send_pserver2pserver_msg";
     std::promise<int32_t> promise;
     std::future<int> fut = promise.get_future();
@@ -124,9 +127,9 @@ class PSServer {
     }
     return itr->second(msg_type, from_pserver_id, msg);
   }
-  virtual int32_t ReceiveFromPServer(int msg_type,
-                                     int pserver_id,
-                                     const std::string &msg) {
+  virtual int32_t ReceiveFromPServer(int msg_type UNUSED,
+                                     int pserver_id UNUSED,
+                                     const std::string &msg UNUSED) {
     LOG(FATAL) << "NotImplementError::PSServer::ReceiveFromPServer";
     return -1;
   }
diff --git a/paddle/fluid/distributed/ps/table/common_table.h b/paddle/fluid/distributed/ps/table/common_table.h
index d1b93af538cde..d56b8cd39a00a 100644
--- a/paddle/fluid/distributed/ps/table/common_table.h
+++ b/paddle/fluid/distributed/ps/table/common_table.h
@@ -70,18 +70,20 @@ class BarrierTable : public Table {
   BarrierTable() {}
   virtual ~BarrierTable() {}
 
-  virtual void *GetShard(size_t shard_idx) { return 0; }
+  virtual void *GetShard(size_t shard_idx UNUSED) { return 0; }
 
-  virtual int32_t Pull(TableContext &context) { return 0; }  // NOLINT
-  virtual int32_t Push(TableContext &context) { return 0; }  // NOLINT
+  virtual int32_t Pull(TableContext &context UNUSED) { return 0; }  // NOLINT
+  virtual int32_t Push(TableContext &context UNUSED) { return 0; }  // NOLINT
 
-  int32_t Shrink(const std::string &param) override { return 0; }
+  int32_t Shrink(const std::string &param UNUSED) override { return 0; }
   virtual void Clear() {}
   virtual int32_t Flush() { return 0; }
-  virtual int32_t Load(const std::string &path, const std::string &param) {
+  virtual int32_t Load(const std::string &path UNUSED,
+                       const std::string &param UNUSED) {
     return 0;
   }
-  virtual int32_t Save(const std::string &path, const std::string &param) {
+  virtual int32_t Save(const std::string &path UNUSED,
+                       const std::string &param UNUSED) {
     return 0;
   }
   virtual int32_t InitializeShard() { return 0; }
diff --git a/paddle/fluid/distributed/ps/table/table.h b/paddle/fluid/distributed/ps/table/table.h
index f07a3f2132217..b64e05e3b0a11 100644
--- a/paddle/fluid/distributed/ps/table/table.h
+++ b/paddle/fluid/distributed/ps/table/table.h
@@ -32,6 +32,7 @@
 #include "paddle/fluid/platform/device_context.h"
 #include "paddle/fluid/platform/place.h"
 #include "paddle/fluid/string/string_helper.h"
+#include "paddle/phi/core/macros.h"
 
 namespace paddle {
 namespace distributed {
@@ -77,22 +78,22 @@ class Table {
   virtual int32_t Push(TableContext &context) = 0;  // NOLINT
 
   // only for barrier
-  virtual int32_t Barrier(const uint32_t trainer_id,
-                          const std::string barrier_type) {
+  virtual int32_t Barrier(const uint32_t trainer_id UNUSED,
+                          const std::string barrier_type UNUSED) {
     return 0;
   }
 
   // only for barrier table
   virtual int32_t SetTableMap(
-      std::unordered_map<uint32_t, std::shared_ptr<Table>> *table_map) {
+      std::unordered_map<uint32_t, std::shared_ptr<Table>> *table_map UNUSED) {
     return 0;
   }
 
   // only for tensor table
   virtual int32_t SetProgramEnv(
-      framework::Scope *scope,
-      platform::Place place,
-      const std::vector<framework::ProgramDesc> *sub_program) {
+      framework::Scope *scope UNUSED,
+      platform::Place place UNUSED,
+      const std::vector<framework::ProgramDesc> *sub_program UNUSED) {
     return 0;
   }
 
@@ -115,23 +116,23 @@ class Table {
                        const std::string &converter) = 0;
   // for cache
   virtual int32_t SaveCache(
-      const std::string &path,
-      const std::string &param,
+      const std::string &path UNUSED,
+      const std::string &param UNUSED,
       paddle::framework::Channel<std::pair<uint64_t, std::string>>
-          &shuffled_channel) {
+          &shuffled_channel UNUSED) {
     return 0;
   }
 
   virtual int64_t CacheShuffle(
-      const std::string &path,
-      const std::string &param,
-      double cache_threshold,
+      const std::string &path UNUSED,
+      const std::string &param UNUSED,
+      double cache_threshold UNUSED,
       std::function<std::future<int32_t>(
           int msg_type, int to_pserver_id, std::string &msg)>  // NOLINT
-          send_msg_func,
+          send_msg_func UNUSED,
       paddle::framework::Channel<std::pair<uint64_t, std::string>>
-          &shuffled_channel,
-      const std::vector<Table *> &table_ptrs) {
+          &shuffled_channel UNUSED,
+      const std::vector<Table *> &table_ptrs UNUSED) {
     return 0;
   }
 
@@ -149,7 +150,7 @@ class Table {
 
   virtual void *GetShard(size_t shard_idx) = 0;
   virtual std::pair<int64_t, int64_t> PrintTableStat() { return {0, 0}; }
-  virtual int32_t CacheTable(uint16_t pass_id) { return 0; }
+  virtual int32_t CacheTable(uint16_t pass_id UNUSED) { return 0; }
 
   // for patch model
   virtual void Revert() {}
diff --git a/paddle/fluid/framework/data_feed.h b/paddle/fluid/framework/data_feed.h
index a6cf44234773b..1057640842c2c 100644
--- a/paddle/fluid/framework/data_feed.h
+++ b/paddle/fluid/framework/data_feed.h
@@ -402,22 +402,22 @@ class CustomParser {
   virtual void Init(const std::vector<SlotConf>& slots) = 0;
   virtual bool Init(const std::vector<AllSlotInfo>& slots) = 0;
   virtual void ParseOneInstance(const char* str, Record* instance) = 0;
-  virtual int ParseInstance(int len,
-                            const char* str,
-                            std::vector<Record>* instances) {
+  virtual int ParseInstance(int len UNUSED,
+                            const char* str UNUSED,
+                            std::vector<Record>* instances UNUSED) {
     return 0;
   }
   virtual bool ParseOneInstance(
-      const std::string& line,
-      std::function<void(std::vector<SlotRecord>&, int)>
-          GetInsFunc) {  // NOLINT
+      const std::string& line UNUSED,
+      std::function<void(std::vector<SlotRecord>&, int)> GetInsFunc
+          UNUSED) {  // NOLINT
     return true;
   }
   virtual bool ParseFileInstance(
-      std::function<int(char* buf, int len)> ReadBuffFunc,
-      std::function<void(std::vector<SlotRecord>&, int, int)>
-          PullRecordsFunc,  // NOLINT
-      int& lines) {         // NOLINT
+      std::function<int(char* buf, int len)> ReadBuffFunc UNUSED,
+      std::function<void(std::vector<SlotRecord>&, int, int)> PullRecordsFunc
+          UNUSED,           // NOLINT
+      int& lines UNUSED) {  // NOLINT
     return false;
   }
 };
@@ -1267,7 +1267,8 @@ class DataFeed {
   virtual void SetInsIdVec(MiniBatchGpuPack* pack) {}
 #endif
 
-  virtual void DumpWalkPath(std::string dump_path, size_t dump_rate) {
+  virtual void DumpWalkPath(std::string dump_path UNUSED,
+                            size_t dump_rate UNUSED) {
     PADDLE_THROW(platform::errors::Unimplemented(
         "This function(DumpWalkPath) is not implemented."));
   }
diff --git a/paddle/fluid/framework/data_set.h b/paddle/fluid/framework/data_set.h
index 22bb5b703944d..1bc60993e36a0 100644
--- a/paddle/fluid/framework/data_set.h
+++ b/paddle/fluid/framework/data_set.h
@@ -25,6 +25,7 @@
 #include <unordered_set>
 #include <utility>
 #include <vector>
+#include "paddle/phi/core/macros.h"
 #ifdef PADDLE_WITH_GLOO
 #include <gloo/broadcast.h>
 
@@ -51,12 +52,12 @@ class Dataset {
   Dataset() {}
   virtual ~Dataset() {}
   // do sample
-  virtual void TDMSample(const std::string tree_name,
-                         const std::string tree_path,
-                         const std::vector<uint16_t> tdm_layer_counts,
-                         const uint16_t start_sample_layer,
-                         const bool with_hierachy,
-                         const uint16_t seed_,
+  virtual void TDMSample(const std::string tree_name UNUSED,
+                         const std::string tree_path UNUSED,
+                         const std::vector<uint16_t> tdm_layer_counts UNUSED,
+                         const uint16_t start_sample_layer UNUSED,
+                         const bool with_hierachy UNUSED,
+                         const uint16_t seed_ UNUSED,
                          const uint16_t sample_slot) {}
   // set file list
   virtual void SetFileList(const std::vector<std::string>& filelist) = 0;
@@ -238,8 +239,9 @@ class DatasetImpl : public Dataset {
   virtual void WaitPreLoadDone();
   virtual void ReleaseMemory();
   virtual void LocalShuffle();
-  virtual void GlobalShuffle(int thread_num = -1) {}
-  virtual void SlotsShuffle(const std::set<std::string>& slots_to_replace) {}
+  virtual void GlobalShuffle(int thread_num UNUSED = -1) {}
+  virtual void SlotsShuffle(
+      const std::set<std::string>& slots_to_replace UNUSED) {}
   virtual const std::vector<T>& GetSlotsOriginalData() {
     return slots_shuffle_original_data_;
   }
@@ -251,12 +253,12 @@ class DatasetImpl : public Dataset {
   virtual void MergeByInsId() {}
   virtual void PreprocessInstance() {}
   virtual void PostprocessInstance() {}
-  virtual void SetCurrentPhase(int current_phase) {}
-  virtual void GenerateLocalTablesUnlock(int table_id,
-                                         int feadim,
-                                         int read_thread_num,
-                                         int consume_thread_num,
-                                         int shard_num) {}
+  virtual void SetCurrentPhase(int current_phase UNUSED) {}
+  virtual void GenerateLocalTablesUnlock(int table_id UNUSED,
+                                         int feadim UNUSED,
+                                         int read_thread_num UNUSED,
+                                         int consume_thread_num UNUSED,
+                                         int shard_num UNUSED) {}
   virtual void ClearLocalTables() {}
   virtual void CreatePreLoadReaders();
   virtual void DestroyPreLoadReaders();
@@ -288,9 +290,9 @@ class DatasetImpl : public Dataset {
   virtual uint32_t GetPassID() { return pass_id_; }
 
  protected:
-  virtual int ReceiveFromClient(int msg_type,
-                                int client_id,
-                                const std::string& msg) {
+  virtual int ReceiveFromClient(int msg_type UNUSED,
+                                int client_id UNUSED,
+                                const std::string& msg UNUSED) {
     // TODO(yaoxuefeng) for SlotRecordDataset
     return -1;
   }
diff --git a/paddle/fluid/framework/device_worker.h b/paddle/fluid/framework/device_worker.h
index 743513e38aad1..177513b70cd3c 100644
--- a/paddle/fluid/framework/device_worker.h
+++ b/paddle/fluid/framework/device_worker.h
@@ -44,6 +44,7 @@ limitations under the License. */
 #include "paddle/fluid/platform/place.h"
 #include "paddle/fluid/platform/timer.h"
 #include "paddle/phi/backends/dynload/port.h"
+#include "paddle/phi/core/macros.h"
 
 namespace paddle {
 namespace framework {
@@ -179,14 +180,14 @@ class DeviceWorker {
   virtual void BindingDataFeedMemory() = 0;
   virtual void SetRootScope(Scope* root_scope);
   virtual void SetDataFeed(DataFeed* data_feed);
-  virtual void SetWorkerNum(int num) {}
-  virtual void CacheProgram(const ProgramDesc& main_program) {}
+  virtual void SetWorkerNum(int num UNUSED) {}
+  virtual void CacheProgram(const ProgramDesc& main_program UNUSED) {}
   virtual void ProduceTasks() {}
   virtual void GetXpuOpIndex() {}
-  virtual void Schedule(int taskid) {}
+  virtual void Schedule(int taskid UNUSED) {}
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-  virtual void SetStream(const gpuStream_t stream) {}
-  virtual void SetEvent(const gpuEvent_t event) {}
+  virtual void SetStream(const gpuStream_t stream UNUSED) {}
+  virtual void SetEvent(const gpuEvent_t event UNUSED) {}
 #endif
   virtual void SetNeedDumpField(bool need_dump_field) {
     need_dump_field_ = need_dump_field;
@@ -256,7 +257,7 @@ class CPUWorkerBase : public DeviceWorker {
   virtual void TrainFiles() = 0;
   virtual void TrainFilesWithProfiler() {}
   virtual void PrintFetchVars() {}
-  virtual void CreateDeviceResource(const ProgramDesc& main_prog) {}
+  virtual void CreateDeviceResource(const ProgramDesc& main_prog UNUSED) {}
 
  protected:
   int thread_id_;
@@ -684,7 +685,7 @@ class SectionWorker : public DeviceWorker {
   void PrepareUnusedVar();
 
   void BindingDataFeedMemory() override {}
-  void CreateDeviceResource(const ProgramDesc& main_prog) override{};
+  void CreateDeviceResource(const ProgramDesc& main_prog UNUSED) override{};
 
   void TrainFiles() override;
   void TrainFilesWithProfiler() override{};
@@ -693,7 +694,7 @@ class SectionWorker : public DeviceWorker {
 
   const platform::Place& place() const { return place_; }
 
-  void SetDeviceIndex(int tid) override {}
+  void SetDeviceIndex(int tid UNUSED) override {}
   void SetThreadIndex(int thread_id) { thread_id_ = thread_id; }
   void SetMicrobatchNum(int num) { num_microbatches_ = num; }
   void SetPipelineStageNum(int num) { num_pipeline_stages_ = num; }
@@ -755,7 +756,7 @@ class HeterSectionWorker : public DeviceWorker {
   ~HeterSectionWorker() override {}
 
   void Initialize(const TrainerDesc& desc) override;
-  void CreateDeviceResource(const ProgramDesc& main_prog) override{};
+  void CreateDeviceResource(const ProgramDesc& main_prog UNUSED) override{};
 
   void TrainFiles() override;
   void TrainFilesWithProfiler() override;
diff --git a/paddle/fluid/framework/no_need_buffer_vars_inference.h b/paddle/fluid/framework/no_need_buffer_vars_inference.h
index 47e74e2d80a39..37f790a0d3f41 100644
--- a/paddle/fluid/framework/no_need_buffer_vars_inference.h
+++ b/paddle/fluid/framework/no_need_buffer_vars_inference.h
@@ -23,6 +23,7 @@
 #include "paddle/fluid/framework/type_defs.h"
 #include "paddle/fluid/imperative/type_defs.h"
 #include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/core/macros.h"
 
 namespace paddle {
 namespace framework {
@@ -83,19 +84,19 @@ class NoNeedBufferVarsInference {
   }
 };
 
-#define DECLARE_NO_NEED_BUFFER_VARS_INFERER(class_type, ...)          \
-  class class_type final                                              \
-      : public ::paddle::framework::NoNeedBufferVarsInference {       \
-   public:                                                            \
-    using ::paddle::framework::NoNeedBufferVarsInference::            \
-        NoNeedBufferVarsInference;                                    \
-                                                                      \
-    const std::unordered_set<std::string> &operator()(                \
-        const ::paddle::framework::InferNoNeedBufferVarsContext &ctx) \
-        const final {                                                 \
-      static std::unordered_set<std::string> __ret__{__VA_ARGS__};    \
-      return __ret__;                                                 \
-    }                                                                 \
+#define DECLARE_NO_NEED_BUFFER_VARS_INFERER(class_type, ...)         \
+  class class_type final                                             \
+      : public ::paddle::framework::NoNeedBufferVarsInference {      \
+   public:                                                           \
+    using ::paddle::framework::NoNeedBufferVarsInference::           \
+        NoNeedBufferVarsInference;                                   \
+                                                                     \
+    const std::unordered_set<std::string> &operator()(               \
+        const ::paddle::framework::InferNoNeedBufferVarsContext &ctx \
+            UNUSED) const final {                                    \
+      static std::unordered_set<std::string> __ret__{__VA_ARGS__};   \
+      return __ret__;                                                \
+    }                                                                \
   }
 
 class InferNoNeedBufferVarsFN {
diff --git a/paddle/fluid/framework/op_registry.h b/paddle/fluid/framework/op_registry.h
index 13ef07ab9855e..941e8ea050e08 100644
--- a/paddle/fluid/framework/op_registry.h
+++ b/paddle/fluid/framework/op_registry.h
@@ -148,7 +148,7 @@ class OpRegistry {
 };
 
 template <typename PlaceType>
-inline void CheckKernelLaunch(const char* op_type) {}
+inline void CheckKernelLaunch(const char* op_type UNUSED) {}
 
 #ifdef PADDLE_WITH_CUDA
 template <>
diff --git a/paddle/phi/core/kernel_registry.h b/paddle/phi/core/kernel_registry.h
index 5d2ab27469af1..3ca455da1dd40 100644
--- a/paddle/phi/core/kernel_registry.h
+++ b/paddle/phi/core/kernel_registry.h
@@ -1365,7 +1365,7 @@ struct KernelRegistrar {
     return 0;                                                               \
   }                                                                         \
   void __PD_KERNEL_args_def_FN_##kernel_name##_##backend##_##layout(        \
-      const ::phi::KernelKey& kernel_key, ::phi::Kernel* kernel)
+      const ::phi::KernelKey& kernel_key UNUSED, ::phi::Kernel* kernel UNUSED)
 #else
 #define __PD_REGISTER_KERNEL_FOR_ALL_DTYPE(                                 \
     reg_type, kernel_name, backend, layout, kernel_fn)                      \
diff --git a/paddle/phi/kernels/empty_kernel.cc b/paddle/phi/kernels/empty_kernel.cc
index f2782a63c9020..0877a8e24468c 100644
--- a/paddle/phi/kernels/empty_kernel.cc
+++ b/paddle/phi/kernels/empty_kernel.cc
@@ -12,17 +12,17 @@
  See the License for the specific language governing permissions and
  limitations under the License. */
 #include "paddle/phi/kernels/empty_kernel.h"
-
 #include "paddle/phi/backends/all_context.h"
 #include "paddle/phi/common/complex.h"
 #include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/macros.h"
 
 namespace phi {
 
 template <typename T, typename Context>
 void EmptyKernel(const Context& dev_ctx,
                  const IntArray& shape,
-                 DataType dtype,
+                 DataType dtype UNUSED,
                  DenseTensor* out) {
   out->Resize(phi::make_ddim(shape.GetData()));
   dev_ctx.template Alloc<T>(out);
@@ -31,7 +31,7 @@ void EmptyKernel(const Context& dev_ctx,
 template <typename T, typename Context>
 void EmptyLikeKernel(const Context& dev_ctx,
                      const DenseTensor& x,
-                     DataType dtype,
+                     DataType dtype UNUSED,
                      DenseTensor* out) {
   dev_ctx.template Alloc<T>(out);
 }
diff --git a/paddle/phi/kernels/funcs/compound_functors.h b/paddle/phi/kernels/funcs/compound_functors.h
index a27d0e5a9f164..0fd3fd0e932fc 100644
--- a/paddle/phi/kernels/funcs/compound_functors.h
+++ b/paddle/phi/kernels/funcs/compound_functors.h
@@ -17,7 +17,7 @@ limitations under the License. */
 #include <string>
 #include <unordered_set>
 #include <vector>
-
+#include "paddle/phi/core/macros.h"
 namespace phi {
 namespace funcs {
 
@@ -33,7 +33,7 @@ struct BinaryCompoundFunctor {
     return func1_(x, intermediat_out);
   }
 
-  inline HOSTDEVICE T GetIntermediateOut(T x, T y) { return func2_(y); }
+  inline HOSTDEVICE T GetIntermediateOut(T x UNUSED, T y) { return func2_(y); }
 
   BinaryFunctor func1_;
   UnaryFunctor func2_;
@@ -64,16 +64,18 @@ struct BinaryCompoundGradDxFunctor {
                               const UnaryFun &unary_fun)
       : d_binary_fun_(d_binary_fun), unary_fun_(unary_fun) {}
 
-  inline HOSTDEVICE T Recompute(T x, T y, T out, T dout) {
+  inline HOSTDEVICE T Recompute(T x, T y, T out UNUSED, T dout) {
     return dout * d_binary_fun_.Dx(x, unary_fun_(y));
   }
 
-  inline HOSTDEVICE T
-  UseIntermediateOut(T x, T y, T intermediate_out, T out, T dout) {
+  inline HOSTDEVICE T UseIntermediateOut(
+      T x, T y UNUSED, T intermediate_out, T out UNUSED, T dout) {
     return dout * d_binary_fun_.Dx(x, intermediate_out);
   }
 
-  inline HOSTDEVICE T GetIntermediateOut(T x, T y) { return unary_fun_(y); }
+  inline HOSTDEVICE T GetIntermediateOut(T x UNUSED, T y) {
+    return unary_fun_(y);
+  }
 
  private:
   DBinaryFun d_binary_fun_;
@@ -210,13 +212,13 @@ struct BinaryCompoundGradDIntermedaiteOutFunctor {
                                             const UnaryFun &unary_fun)
       : d_binary_fun_(d_binary_fun), unary_fun_(unary_fun) {}
 
-  inline HOSTDEVICE T Recompute(T x, T y, T out, T dout) {
+  inline HOSTDEVICE T Recompute(T x, T y, T out UNUSED, T dout) {
     return dout * d_binary_fun_.Dy(x, unary_fun_(y));
   }
 
   inline HOSTDEVICE T UseIntermediateOut(T x,
                                          T intermediate_out,
-                                         T out,
+                                         T out UNUSED,
                                          T dout) {
     return dout * d_binary_fun_.Dy(x, intermediate_out);
   }
diff --git a/paddle/phi/kernels/funcs/elementwise_functor.h b/paddle/phi/kernels/funcs/elementwise_functor.h
index 3543c0c6aa066..dc78bd7098411 100644
--- a/paddle/phi/kernels/funcs/elementwise_functor.h
+++ b/paddle/phi/kernels/funcs/elementwise_functor.h
@@ -18,6 +18,7 @@ limitations under the License. */
 #include "paddle/phi/common/float16.h"
 #include "paddle/phi/core/enforce.h"
 #include "paddle/phi/core/hostdevice.h"
+#include "paddle/phi/core/macros.h"
 #if defined(__xpu__)
 #include <xpu/runtime.h>
 
@@ -259,7 +260,7 @@ template <>
 struct FMaxGradDx<dtype::float16> {
   HOSTDEVICE dtype::float16 operator()(dtype::float16 x,
                                        dtype::float16 y,
-                                       dtype::float16 out,
+                                       dtype::float16 out UNUSED,
                                        dtype::float16 dout) const {
     return dout * static_cast<dtype::float16>((x >= y) || dtype::isnan(y));
   }
@@ -267,7 +268,7 @@ struct FMaxGradDx<dtype::float16> {
 
 template <>
 struct FMaxGradDx<int> {
-  HOSTDEVICE int operator()(int x, int y, int out, int dout) const {
+  HOSTDEVICE int operator()(int x, int y, int out UNUSED, int dout) const {
     return dout * static_cast<int>((x >= y));
   }
 };
@@ -276,7 +277,7 @@ template <>
 struct FMaxGradDx<int64_t> {
   HOSTDEVICE int64_t operator()(int64_t x,
                                 int64_t y,
-                                int64_t out,
+                                int64_t out UNUSED,
                                 int64_t dout) const {
     return dout * static_cast<int64_t>((x >= y));
   }
@@ -293,7 +294,7 @@ template <>
 struct FMaxGradDy<dtype::float16> {
   HOSTDEVICE dtype::float16 operator()(dtype::float16 x,
                                        dtype::float16 y,
-                                       dtype::float16 out,
+                                       dtype::float16 out UNUSED,
                                        dtype::float16 dout) const {
     return dout * static_cast<dtype::float16>(!((x >= y) || dtype::isnan(y)));
   }
@@ -303,7 +304,7 @@ template <>
 struct FMaxGradDy<int64_t> {
   HOSTDEVICE int64_t operator()(int64_t x,
                                 int64_t y,
-                                int64_t out,
+                                int64_t out UNUSED,
                                 int64_t dout) const {
     return dout * static_cast<int64_t>(!((x >= y)));
   }
@@ -311,7 +312,7 @@ struct FMaxGradDy<int64_t> {
 
 template <>
 struct FMaxGradDy<int> {
-  HOSTDEVICE int operator()(int x, int y, int out, int dout) const {
+  HOSTDEVICE int operator()(int x, int y, int out UNUSED, int dout) const {
     return dout * static_cast<int>(!((x >= y)));
   }
 };
@@ -327,7 +328,7 @@ template <>
 struct FMinGradDx<dtype::float16> {
   HOSTDEVICE dtype::float16 operator()(dtype::float16 x,
                                        dtype::float16 y,
-                                       dtype::float16 out,
+                                       dtype::float16 out UNUSED,
                                        dtype::float16 dout) const {
     return dout * static_cast<dtype::float16>((x <= y) || dtype::isnan(y));
   }
@@ -335,7 +336,7 @@ struct FMinGradDx<dtype::float16> {
 
 template <>
 struct FMinGradDx<int> {
-  HOSTDEVICE int operator()(int x, int y, int out, int dout) const {
+  HOSTDEVICE int operator()(int x, int y, int out UNUSED, int dout) const {
     return dout * static_cast<int>((x <= y));
   }
 };
@@ -344,7 +345,7 @@ template <>
 struct FMinGradDx<int64_t> {
   HOSTDEVICE int64_t operator()(int64_t x,
                                 int64_t y,
-                                int64_t out,
+                                int64_t out UNUSED,
                                 int64_t dout) const {
     return dout * static_cast<int64_t>((x <= y));
   }
@@ -361,7 +362,7 @@ template <>
 struct FMinGradDy<dtype::float16> {
   HOSTDEVICE dtype::float16 operator()(dtype::float16 x,
                                        dtype::float16 y,
-                                       dtype::float16 out,
+                                       dtype::float16 out UNUSED,
                                        dtype::float16 dout) const {
     return dout * static_cast<dtype::float16>(!((x <= y) || dtype::isnan(y)));
   }
@@ -369,7 +370,7 @@ struct FMinGradDy<dtype::float16> {
 
 template <>
 struct FMinGradDy<int> {
-  HOSTDEVICE int operator()(int x, int y, int out, int dout) const {
+  HOSTDEVICE int operator()(int x, int y, int out UNUSED, int dout) const {
     return dout * static_cast<int>(!((x <= y)));
   }
 };
@@ -378,7 +379,7 @@ template <>
 struct FMinGradDy<int64_t> {
   HOSTDEVICE int64_t operator()(int64_t x,
                                 int64_t y,
-                                int64_t out,
+                                int64_t out UNUSED,
                                 int64_t dout) const {
     return dout * static_cast<int64_t>(!((x <= y)));
   }
diff --git a/paddle/phi/kernels/funcs/for_range.h b/paddle/phi/kernels/funcs/for_range.h
index 4625414e7a5be..9648a7d845ff0 100644
--- a/paddle/phi/kernels/funcs/for_range.h
+++ b/paddle/phi/kernels/funcs/for_range.h
@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 #include "paddle/phi/backends/all_context.h"
 #include "paddle/phi/backends/gpu/gpu_launch_config.h"
-
+#include "paddle/phi/core/macros.h"
 namespace phi {
 namespace funcs {
 
@@ -29,7 +29,8 @@ struct ForRange {
 
 template <>
 struct ForRange<phi::CPUContext> {
-  ForRange(const phi::CPUContext& dev_ctx, size_t limit) : limit_(limit) {}
+  ForRange(const phi::CPUContext& dev_ctx UNUSED, size_t limit)
+      : limit_(limit) {}
 
   template <typename Function>
   void operator()(Function func) const {
diff --git a/paddle/phi/kernels/funcs/reduce_functor.h b/paddle/phi/kernels/funcs/reduce_functor.h
index e0e7ec3d403f1..f1d3772cc25a5 100644
--- a/paddle/phi/kernels/funcs/reduce_functor.h
+++ b/paddle/phi/kernels/funcs/reduce_functor.h
@@ -14,9 +14,9 @@
 
 #pragma once
 
+#include "paddle/phi/core/macros.h"
 #include "paddle/phi/kernels/funcs/eigen/common.h"
 #include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
-
 namespace phi {
 namespace funcs {
 
@@ -130,12 +130,12 @@ struct SumGradFunctor {
             typename DY,
             typename Dim>
   void operator()(const DeviceContext& place,
-                  X* x,
-                  Y* y,
+                  X* x UNUSED,
+                  Y* y UNUSED,
                   DX* dx,
                   DY* dy,
                   const Dim& dim,
-                  int size) {
+                  int size UNUSED) {
     dx->device(place) = dy->broadcast(dim);
   }
 };
@@ -171,7 +171,7 @@ struct MaxOrMinGradFunctor {
                   DX* dx,
                   DY* dy,
                   const Dim& dim,
-                  int size) {
+                  int size UNUSED) {
     auto equals = (*x) == y->broadcast(dim);
     auto ones = dx->constant(1);
     auto zeros = dx->constant(0);
diff --git a/paddle/phi/kernels/impl/meshgrid_grad_kernel_impl.h b/paddle/phi/kernels/impl/meshgrid_grad_kernel_impl.h
index 386bb1b47ef6d..bdedcee095707 100644
--- a/paddle/phi/kernels/impl/meshgrid_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/meshgrid_grad_kernel_impl.h
@@ -15,15 +15,15 @@
 #pragma once
 
 #include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/macros.h"
 #include "paddle/phi/kernels/funcs/eigen/common.h"
 #include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
 #include "paddle/phi/kernels/meshgrid_grad_kernel.h"
-
 namespace phi {
 
 template <typename T, typename Context, int Rank>
 void MeshgridBackward(const Context& ctx,
-                      const std::vector<const DenseTensor*>& ins,
+                      const std::vector<const DenseTensor*>& ins UNUSED,
                       const std::vector<const DenseTensor*>& out_grad,
                       std::vector<DenseTensor*> outs) {
   int n = out_grad.size();
diff --git a/paddle/phi/kernels/impl/reduce_grad.h b/paddle/phi/kernels/impl/reduce_grad.h
index e9d1aec0f09c5..5665c9713c476 100644
--- a/paddle/phi/kernels/impl/reduce_grad.h
+++ b/paddle/phi/kernels/impl/reduce_grad.h
@@ -14,10 +14,10 @@
 
 #pragma once
 
+#include "paddle/phi/core/macros.h"
 #include "paddle/phi/kernels/cast_kernel.h"
 #include "paddle/phi/kernels/empty_kernel.h"
 #include "paddle/phi/kernels/funcs/reduce_grad_functions.h"
-
 namespace phi {
 
 template <typename Context,
@@ -27,11 +27,11 @@ template <typename Context,
           bool kNoNeedBufferY = false>
 void ComputeFromInput(const Context& dev_ctx,
                       const DenseTensor& x,
-                      const DenseTensor& out_grad,
+                      const DenseTensor& out_grad UNUSED,
                       const paddle::optional<DenseTensor>& out,
                       const DenseTensor& input2,
                       const std::vector<int64_t>& dims,
-                      bool keep_dim,
+                      bool keep_dim UNUSED,
                       bool reduce_all,
                       DenseTensor* x_grad) {
   reduce_all = recompute_reduce_all(x, dims, reduce_all);
diff --git a/paddle/phi/kernels/impl/set_value_grad_kernel_impl.h b/paddle/phi/kernels/impl/set_value_grad_kernel_impl.h
index 1292a3af36352..3d2a0a3d0db67 100644
--- a/paddle/phi/kernels/impl/set_value_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/set_value_grad_kernel_impl.h
@@ -54,7 +54,7 @@ void SetValueGradImpl(const Context& dev_ctx,
                       const IntArray& steps,
                       const std::vector<int64_t>& axes,
                       const std::vector<int64_t>& decrease_axes,
-                      const std::vector<int64_t>& none_axes,
+                      const std::vector<int64_t>& none_axes UNUSED,
                       DenseTensor* x_grad,
                       DenseTensor* value_grad) {
   PADDLE_ENFORCE_EQ(
diff --git a/paddle/phi/kernels/onednn/conv_handler.h b/paddle/phi/kernels/onednn/conv_handler.h
index 102686d24ad4a..2be0ba5649711 100644
--- a/paddle/phi/kernels/onednn/conv_handler.h
+++ b/paddle/phi/kernels/onednn/conv_handler.h
@@ -17,8 +17,8 @@
 #include "paddle/phi/backends/onednn/onednn_helper.h"
 #include "paddle/phi/backends/onednn/onednn_reuse.h"
 #include "paddle/phi/core/expect.h"
+#include "paddle/phi/core/macros.h"
 #include "paddle/phi/kernels/cpu/conv_util.h"
-
 namespace phi {
 namespace onednn {
 
@@ -50,7 +50,7 @@ class ConvOneDNNHandlerT
                      const std::string& padding_algorithm,
                      const std::vector<int>& dilations_in,
                      int groups,
-                     const std::string& data_format,
+                     const std::string& data_format UNUSED,
                      bool is_test,
                      bool is_BFLOAT16,
                      const std::string& fuse_activation,

From 8386417ed2e08f465ee69492b4ef3131afe47b78 Mon Sep 17 00:00:00 2001
From: lijialin03 <124568209+lijialin03@users.noreply.github.com>
Date: Wed, 26 Apr 2023 16:20:52 +0800
Subject: [PATCH 081/405] add paddle.optimizer.LBFGS API and a modify its test
 case test=develop (#51912)

* modify numel in lbfgs and add a new test case. test=develop

* change param 'lr' to 'learning_rate' in lbfgs and its test

* add opt LBFGS and change test
---
 .../fluid/tests/unittests/test_lbfgs_class.py | 560 ++++++++++++++
 .../fluid/tests/unittests/test_lbfgs_v2.py    | 274 -------
 python/paddle/incubate/optimizer/lbfgs.py     |  21 +-
 python/paddle/optimizer/__init__.py           |   2 +
 python/paddle/optimizer/lbfgs.py              | 701 ++++++++++++++++++
 5 files changed, 1275 insertions(+), 283 deletions(-)
 create mode 100644 python/paddle/fluid/tests/unittests/test_lbfgs_class.py
 delete mode 100644 python/paddle/fluid/tests/unittests/test_lbfgs_v2.py
 create mode 100644 python/paddle/optimizer/lbfgs.py

diff --git a/python/paddle/fluid/tests/unittests/test_lbfgs_class.py b/python/paddle/fluid/tests/unittests/test_lbfgs_class.py
new file mode 100644
index 0000000000000..44c854f2119ef
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_lbfgs_class.py
@@ -0,0 +1,560 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+import paddle
+from paddle.incubate.optimizer import lbfgs as incubate_lbfgs
+from paddle.incubate.optimizer import line_search_dygraph
+from paddle.optimizer import lbfgs
+
+np.random.seed(123)
+
+# func()should be func(w, x)where w is parameter to be optimize ,x is input of optimizer func
+# np_w is the init parameter of w
+
+
+class Net(paddle.nn.Layer):
+    def __init__(self, np_w, func):
+        super().__init__()
+        self.func = func
+        w = paddle.to_tensor(np_w)
+        self.w = paddle.create_parameter(
+            shape=w.shape,
+            dtype=w.dtype,
+            default_initializer=paddle.nn.initializer.Assign(w),
+        )
+
+    def forward(self, x):
+        return self.func(self.w, x)
+
+
+def train_step(inputs, targets, net, opt):
+    def closure():
+        outputs = net(inputs)
+        loss = paddle.nn.functional.mse_loss(outputs, targets)
+        opt.clear_grad()
+        loss.backward()
+        return loss
+
+    loss = opt.step(closure)
+    return loss
+
+
+class TestLbfgs(unittest.TestCase):
+    def test_function_fix_incubate(self):
+        paddle.disable_static()
+        np_w = np.random.rand(1).astype(np.float32)
+
+        input = np.random.rand(1).astype(np.float32)
+        weights = [np.random.rand(1).astype(np.float32) for i in range(5)]
+        targets = [weights[i] * input for i in range(5)]
+
+        def func(w, x):
+            return w * x
+
+        net = Net(np_w, func)
+        opt = incubate_lbfgs.LBFGS(
+            learning_rate=1,
+            max_iter=10,
+            max_eval=None,
+            tolerance_grad=1e-07,
+            tolerance_change=1e-09,
+            history_size=5,
+            line_search_fn='strong_wolfe',
+            parameters=net.parameters(),
+        )
+
+        for weight, target in zip(weights, targets):
+            input = paddle.to_tensor(input)
+            target = paddle.to_tensor(target)
+            loss = 1
+            while loss > 1e-4:
+                loss = train_step(input, target, net, opt)
+            np.testing.assert_allclose(net.w, weight, rtol=1e-05)
+
+    def test_inf_minima_incubate(self):
+        # not converage
+        input = np.random.rand(1).astype(np.float32)
+
+        def outputs1(x):
+            # weight[0] = 1.01 weight[1] = 0.99
+            return x * x * x - 3 * x * x + 3 * 1.01 * 0.99 * x
+
+        def outputs2(x):
+            # weight[0] = 4 weight[1] = 2
+            return pow(x, 4) + 5 * pow(x, 2)
+
+        targets = [outputs1(input), outputs2(input)]
+        input = paddle.to_tensor(input)
+
+        def func1(extream_point, x):
+            return (
+                x * x * x
+                - 3 * x * x
+                + 3 * extream_point[0] * extream_point[1] * x
+            )
+
+        def func2(extream_point, x):
+            return pow(x, extream_point[0]) + 5 * pow(x, extream_point[1])
+
+        extream_point = np.array([-2.34, 1.45]).astype('float32')
+        net1 = Net(extream_point, func1)
+        # converge of old_sk.pop()
+        opt1 = incubate_lbfgs.LBFGS(
+            learning_rate=1,
+            max_iter=10,
+            max_eval=None,
+            tolerance_grad=1e-07,
+            tolerance_change=1e-09,
+            history_size=1,
+            line_search_fn='strong_wolfe',
+            parameters=net1.parameters(),
+        )
+
+        net2 = Net(extream_point, func2)
+        # converge of line_search = None
+        opt2 = incubate_lbfgs.LBFGS(
+            learning_rate=1,
+            max_iter=50,
+            max_eval=None,
+            tolerance_grad=1e-07,
+            tolerance_change=1e-09,
+            history_size=10,
+            line_search_fn=None,
+            parameters=net2.parameters(),
+        )
+
+        n_iter = 0
+        while n_iter < 20:
+            loss = train_step(input, paddle.to_tensor(targets[0]), net1, opt1)
+            n_iter = opt1.state_dict()["state"]["func_evals"]
+
+        n_iter = 0
+        while n_iter < 10:
+            loss = train_step(input, paddle.to_tensor(targets[1]), net2, opt2)
+            n_iter = opt1.state_dict()["state"]["func_evals"]
+
+    def test_error_incubate(self):
+        # test parameter is not Paddle Tensor
+        def error_func1():
+            extream_point = np.array([-1, 2]).astype('float32')
+            extream_point = paddle.to_tensor(extream_point)
+            return incubate_lbfgs.LBFGS(
+                learning_rate=1,
+                max_iter=10,
+                max_eval=None,
+                tolerance_grad=1e-07,
+                tolerance_change=1e-09,
+                history_size=3,
+                line_search_fn='strong_wolfe',
+                parameters=extream_point,
+            )
+
+        self.assertRaises(TypeError, error_func1)
+
+    def test_error2_incubate(self):
+        # not converage
+        input = np.random.rand(1).astype(np.float32)
+
+        def outputs2(x):
+            # weight[0] = 4 weight[1] = 2
+            return pow(x, 4) + 5 * pow(x, 2)
+
+        targets = [outputs2(input)]
+        input = paddle.to_tensor(input)
+
+        def func2(extream_point, x):
+            return pow(x, extream_point[0]) + 5 * pow(x, extream_point[1])
+
+        extream_point = np.array([-2.34, 1.45]).astype('float32')
+        net2 = Net(extream_point, func2)
+        # converge of line_search = None
+        opt2 = incubate_lbfgs.LBFGS(
+            learning_rate=1,
+            max_iter=50,
+            max_eval=None,
+            tolerance_grad=1e-07,
+            tolerance_change=1e-09,
+            history_size=10,
+            line_search_fn='None',
+            parameters=net2.parameters(),
+        )
+
+        def error_func():
+            n_iter = 0
+            while n_iter < 10:
+                loss = train_step(
+                    input, paddle.to_tensor(targets[0]), net2, opt2
+                )
+                n_iter = opt2.state_dict()["state"]["func_evals"]
+
+        self.assertRaises(RuntimeError, error_func)
+
+    def test_line_search_incubate(self):
+        def func1(x, alpha, d):
+            return paddle.to_tensor(x + alpha * d), paddle.to_tensor([0.0])
+
+        def func2(x, alpha, d):
+            return paddle.to_tensor(x + alpha * d), paddle.to_tensor([1.0])
+
+        def func3(x, alpha, d):
+            return paddle.to_tensor(x + alpha * d), paddle.to_tensor([-1.0])
+
+        line_search_dygraph._strong_wolfe(
+            func1,
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([0.001]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([0.0]),
+            max_ls=1,
+        )
+
+        line_search_dygraph._strong_wolfe(
+            func1,
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([0.001]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([0.0]),
+            max_ls=0,
+        )
+
+        line_search_dygraph._strong_wolfe(
+            func2,
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([-0.001]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            max_ls=1,
+        )
+
+        line_search_dygraph._strong_wolfe(
+            func3,
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([-0.001]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            max_ls=1,
+        )
+
+        line_search_dygraph._cubic_interpolate(
+            paddle.to_tensor([2.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([2.0]),
+            paddle.to_tensor([0.0]),
+            [0.1, 0.5],
+        )
+
+        line_search_dygraph._cubic_interpolate(
+            paddle.to_tensor([2.0]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([-3.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([-0.1]),
+            [0.1, 0.5],
+        )
+
+    def test_error3_incubate(self):
+        # test parameter shape size <= 0
+        def error_func3():
+            extream_point = np.array([-1, 2]).astype('float32')
+            extream_point = paddle.to_tensor(extream_point)
+
+            def func(w, x):
+                return w * x
+
+            net = Net(extream_point, func)
+            net.w = paddle.create_parameter(
+                shape=[-1, 2],
+                dtype=net.w.dtype,
+            )
+            opt = incubate_lbfgs.LBFGS(
+                learning_rate=1,
+                max_iter=10,
+                max_eval=None,
+                tolerance_grad=1e-07,
+                tolerance_change=1e-09,
+                history_size=5,
+                line_search_fn='strong_wolfe',
+                parameters=net.parameters(),
+            )
+
+        self.assertRaises(AssertionError, error_func3)
+
+    def test_function_fix(self):
+        paddle.disable_static()
+        np_w = np.random.rand(1).astype(np.float32)
+
+        input = np.random.rand(1).astype(np.float32)
+        weights = [np.random.rand(1).astype(np.float32) for i in range(5)]
+        targets = [weights[i] * input for i in range(5)]
+
+        def func(w, x):
+            return w * x
+
+        net = Net(np_w, func)
+        opt = lbfgs.LBFGS(
+            learning_rate=1,
+            max_iter=10,
+            max_eval=None,
+            tolerance_grad=1e-07,
+            tolerance_change=1e-09,
+            history_size=5,
+            line_search_fn='strong_wolfe',
+            parameters=net.parameters(),
+        )
+
+        for weight, target in zip(weights, targets):
+            input = paddle.to_tensor(input)
+            target = paddle.to_tensor(target)
+            loss = 1
+            while loss > 1e-4:
+                loss = train_step(input, target, net, opt)
+            np.testing.assert_allclose(net.w, weight, rtol=1e-05)
+
+    def test_inf_minima(self):
+        # not converage
+        input = np.random.rand(1).astype(np.float32)
+
+        def outputs1(x):
+            # weight[0] = 1.01 weight[1] = 0.99
+            return x * x * x - 3 * x * x + 3 * 1.01 * 0.99 * x
+
+        def outputs2(x):
+            # weight[0] = 4 weight[1] = 2
+            return pow(x, 4) + 5 * pow(x, 2)
+
+        targets = [outputs1(input), outputs2(input)]
+        input = paddle.to_tensor(input)
+
+        def func1(extream_point, x):
+            return (
+                x * x * x
+                - 3 * x * x
+                + 3 * extream_point[0] * extream_point[1] * x
+            )
+
+        def func2(extream_point, x):
+            return pow(x, extream_point[0]) + 5 * pow(x, extream_point[1])
+
+        extream_point = np.array([-2.34, 1.45]).astype('float32')
+        net1 = Net(extream_point, func1)
+        # converge of old_sk.pop()
+        opt1 = lbfgs.LBFGS(
+            learning_rate=1,
+            max_iter=10,
+            max_eval=None,
+            tolerance_grad=1e-07,
+            tolerance_change=1e-09,
+            history_size=1,
+            line_search_fn='strong_wolfe',
+            parameters=net1.parameters(),
+        )
+
+        net2 = Net(extream_point, func2)
+        # converge of line_search = None
+        opt2 = lbfgs.LBFGS(
+            learning_rate=1,
+            max_iter=50,
+            max_eval=None,
+            tolerance_grad=1e-07,
+            tolerance_change=1e-09,
+            history_size=10,
+            line_search_fn=None,
+            parameters=net2.parameters(),
+        )
+
+        n_iter = 0
+        while n_iter < 20:
+            loss = train_step(input, paddle.to_tensor(targets[0]), net1, opt1)
+            n_iter = opt1.state_dict()["state"]["func_evals"]
+
+        n_iter = 0
+        while n_iter < 10:
+            loss = train_step(input, paddle.to_tensor(targets[1]), net2, opt2)
+            n_iter = opt1.state_dict()["state"]["func_evals"]
+
+    def test_error(self):
+        # test parameter is not Paddle Tensor
+        def error_func1():
+            extream_point = np.array([-1, 2]).astype('float32')
+            extream_point = paddle.to_tensor(extream_point)
+            return lbfgs.LBFGS(
+                learning_rate=1,
+                max_iter=10,
+                max_eval=None,
+                tolerance_grad=1e-07,
+                tolerance_change=1e-09,
+                history_size=3,
+                line_search_fn='strong_wolfe',
+                parameters=extream_point,
+            )
+
+        self.assertRaises(TypeError, error_func1)
+
+    def test_error2(self):
+        # not converage
+        input = np.random.rand(1).astype(np.float32)
+
+        def outputs2(x):
+            # weight[0] = 4 weight[1] = 2
+            return pow(x, 4) + 5 * pow(x, 2)
+
+        targets = [outputs2(input)]
+        input = paddle.to_tensor(input)
+
+        def func2(extream_point, x):
+            return pow(x, extream_point[0]) + 5 * pow(x, extream_point[1])
+
+        extream_point = np.array([-2.34, 1.45]).astype('float32')
+        net2 = Net(extream_point, func2)
+        # converge of line_search = None
+        opt2 = lbfgs.LBFGS(
+            learning_rate=1,
+            max_iter=50,
+            max_eval=None,
+            tolerance_grad=1e-07,
+            tolerance_change=1e-09,
+            history_size=10,
+            line_search_fn='None',
+            parameters=net2.parameters(),
+        )
+
+        def error_func():
+            n_iter = 0
+            while n_iter < 10:
+                loss = train_step(
+                    input, paddle.to_tensor(targets[0]), net2, opt2
+                )
+                n_iter = opt2.state_dict()["state"]["func_evals"]
+
+        self.assertRaises(RuntimeError, error_func)
+
+    def test_line_search(self):
+        def func1(x, alpha, d):
+            return paddle.to_tensor(x + alpha * d), paddle.to_tensor([0.0])
+
+        def func2(x, alpha, d):
+            return paddle.to_tensor(x + alpha * d), paddle.to_tensor([1.0])
+
+        def func3(x, alpha, d):
+            return paddle.to_tensor(x + alpha * d), paddle.to_tensor([-1.0])
+
+        lbfgs._strong_wolfe(
+            func1,
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([0.001]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([0.0]),
+            max_ls=1,
+        )
+
+        lbfgs._strong_wolfe(
+            func1,
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([0.001]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([0.0]),
+            max_ls=0,
+        )
+
+        lbfgs._strong_wolfe(
+            func2,
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([-0.001]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            max_ls=1,
+        )
+
+        lbfgs._strong_wolfe(
+            func3,
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([-0.001]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            max_ls=1,
+        )
+
+        lbfgs._cubic_interpolate(
+            paddle.to_tensor([2.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([2.0]),
+            paddle.to_tensor([0.0]),
+            [0.1, 0.5],
+        )
+
+        lbfgs._cubic_interpolate(
+            paddle.to_tensor([2.0]),
+            paddle.to_tensor([0.0]),
+            paddle.to_tensor([-3.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([1.0]),
+            paddle.to_tensor([-0.1]),
+            [0.1, 0.5],
+        )
+
+    def test_error3(self):
+        # test parameter shape size <= 0
+        def error_func3():
+            extream_point = np.array([-1, 2]).astype('float32')
+            extream_point = paddle.to_tensor(extream_point)
+
+            def func(w, x):
+                return w * x
+
+            net = Net(extream_point, func)
+            net.w = paddle.create_parameter(
+                shape=[-1, 2],
+                dtype=net.w.dtype,
+            )
+            opt = lbfgs.LBFGS(
+                learning_rate=1,
+                max_iter=10,
+                max_eval=None,
+                tolerance_grad=1e-07,
+                tolerance_change=1e-09,
+                history_size=5,
+                line_search_fn='strong_wolfe',
+                parameters=net.parameters(),
+            )
+
+        self.assertRaises(AssertionError, error_func3)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/python/paddle/fluid/tests/unittests/test_lbfgs_v2.py b/python/paddle/fluid/tests/unittests/test_lbfgs_v2.py
deleted file mode 100644
index 9617938967cd3..0000000000000
--- a/python/paddle/fluid/tests/unittests/test_lbfgs_v2.py
+++ /dev/null
@@ -1,274 +0,0 @@
-# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-
-import numpy as np
-
-import paddle
-from paddle.incubate.optimizer import LBFGS
-from paddle.incubate.optimizer.line_search_dygraph import (
-    _cubic_interpolate,
-    _strong_wolfe,
-)
-
-np.random.seed(123)
-
-# func()should be func(w, x)where w is parameter to be optimize ,x is input of optimizer func
-# np_w is the init parameter of w
-
-
-class Net(paddle.nn.Layer):
-    def __init__(self, np_w, func):
-        super().__init__()
-        self.func = func
-        w = paddle.to_tensor(np_w)
-        self.w = paddle.create_parameter(
-            shape=w.shape,
-            dtype=w.dtype,
-            default_initializer=paddle.nn.initializer.Assign(w),
-        )
-
-    def forward(self, x):
-        return self.func(self.w, x)
-
-
-def train_step(inputs, targets, net, opt):
-    def closure():
-        outputs = net(inputs)
-        loss = paddle.nn.functional.mse_loss(outputs, targets)
-        opt.clear_grad()
-        loss.backward()
-        return loss
-
-    loss = opt.step(closure)
-    return loss
-
-
-class TestLbfgs(unittest.TestCase):
-    def test_function_fix(self):
-        paddle.disable_static()
-        np_w = np.random.rand(1).astype(np.float32)
-
-        input = np.random.rand(1).astype(np.float32)
-        weights = [np.random.rand(1).astype(np.float32) for i in range(5)]
-        targets = [weights[i] * input for i in range(5)]
-
-        def func(w, x):
-            return w * x
-
-        net = Net(np_w, func)
-        opt = LBFGS(
-            lr=1,
-            max_iter=10,
-            max_eval=None,
-            tolerance_grad=1e-07,
-            tolerance_change=1e-09,
-            history_size=5,
-            line_search_fn='strong_wolfe',
-            parameters=net.parameters(),
-        )
-
-        for weight, target in zip(weights, targets):
-            input = paddle.to_tensor(input)
-            target = paddle.to_tensor(target)
-            loss = 1
-            while loss > 1e-4:
-                loss = train_step(input, target, net, opt)
-            np.testing.assert_allclose(net.w, weight, rtol=1e-05)
-
-    def test_inf_minima(self):
-        # not converage
-        input = np.random.rand(1).astype(np.float32)
-
-        def outputs1(x):
-            # weight[0] = 1.01 weight[1] = 0.99
-            return x * x * x - 3 * x * x + 3 * 1.01 * 0.99 * x
-
-        def outputs2(x):
-            # weight[0] = 4 weight[1] = 2
-            return pow(x, 4) + 5 * pow(x, 2)
-
-        targets = [outputs1(input), outputs2(input)]
-        input = paddle.to_tensor(input)
-
-        def func1(extream_point, x):
-            return (
-                x * x * x
-                - 3 * x * x
-                + 3 * extream_point[0] * extream_point[1] * x
-            )
-
-        def func2(extream_point, x):
-            return pow(x, extream_point[0]) + 5 * pow(x, extream_point[1])
-
-        extream_point = np.array([-2.34, 1.45]).astype('float32')
-        net1 = Net(extream_point, func1)
-        # converge of old_sk.pop()
-        opt1 = LBFGS(
-            lr=1,
-            max_iter=10,
-            max_eval=None,
-            tolerance_grad=1e-07,
-            tolerance_change=1e-09,
-            history_size=1,
-            line_search_fn='strong_wolfe',
-            parameters=net1.parameters(),
-        )
-
-        net2 = Net(extream_point, func2)
-        # converge of line_search = None
-        opt2 = LBFGS(
-            lr=1,
-            max_iter=50,
-            max_eval=None,
-            tolerance_grad=1e-07,
-            tolerance_change=1e-09,
-            history_size=10,
-            line_search_fn=None,
-            parameters=net2.parameters(),
-        )
-
-        n_iter = 0
-        while n_iter < 20:
-            loss = train_step(input, paddle.to_tensor(targets[0]), net1, opt1)
-            n_iter = opt1.state_dict()["state"]["func_evals"]
-
-        n_iter = 0
-        while n_iter < 10:
-            loss = train_step(input, paddle.to_tensor(targets[1]), net2, opt2)
-            n_iter = opt1.state_dict()["state"]["func_evals"]
-
-    def test_error(self):
-        # test parameter is not Paddle Tensor
-        def error_func1():
-            extream_point = np.array([-1, 2]).astype('float32')
-            extream_point = paddle.to_tensor(extream_point)
-            return LBFGS(
-                lr=1,
-                max_iter=10,
-                max_eval=None,
-                tolerance_grad=1e-07,
-                tolerance_change=1e-09,
-                history_size=3,
-                line_search_fn='strong_wolfe',
-                parameters=extream_point,
-            )
-
-        self.assertRaises(TypeError, error_func1)
-
-    def test_error2(self):
-        # not converage
-        input = np.random.rand(1).astype(np.float32)
-
-        def outputs2(x):
-            # weight[0] = 4 weight[1] = 2
-            return pow(x, 4) + 5 * pow(x, 2)
-
-        targets = [outputs2(input)]
-        input = paddle.to_tensor(input)
-
-        def func2(extream_point, x):
-            return pow(x, extream_point[0]) + 5 * pow(x, extream_point[1])
-
-        extream_point = np.array([-2.34, 1.45]).astype('float32')
-        net2 = Net(extream_point, func2)
-        # converge of line_search = None
-        opt2 = LBFGS(
-            lr=1,
-            max_iter=50,
-            max_eval=None,
-            tolerance_grad=1e-07,
-            tolerance_change=1e-09,
-            history_size=10,
-            line_search_fn='None',
-            parameters=net2.parameters(),
-        )
-
-        def error_func():
-            n_iter = 0
-            while n_iter < 10:
-                loss = train_step(
-                    input, paddle.to_tensor(targets[0]), net2, opt2
-                )
-                n_iter = opt2.state_dict()["state"]["func_evals"]
-
-        self.assertRaises(RuntimeError, error_func)
-
-    def test_line_search(self):
-        def func1(x, alpha, d):
-            return paddle.to_tensor(x + alpha * d), paddle.to_tensor([0.0])
-
-        def func2(x, alpha, d):
-            return paddle.to_tensor(x + alpha * d), paddle.to_tensor([1.0])
-
-        def func3(x, alpha, d):
-            return paddle.to_tensor(x + alpha * d), paddle.to_tensor([-1.0])
-
-        _strong_wolfe(
-            func1,
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([0.001]),
-            paddle.to_tensor([0.0]),
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([0.0]),
-            paddle.to_tensor([0.0]),
-            max_ls=0,
-        )
-
-        _strong_wolfe(
-            func2,
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([-0.001]),
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([1.0]),
-            max_ls=1,
-        )
-
-        _strong_wolfe(
-            func3,
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([-0.001]),
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([1.0]),
-            max_ls=1,
-        )
-
-        _cubic_interpolate(
-            paddle.to_tensor([2.0]),
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([0.0]),
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([2.0]),
-            paddle.to_tensor([0.0]),
-            [0.1, 0.5],
-        )
-
-        _cubic_interpolate(
-            paddle.to_tensor([2.0]),
-            paddle.to_tensor([0.0]),
-            paddle.to_tensor([-3.0]),
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([1.0]),
-            paddle.to_tensor([-0.1]),
-            [0.1, 0.5],
-        )
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/python/paddle/incubate/optimizer/lbfgs.py b/python/paddle/incubate/optimizer/lbfgs.py
index 937a3b2f9af3f..ae7511ae03e94 100644
--- a/python/paddle/incubate/optimizer/lbfgs.py
+++ b/python/paddle/incubate/optimizer/lbfgs.py
@@ -18,10 +18,12 @@
 
 import paddle
 from paddle.optimizer import Optimizer
+from paddle.utils import deprecated
 
 from .line_search_dygraph import _strong_wolfe
 
 
+@deprecated(since="2.5.0", update_to="paddle.optimizer.LBFGS", level=1)
 class LBFGS(Optimizer):
     r"""
     The L-BFGS is a quasi-Newton method for solving an unconstrained optimization problem over a differentiable function.
@@ -40,7 +42,7 @@ class LBFGS(Optimizer):
         Jorge Nocedal, Stephen J. Wright, Numerical Optimization, Second Edition, 2006. pp179: Algorithm 7.5 (L-BFGS).
 
     Args:
-        lr (float, optional): learning rate .The default value is 1.
+        learning_rate (float, optional): learning rate .The default value is 1.
         max_iter (int, optional): maximal number of iterations per optimization step.
             The default value is 20.
         max_eval (int, optional): maximal number of function evaluations per optimization
@@ -97,7 +99,7 @@ def forward(self, x):
                     return self.w * x
 
             net = Net()
-            opt = LBFGS(lr=1, max_iter=1, max_eval=None, tolerance_grad=1e-07, tolerance_change=1e-09, history_size=100, line_search_fn='strong_wolfe', parameters=net.parameters())
+            opt = LBFGS(learning_rate=1, max_iter=1, max_eval=None, tolerance_grad=1e-07, tolerance_change=1e-09, history_size=100, line_search_fn='strong_wolfe', parameters=net.parameters())
             def train_step(inputs, targets):
                 def closure():
                     outputs = net(inputs)
@@ -118,7 +120,7 @@ def closure():
 
     def __init__(
         self,
-        lr=1.0,
+        learning_rate=1.0,
         max_iter=20,
         max_eval=None,
         tolerance_grad=1e-7,
@@ -133,7 +135,7 @@ def __init__(
         if max_eval is None:
             max_eval = max_iter * 5 // 4
 
-        self.lr = lr
+        self.learning_rate = learning_rate
         self.max_iter = max_iter
         self.max_eval = max_eval
         self.tolerance_grad = tolerance_grad
@@ -202,7 +204,7 @@ def _gather_flat_grad(self):
     def _add_grad(self, alpha, direction):
         offset = 0
         for p in self._params:
-            numel = p.numel().item()
+            numel = reduce(lambda x, y: x * y, p.shape)
             p = paddle.assign(
                 p.add(
                     direction[offset : offset + numel].reshape(p.shape) * alpha
@@ -234,11 +236,10 @@ def step(self, closure):
         """
 
         with paddle.no_grad():
-
             # Make sure the closure is always called with grad enabled
             closure = paddle.enable_grad()(closure)
 
-            lr = self.lr
+            learning_rate = self.learning_rate
             max_iter = self.max_iter
             max_eval = self.max_eval
             tolerance_grad = self.tolerance_grad
@@ -342,9 +343,11 @@ def step(self, closure):
                 ############################################################
                 # reset initial guess for step size
                 if state['n_iter'] == 1:
-                    alpha = min(1.0, 1.0 / flat_grad.abs().sum()) * lr
+                    alpha = (
+                        min(1.0, 1.0 / flat_grad.abs().sum()) * learning_rate
+                    )
                 else:
-                    alpha = lr
+                    alpha = learning_rate
 
                 # directional derivative
                 gtd = flat_grad.dot(d)
diff --git a/python/paddle/optimizer/__init__.py b/python/paddle/optimizer/__init__.py
index cef51897b20ab..7d9737dc7da1f 100644
--- a/python/paddle/optimizer/__init__.py
+++ b/python/paddle/optimizer/__init__.py
@@ -22,6 +22,7 @@
 from .sgd import SGD  # noqa: F401
 from .momentum import Momentum  # noqa: F401
 from .lamb import Lamb  # noqa: F401
+from .lbfgs import LBFGS  # noqa: F401
 from . import lr  # noqa: F401
 
 __all__ = [  # noqa
@@ -35,4 +36,5 @@
     'SGD',
     'Momentum',
     'Lamb',
+    'LBFGS',
 ]
diff --git a/python/paddle/optimizer/lbfgs.py b/python/paddle/optimizer/lbfgs.py
new file mode 100644
index 0000000000000..aaa933541962f
--- /dev/null
+++ b/python/paddle/optimizer/lbfgs.py
@@ -0,0 +1,701 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import defaultdict
+from functools import reduce
+
+import paddle
+
+from ..fluid import framework
+from .optimizer import Optimizer
+
+__all__ = []
+
+
+def _cubic_interpolate(x1, f1, g1, x2, f2, g2, bounds=None):
+    r"""Cubic interpolation between (x1, f1, g1) and (x2, f2, g2).
+        Use two points and their gradient to determine a cubic function and get the minimun point
+        between them in the cubic curve.
+
+    Reference:
+        Jorge Nocedal, Stephen J. Wright, Numerical Optimization, Second Edition, 2006.
+        pp59: formula 3.59
+
+    Args:
+        x1, f1, g1: point1's position, value and gradient.
+        x2, f2, g2: point2's position, value and gradient.
+        bounds: bounds of interpolation area
+
+    Returns:
+        min_pos: the minimun point between the specified points in the cubic curve.
+    """
+    # Compute bounds of interpolation area
+    if bounds is not None:
+        xmin_bound, xmax_bound = bounds
+    else:
+        xmin_bound, xmax_bound = (x1, x2) if x1 <= x2 else (x2, x1)
+
+    d1 = g1 + g2 - 3 * (f1 - f2) / (x1 - x2)
+    d2_square = d1**2 - g1 * g2
+    if d2_square >= 0:
+        d2 = d2_square.sqrt()
+        if x1 <= x2:
+            min_pos = x2 - (x2 - x1) * ((g2 + d2 - d1) / (g2 - g1 + 2 * d2))
+        else:
+            min_pos = x1 - (x1 - x2) * ((g1 + d2 - d1) / (g1 - g2 + 2 * d2))
+        return min(max(min_pos, xmin_bound), xmax_bound)
+    else:
+        return (xmin_bound + xmax_bound) / 2.0
+
+
+def _strong_wolfe(
+    obj_func,
+    xk,
+    alpha,
+    d,
+    loss,
+    grad,
+    gtd,
+    c1=1e-4,
+    c2=0.9,
+    tolerance_change=1e-9,
+    max_ls=25,
+):
+    r"""Implements of line search algorithm that satisfies the strong Wolfe conditions using double zoom.
+
+    Reference:
+        Jorge Nocedal, Stephen J. Wright, Numerical Optimization, Second Edition, 2006.
+        pp60: Algorithm 3.5 (Line Search Algorithm).
+
+    Args:
+        obj_func: the objective function to minimize. ```` accepts a multivariate input and returns a scalar.
+        xk (Tensor): the starting point of the iterates.
+        alpha (Scalar): the initial step size.
+        d (Tensor): search direction.
+        loss (scalar): the initial loss
+        grad (Tensor): the initial grad
+        c1 (Scalar): parameter for sufficient decrease condition.
+        c2 (Scalar): parameter for curvature condition.
+        tolerance_change (Scalar): terminates if the change of function value/position/parameter between
+            two iterations is smaller than this value.
+        max_ls(int): max iteration of line search.
+        alpha_max (float): max step length.
+
+    Returns:
+        loss_new (Scaler): loss of obj_func at final alpha.
+        grad_new, (Tensor): derivative of obj_func at final alpha.
+        alpha(Tensor): optimal step length, or 0. if the line search algorithm did not converge.
+        ls_func_evals (Scaler): number of objective function called in line search process.
+
+    Following summarizes the essentials of the strong Wolfe line search algorithm.
+    Some notations used in the description:
+
+        - `func` denotes the objective function.
+        - `obi_func` is a function of step size alpha, restricting `obj_func` on a line.
+
+            obi_func = func(xk + alpha * d),
+            where xk is the position of k'th iterate, d is the line search direction(decent direction),
+            and a is the step size.
+        - alpha : substitute of alpha
+        - a1 is alpha of last iteration, which is alpha_(i-1).
+        - a2 is alpha of current iteration, which is alpha_i.
+        - a_lo is alpha in left position when calls zoom, which is alpha_low.
+        - a_hi is alpha in right position when calls zoom, which is alpha_high.
+
+    Line Search Algorithm:
+        repeat
+            Compute obi_func(a2) and derphi(a2).
+            1. If obi_func(a2) > obi_func(0) + c_1 * a2 * obi_func'(0) or [obi_func(a2) >= obi_func(a1) and i > 1],
+                alpha= zoom(a1, a2) and stop;
+
+            2. If |obi_func'(a2)| <= -c_2 * obi_func'(0),
+                alpha= a2 and stop;
+
+            3. If obi_func'(a2) >= 0,
+                alpha= zoom(a2, a1) and stop;
+
+            a1 = a2
+            a2 = min(2 * a2, a2)
+            i = i + 1
+        end(repeat)
+
+    zoom(a_lo, a_hi) Algorithm:
+        repeat
+            aj = cubic_interpolation(a_lo, a_hi)
+            Compute obi_func(aj) and derphi(aj).
+            1. If obi_func(aj) > obi_func(0) + c_1 * aj * obi_func'(0) or obi_func(aj) >= obi_func(a_lo),
+                then a_hi <- aj;
+            2.
+                2.1. If |obi_func'(aj)| <= -c_2 * obi_func'(0), then alpha= a2 and stop;
+
+                2.2. If obi_func'(aj) * (a2 - a1) >= 0, then a_hi = a_lo
+
+                a_lo = aj;
+        end(repeat)
+    """
+
+    d_norm = d.abs().max()
+    grad = grad.clone()
+    # evaluate objective and gradient using initial step
+    loss_new, grad_new = obj_func(xk, alpha, d)
+    ls_func_evals = 1
+    gtd_new = paddle.dot(grad_new, d)
+
+    # bracket an interval containing a point satisfying the Wolfe criteria
+    t_prev, f_prev, g_prev, gtd_prev = (
+        paddle.to_tensor(0, dtype=grad.dtype),
+        loss,
+        grad,
+        gtd,
+    )
+    done = False
+    ls_iter = 0
+    while ls_iter < max_ls:
+        # check conditions
+        if loss_new > (loss + c1 * alpha * gtd) or (
+            ls_iter > 1 and loss_new >= f_prev
+        ):
+            bracket = [t_prev, alpha]
+            bracket_f = [f_prev, loss_new]
+            bracket_g = [g_prev, grad_new.clone()]
+            bracket_gtd = [gtd_prev, gtd_new]
+            break
+
+        if paddle.abs(gtd_new) <= -c2 * gtd:
+            bracket = [alpha]
+            bracket_f = [loss_new]
+            bracket_g = [grad_new]
+            done = True
+            break
+
+        if gtd_new >= 0:
+            bracket = [t_prev, alpha]
+            bracket_f = [f_prev, loss_new]
+            bracket_g = [g_prev, grad_new.clone()]
+            bracket_gtd = [gtd_prev, gtd_new]
+            break
+
+        # interpolate
+        min_step = alpha + 0.01 * (alpha - t_prev)
+        max_step = alpha * 10
+        tmp = alpha
+        alpha = _cubic_interpolate(
+            t_prev,
+            f_prev,
+            gtd_prev,
+            alpha,
+            loss_new,
+            gtd_new,
+            bounds=(min_step, max_step),
+        )
+
+        # next step
+        t_prev = tmp
+        f_prev = loss_new
+        g_prev = grad_new.clone()
+        gtd_prev = gtd_new
+
+        loss_new, grad_new = obj_func(xk, alpha, d)
+        ls_func_evals += 1
+        gtd_new = grad_new.dot(d)
+        ls_iter += 1
+
+    # reached max number of iterations?
+    if ls_iter == max_ls:
+        bracket = [0, alpha]
+        bracket_f = [loss, loss_new]
+        bracket_g = [grad, grad_new]
+
+    # zoom phase: we now have a point satisfying the criteria, or
+    # a bracket around it. We refine the bracket until we find the
+    # exact point satisfying the criteria
+    insuf_progress = False
+    # find high and low points in bracket
+    low_pos, high_pos = (0, 1) if bracket_f[0] <= bracket_f[-1] else (1, 0)
+    while not done and ls_iter < max_ls:
+        # line-search bracket is so small
+        if paddle.abs(bracket[1] - bracket[0]) * d_norm < tolerance_change:
+            break
+
+        # compute new trial value
+        alpha = _cubic_interpolate(
+            bracket[0],
+            bracket_f[0],
+            bracket_gtd[0],
+            bracket[1],
+            bracket_f[1],
+            bracket_gtd[1],
+        )
+
+        # test that we are making sufficient progress:
+        # in case `alpha` is so close to boundary, we mark that we are making
+        # insufficient progress, and if
+        #   + we have made insufficient progress in the last step, or
+        #   + `alpha` is at one of the boundary,
+        # we will move `alpha` to a position which is `0.1 * len(bracket)`
+        # away from the nearest boundary point.
+
+        eps = 0.1 * (max(bracket) - min(bracket))
+        if min(max(bracket) - alpha, alpha - min(bracket)) < eps:
+            # interpolation close to boundary
+            if insuf_progress or alpha >= max(bracket) or alpha <= min(bracket):
+                # evaluate at 0.1 away from boundary
+                if paddle.abs(alpha - max(bracket)) < paddle.abs(
+                    alpha - min(bracket)
+                ):
+                    alpha = max(bracket) - eps
+                else:
+                    alpha = min(bracket) + eps
+                insuf_progress = False
+            else:
+                insuf_progress = True
+        else:
+            insuf_progress = False
+        # Evaluate new point
+        loss_new, grad_new = obj_func(xk, alpha, d)
+        ls_func_evals += 1
+        gtd_new = grad_new.dot(d)
+        ls_iter += 1
+
+        if (
+            loss_new > (loss + c1 * alpha * gtd)
+            or loss_new >= bracket_f[low_pos]
+        ):
+            # Armijo condition not satisfied or not lower than lowest point
+            bracket[high_pos] = alpha
+            bracket_f[high_pos] = loss_new
+            # bracket_g[high_pos] = grad_new.clone(memory_format=torch.contiguous_format)
+            bracket_g[high_pos] = grad_new.clone()
+            bracket_gtd[high_pos] = gtd_new
+            low_pos, high_pos = (
+                (0, 1) if bracket_f[0] <= bracket_f[1] else (1, 0)
+            )
+        else:
+            if paddle.abs(gtd_new) <= -c2 * gtd:
+                # Wolfe conditions satisfied
+                done = True
+            elif gtd_new * (bracket[high_pos] - bracket[low_pos]) >= 0:
+                # old high becomes new low
+                bracket[high_pos] = bracket[low_pos]
+                bracket_f[high_pos] = bracket_f[low_pos]
+                bracket_g[high_pos] = bracket_g[low_pos]
+                bracket_gtd[high_pos] = bracket_gtd[low_pos]
+
+            # new point becomes new low
+            bracket[low_pos] = alpha
+            bracket_f[low_pos] = loss_new
+            bracket_g[low_pos] = grad_new.clone()
+            bracket_gtd[low_pos] = gtd_new
+
+    # return stuff
+    alpha = bracket[low_pos]
+    loss_new = bracket_f[low_pos]
+    grad_new = bracket_g[low_pos]
+    return loss_new, grad_new, alpha, ls_func_evals
+
+
+class LBFGS(Optimizer):
+    r"""
+    The L-BFGS is a quasi-Newton method for solving an unconstrained optimization problem over a differentiable function.
+    Closely related is the Newton method for minimization. Consider the iterate update formula:
+
+    .. math::
+        x_{k+1} = x_{k} + H_k \nabla{f_k}
+
+    If :math:`H_k` is the inverse Hessian of :math:`f` at :math:`x_k`, then it's the Newton method.
+    If :math:`H_k` is symmetric and positive definite, used as an approximation of the inverse Hessian, then
+    it's a quasi-Newton. In practice, the approximated Hessians are obtained
+    by only using the gradients, over either whole or part of the search
+    history, the former is BFGS, the latter is L-BFGS.
+
+    Reference:
+        Jorge Nocedal, Stephen J. Wright, Numerical Optimization, Second Edition, 2006. pp179: Algorithm 7.5 (L-BFGS).
+
+    Args:
+        learning_rate (float, optional): learning rate .The default value is 1.
+        max_iter (int, optional): maximal number of iterations per optimization step.
+            The default value is 20.
+        max_eval (int, optional): maximal number of function evaluations per optimization
+            step. The default value is max_iter * 1.25.
+        tolerance_grad (float, optional): termination tolerance on first order optimality
+            The default value is 1e-5.
+        tolerance_change (float, optional): termination tolerance on function
+            value/parameter changes. The default value is 1e-9.
+        history_size (int, optional): update history size. The default value is 100.
+        line_search_fn (string, optional): either 'strong_wolfe' or None. The default value is strong_wolfe.
+        parameters (list|tuple, optional): List/Tuple of ``Tensor`` names to update to minimize ``loss``. \
+            This parameter is required in dygraph mode. The default value is None.
+        weight_decay (float|WeightDecayRegularizer, optional): The strategy of regularization. \
+            It canbe a float value as coeff of L2 regularization or \
+            :ref:`api_fluid_regularizer_L1Decay`, :ref:`api_fluid_regularizer_L2Decay`.
+            If a parameter has set regularizer using :ref:`api_fluid_ParamAttr` already, \
+            the regularization setting here in optimizer will be ignored for this parameter. \
+            Otherwise, the regularization setting here in optimizer will take effect. \
+            Default None, meaning there is no regularization.
+        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of \
+            some derived class of ``GradientClipBase`` . There are three cliping strategies \
+            ( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` , \
+            :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
+        name (str, optional): Normally there is no need for user to set this property.
+            For more information, please refer to :ref:`api_guide_Name`.
+            The default value is None.
+
+    Return:
+        loss (Tensor): the final loss of closure.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+            import numpy as np
+            from paddle.incubate.optimizer import LBFGS
+
+            paddle.disable_static()
+            np.random.seed(0)
+            np_w = np.random.rand(1).astype(np.float32)
+            np_x = np.random.rand(1).astype(np.float32)
+
+            inputs = [np.random.rand(1).astype(np.float32) for i in range(10)]
+            # y = 2x
+            targets = [2 * x for x in inputs]
+
+            class Net(paddle.nn.Layer):
+                def __init__(self):
+                    super().__init__()
+                    w = paddle.to_tensor(np_w)
+                    self.w = paddle.create_parameter(shape=w.shape, dtype=w.dtype, default_initializer=paddle.nn.initializer.Assign(w))
+
+                def forward(self, x):
+                    return self.w * x
+
+            net = Net()
+            opt = LBFGS(learning_rate=1, max_iter=1, max_eval=None, tolerance_grad=1e-07, tolerance_change=1e-09, history_size=100, line_search_fn='strong_wolfe', parameters=net.parameters())
+            def train_step(inputs, targets):
+                def closure():
+                    outputs = net(inputs)
+                    loss = paddle.nn.functional.mse_loss(outputs, targets)
+                    print('loss: ', loss.item())
+                    opt.clear_grad()
+                    loss.backward()
+                    return loss
+                opt.step(closure)
+
+
+            for input, target in zip(inputs, targets):
+                input = paddle.to_tensor(input)
+                target = paddle.to_tensor(target)
+                train_step(input, target)
+
+    """
+
+    def __init__(
+        self,
+        learning_rate=1.0,
+        max_iter=20,
+        max_eval=None,
+        tolerance_grad=1e-7,
+        tolerance_change=1e-9,
+        history_size=100,
+        line_search_fn=None,
+        parameters=None,
+        weight_decay=None,
+        grad_clip=None,
+        name=None,
+    ):
+        if max_eval is None:
+            max_eval = max_iter * 5 // 4
+
+        self.learning_rate = learning_rate
+        self.max_iter = max_iter
+        self.max_eval = max_eval
+        self.tolerance_grad = tolerance_grad
+        self.tolerance_change = tolerance_change
+        self.history_size = history_size
+        self.line_search_fn = line_search_fn
+
+        if isinstance(parameters, paddle.Tensor):
+            raise TypeError(
+                "parameters argument given to the optimizer should be "
+                "an iterable of Tensors or dicts, but got " + type(parameters)
+            )
+
+        self.state = defaultdict(dict)
+
+        super().__init__(
+            learning_rate=1.0,
+            parameters=parameters,
+            weight_decay=weight_decay,
+            grad_clip=grad_clip,
+            name=name,
+        )
+
+        if not isinstance(self._parameter_list[0], dict):
+            self._params = self._parameter_list
+        else:
+            for idx, param_group in enumerate(self._param_groups):
+                self._params = param_group['params']
+
+        self._numel_cache = None
+
+    def state_dict(self):
+        r"""Returns the state of the optimizer as a :class:`dict`.
+
+        Return:
+            state, a dict holding current optimization state. Its content
+                differs between optimizer classes.
+        """
+
+        packed_state = {}
+        for k, v in self.state.items():
+            packed_state.update({k: v})
+
+        return {'state': packed_state}
+
+    def _numel(self):
+        # compute the number of all parameters
+        if self._numel_cache is None:
+            self._numel_cache = reduce(
+                lambda total, p: total + p.numel(), self._params, 0
+            )
+        return self._numel_cache
+
+    # flatten grad of all parameters
+    def _gather_flat_grad(self):
+        views = []
+        for p in self._params:
+            if p.grad is None:
+                view = paddle.zeros_like(p).reshape([-1])
+            else:
+                view = p.grad.reshape([-1])
+            views.append(view)
+        return paddle.concat(views, axis=0)
+
+    # compute xk = xk + alpha * direction
+    def _add_grad(self, alpha, direction):
+        offset = 0
+        for p in self._params:
+            numel = reduce(lambda x, y: x * y, p.shape)
+            p = paddle.assign(
+                p.add(
+                    direction[offset : offset + numel].reshape(p.shape) * alpha
+                ),
+                p,
+            )
+            offset += numel
+        assert offset == self._numel()
+
+    def _clone_param(self):
+        return [p.clone() for p in self._params]
+
+    def _set_param(self, params_data):
+        for p, pdata in zip(self._params, params_data):
+            paddle.assign(pdata, p)
+
+    def _directional_evaluate(self, closure, x, alpha, d):
+        self._add_grad(alpha, d)
+        loss = float(closure())
+        flat_grad = self._gather_flat_grad()
+        self._set_param(x)
+        return loss, flat_grad
+
+    @framework.non_static_only
+    def step(self, closure):
+        """Performs a single optimization step.
+        Args:
+            closure (callable): A closure that reevaluates the model
+                and returns the loss.
+        """
+
+        with paddle.no_grad():
+            # Make sure the closure is always called with grad enabled
+            closure = paddle.enable_grad()(closure)
+
+            learning_rate = self.learning_rate
+            max_iter = self.max_iter
+            max_eval = self.max_eval
+            tolerance_grad = self.tolerance_grad
+            tolerance_change = self.tolerance_change
+            line_search_fn = self.line_search_fn
+            history_size = self.history_size
+            state = self.state
+            state.setdefault('func_evals', 0)
+            state.setdefault('n_iter', 0)
+
+            # evaluate initial f(x) and df/dx
+            orig_loss = closure()
+            loss = float(orig_loss)
+
+            current_evals = 1
+            state['func_evals'] += 1
+
+            flat_grad = self._gather_flat_grad()
+            opt_cond = flat_grad.abs().max() <= tolerance_grad
+
+            # optimal condition
+            if opt_cond:
+                return orig_loss
+
+            # tensors cached in state (for tracing)
+            d = state.get('d')
+            alpha = state.get('alpha')
+            old_yk = state.get('old_yk')
+            old_sk = state.get('old_sk')
+            ro = state.get('ro')
+            H_diag = state.get('H_diag')
+            prev_flat_grad = state.get('prev_flat_grad')
+            prev_loss = state.get('prev_loss')
+
+            n_iter = 0
+            # optimize for a max of max_iter iterations
+            while n_iter < max_iter:
+                # keep track of nb of iterations
+                n_iter += 1
+                state['n_iter'] += 1
+
+                ############################################################
+                # compute gradient descent direction
+                ############################################################
+                if state['n_iter'] == 1:
+                    d = flat_grad.neg()
+                    old_yk = []
+                    old_sk = []
+                    ro = []
+                    H_diag = paddle.to_tensor(1.0, dtype=orig_loss.dtype)
+                else:
+                    # do lbfgs update (update memory)
+                    y = flat_grad.subtract(prev_flat_grad)
+                    s = d.multiply(paddle.to_tensor(alpha, dtype=d.dtype))
+                    ys = y.dot(s)
+                    if ys > 1e-10:
+                        # updating memory
+                        if len(old_yk) == history_size:
+                            # shift history by one (limited-memory)
+                            old_yk.pop(0)
+                            old_sk.pop(0)
+                            ro.pop(0)
+
+                        # store new direction/step
+                        old_yk.append(y)
+                        old_sk.append(s)
+                        ro.append(1.0 / ys)
+
+                        # update scale of initial Hessian approximation
+                        H_diag = ys / y.dot(y)  # (y*y)
+
+                    # compute the approximate (L-BFGS) inverse Hessian
+                    # multiplied by the gradient
+                    num_old = len(old_yk)
+
+                    if 'al' not in state:
+                        state['al'] = [None] * history_size
+                    al = state['al']
+
+                    # iteration in L-BFGS loop collapsed to use just one buffer
+                    q = flat_grad.neg()
+                    for i in range(num_old - 1, -1, -1):
+                        al[i] = old_sk[i].dot(q) * ro[i]
+                        paddle.assign(q.add(old_yk[i] * (-al[i])), q)
+
+                    # multiply by initial Hessian
+                    # r/d is the final direction
+                    d = r = paddle.multiply(q, H_diag)
+                    for i in range(num_old):
+                        be_i = old_yk[i].dot(r) * ro[i]
+                        paddle.assign(r.add(old_sk[i] * (al[i] - be_i)), r)
+
+                if prev_flat_grad is None:
+                    prev_flat_grad = flat_grad.clone()
+                else:
+                    paddle.assign(flat_grad, prev_flat_grad)
+                prev_loss = loss
+
+                ############################################################
+                # compute step length
+                ############################################################
+                # reset initial guess for step size
+                if state['n_iter'] == 1:
+                    alpha = (
+                        min(1.0, 1.0 / flat_grad.abs().sum()) * learning_rate
+                    )
+                else:
+                    alpha = learning_rate
+
+                # directional derivative
+                gtd = flat_grad.dot(d)
+
+                # directional derivative is below tolerance
+                if gtd > -tolerance_change:
+                    break
+
+                # optional line search: user function
+                ls_func_evals = 0
+                if line_search_fn is not None:
+                    # perform line search, using user function
+                    if line_search_fn != "strong_wolfe":
+                        raise RuntimeError("only 'strong_wolfe' is supported")
+                    else:
+                        x_init = self._clone_param()
+
+                        def obj_func(x, alpha, d):
+                            return self._directional_evaluate(
+                                closure, x, alpha, d
+                            )
+
+                        loss, flat_grad, alpha, ls_func_evals = _strong_wolfe(
+                            obj_func, x_init, alpha, d, loss, flat_grad, gtd
+                        )
+                    self._add_grad(alpha, d)
+                    opt_cond = flat_grad.abs().max() <= tolerance_grad
+                else:
+                    # no line search, simply move with fixed-step
+                    self._add_grad(alpha, d)
+                    if n_iter != max_iter:
+                        with paddle.enable_grad():
+                            loss = float(closure())
+                        flat_grad = self._gather_flat_grad()
+                        opt_cond = flat_grad.abs().max() <= tolerance_grad
+                        ls_func_evals = 1
+
+                # update func eval
+                current_evals += ls_func_evals
+                state['func_evals'] += ls_func_evals
+
+                # optimal condition
+                if opt_cond:
+                    break
+
+                # lack of progress
+                if (d * alpha).abs().max() <= tolerance_change:
+                    break
+
+                if abs(loss - prev_loss) < tolerance_change:
+                    break
+
+                # check conditions
+                if current_evals >= max_eval:
+                    break
+
+                if n_iter == max_iter:
+                    break
+
+            state['d'] = d
+            state['alpha'] = alpha
+            state['old_yk'] = old_yk
+            state['old_sk'] = old_sk
+            state['ro'] = ro
+            state['H_diag'] = H_diag
+            state['prev_flat_grad'] = prev_flat_grad
+            state['prev_loss'] = prev_loss
+
+        return orig_loss

From 1a790edf0804a0e22ad5bd99f01c00f2e3bcf74a Mon Sep 17 00:00:00 2001
From: zqw_1997 <118182234+zhengqiwen1997@users.noreply.github.com>
Date: Wed, 26 Apr 2023 16:43:09 +0800
Subject: [PATCH 082/405] [Zero-Dim] Support output 0D for gather_nd, einsum. 
 (#53175)

* add test cases, test=allcase

* fix test cases, test=allcase

* fix test cases, test=allcase

* assert_allclose, test=allcase

* 1e-5 to 1e-4, test=allcase

* change rtol from 1e-4 to 1e-3, test=allcase
---
 .../tests/unittests/test_zero_dim_tensor.py   | 106 ++++++++++++++++++
 python/paddle/tensor/einsum.py                |   4 +-
 2 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
index 9c049ddbf435d..0e2449e7799c2 100644
--- a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
+++ b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
@@ -17,6 +17,7 @@
 # 0D Tensor's shape is always [], numel is 1
 # which can be created by paddle.rand([])
 
+import os
 import unittest
 
 import numpy as np
@@ -1719,6 +1720,75 @@ def test_gather_xD_axis_1(self):
         self.assertEqual(x.grad.shape, [2, 3])
         self.assertEqual(out.grad.shape, [2])
 
+    def test_gather_nd(self):
+        x1 = paddle.to_tensor([1.0, 3.0, 5.0, 7.0, 9.0], stop_gradient=False)
+        x2 = paddle.to_tensor(
+            [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], stop_gradient=False
+        )
+
+        index1 = paddle.full([1], 1, 'int64')
+        index2 = paddle.full([2], 1, 'int64')
+
+        out1 = paddle.gather_nd(x1, index1)
+        out2 = paddle.gather_nd(x2, index2)
+
+        out1.retain_grads()
+        out2.retain_grads()
+
+        out1.backward()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out2.shape, [])
+        np.testing.assert_array_equal(out1, np.array(3.0))
+        np.testing.assert_array_equal(out2, np.array(5.0))
+        self.assertEqual(x1.grad.shape, [5])
+        self.assertEqual(x2.grad.shape, [2, 3])
+        self.assertEqual(out1.grad.shape, [])
+        self.assertEqual(out2.grad.shape, [])
+
+    def test_einsum(self):
+        os.environ['FLAGS_new_einsum'] = "0"
+        x = paddle.rand([5])
+        # sum
+        out1 = paddle.einsum('i->', x)
+        expect1 = np.einsum('i->', x)
+        # dot
+        out2 = paddle.einsum('i,i->', x, x)
+        expect2 = np.einsum('i,i->', x, x)
+
+        out1.retain_grads()
+        out2.retain_grads()
+
+        out1.backward()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out2.shape, [])
+        np.testing.assert_allclose(out1, expect1, rtol=1e-03)
+        np.testing.assert_allclose(out2, expect2, rtol=1e-03)
+
+    def test_einsum_V2(self):
+        os.environ['FLAGS_new_einsum'] = "1"
+        x = paddle.rand([5])
+        # sum
+        out1 = paddle.einsum('i->', x)
+        expect1 = np.einsum('i->', x)
+        # dot
+        out2 = paddle.einsum('i,i->', x, x)
+        expect2 = np.einsum('i,i->', x, x)
+
+        out1.retain_grads()
+        out2.retain_grads()
+
+        out1.backward()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out2.shape, [])
+        np.testing.assert_allclose(out1, expect1, rtol=1e-03)
+        np.testing.assert_allclose(out2, expect2, rtol=1e-03)
+
     def test_scatter_1D(self):
         x = paddle.to_tensor([1.0, 3.0, 5.0, 7.0, 9.0], stop_gradient=False)
         index = paddle.full([], 2, 'int64')
@@ -3520,6 +3590,42 @@ def test_gather_XD_axis_1(self):
         self.assertEqual(res[1].shape, (2, 3))
         self.assertEqual(res[2].shape, (2,))
 
+    @prog_scope()
+    def test_gather_nd(self):
+        x1 = paddle.full([10], 1.0, 'float32')
+        x1.stop_gradient = False
+        x2 = paddle.full([2, 3], 1.0, 'float32')
+        x2.stop_gradient = False
+
+        index1 = paddle.full([1], 1, 'int64')
+        index2 = paddle.full([2], 1, 'int64')
+
+        out1 = paddle.gather_nd(x1, index1)
+        out2 = paddle.gather_nd(x2, index2)
+        paddle.static.append_backward(out1.sum())
+        paddle.static.append_backward(out2.sum())
+
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(
+            prog,
+            fetch_list=[
+                out1,
+                out2,
+                x1.grad_name,
+                x2.grad_name,
+                out1.grad_name,
+                out2.grad_name,
+            ],
+        )
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, ())
+        np.testing.assert_array_equal(res[0], 1.0)
+        np.testing.assert_array_equal(res[1], 1.0)
+        self.assertEqual(res[2].shape, (10,))
+        self.assertEqual(res[3].shape, (2, 3))
+        self.assertEqual(res[4].shape, ())
+        self.assertEqual(res[5].shape, ())
+
     @prog_scope()
     def test_scatter_1D(self):
         x = paddle.full([10], 1.0, 'float32')
diff --git a/python/paddle/tensor/einsum.py b/python/paddle/tensor/einsum.py
index 7ab104dde94b0..082300763740a 100644
--- a/python/paddle/tensor/einsum.py
+++ b/python/paddle/tensor/einsum.py
@@ -966,8 +966,8 @@ def einsum(equation, *operands):
 
             # dot
             print(paddle.einsum('i,i->', x, x))
-            # Tensor(shape=[1], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
-            #   [1.45936954])
+            # Tensor(shape=[], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
+            #   1.45936954)
 
             # outer
             print(paddle.einsum("i,j->ij", x, y))

From e72cad592400d00009bc208fbbec9a532698b7cd Mon Sep 17 00:00:00 2001
From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com>
Date: Wed, 26 Apr 2023 17:02:44 +0800
Subject: [PATCH 083/405] modify approve rules (#53270)

* delete prim flag for matmul_2_grad

* delete prim flag for matmul_2_grad

* recover multiply prune

* add rules

* add rules

* add rules

* modify approve rules

* fix conflict

* recover block
---
 tools/check_file_diff_approvals.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/check_file_diff_approvals.sh b/tools/check_file_diff_approvals.sh
index b67a343b8ea8f..55efe2d47c395 100644
--- a/tools/check_file_diff_approvals.sh
+++ b/tools/check_file_diff_approvals.sh
@@ -82,9 +82,10 @@ API_FILES=("CMakeLists.txt"
            "paddle/phi/core/infermeta_utils.h"
            "paddle/fluid/prim/api/composite_backward/composite_backward_api.h"
            "paddle/fluid/prim/api/manual_prim/prim_manual_api.h"
+           "paddle/fluid/prim/api/api.yaml"
            "python/paddle/incubate/autograd/composite_rules.py"
 	   "python/paddle/incubate/autograd/primitives.py"
-       "paddle/fluid/prim/api/api.yaml"
+
            )
 
 approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000`

From 1164626c7e9cae1f9ca468fe78493d59649c5689 Mon Sep 17 00:00:00 2001
From: Ruibiao Chen <chenruibiao@baidu.com>
Date: Wed, 26 Apr 2023 17:03:25 +0800
Subject: [PATCH 084/405] Fix fused_attention_op and fused_feedforward_op bugs
 in xpu (#53318)

* Fix fused_attention_op and fused_feedforward_op bugs in xpu

* Fix d_x alloc errors for fused_feedforward_grad_kernel
---
 ...rd_grad_xpu_kernel.cc => fused_feedforward_grad_kernel.cc} | 4 ++--
 ..._feedforward_xpu_kernel.cc => fused_feedforward_kernel.cc} | 2 +-
 paddle/phi/kernels/xpu/fused_attention_kernel.cc              | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)
 rename paddle/phi/kernels/fusion/xpu/{fused_feedforward_grad_xpu_kernel.cc => fused_feedforward_grad_kernel.cc} (99%)
 rename paddle/phi/kernels/fusion/xpu/{fused_feedforward_xpu_kernel.cc => fused_feedforward_kernel.cc} (99%)

diff --git a/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_xpu_kernel.cc b/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_kernel.cc
similarity index 99%
rename from paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_xpu_kernel.cc
rename to paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_kernel.cc
index cb10930dc9b3e..6798df360de19 100644
--- a/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_xpu_kernel.cc
+++ b/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_kernel.cc
@@ -478,7 +478,7 @@ void FusedFeedForwardGradKernel(
                                      dropout2_fix_seed,
                                      nullptr,
                                      dropout2_seed_val);
-
+  dev_ctx.template Alloc<T>(d_x);
   dev_ctx.template Alloc<float>(d_ln_scale);
   dev_ctx.template Alloc<float>(d_ln_bias);
   dev_ctx.template Alloc<T>(d_linear1_bias);
@@ -529,7 +529,7 @@ void FusedFeedForwardGradKernel(
 }  // namespace fusion
 }  // namespace phi
 
-PD_REGISTER_KERNEL(fused_feedward_grad,
+PD_REGISTER_KERNEL(fused_feedforward_grad,
                    XPU,
                    ALL_LAYOUT,
                    phi::fusion::FusedFeedForwardGradKernel,
diff --git a/paddle/phi/kernels/fusion/xpu/fused_feedforward_xpu_kernel.cc b/paddle/phi/kernels/fusion/xpu/fused_feedforward_kernel.cc
similarity index 99%
rename from paddle/phi/kernels/fusion/xpu/fused_feedforward_xpu_kernel.cc
rename to paddle/phi/kernels/fusion/xpu/fused_feedforward_kernel.cc
index 35039ba571e57..221305014190b 100644
--- a/paddle/phi/kernels/fusion/xpu/fused_feedforward_xpu_kernel.cc
+++ b/paddle/phi/kernels/fusion/xpu/fused_feedforward_kernel.cc
@@ -377,7 +377,7 @@ void FusedFeedForwardKernel(const Context& dev_ctx,
 }  // namespace fusion
 }  // namespace phi
 
-PD_REGISTER_KERNEL(fused_feedward,
+PD_REGISTER_KERNEL(fused_feedforward,
                    XPU,
                    ALL_LAYOUT,
                    phi::fusion::FusedFeedForwardKernel,
diff --git a/paddle/phi/kernels/xpu/fused_attention_kernel.cc b/paddle/phi/kernels/xpu/fused_attention_kernel.cc
index e91c109b37502..d18dda47866ef 100644
--- a/paddle/phi/kernels/xpu/fused_attention_kernel.cc
+++ b/paddle/phi/kernels/xpu/fused_attention_kernel.cc
@@ -181,12 +181,12 @@ void FusedAttentionKernel(const Context &dev_ctx,
   float *ln_mean_ptr =
       (ln_mean == nullptr)
           ? (nullptr)
-          : reinterpret_cast<float *>(dev_ctx.template Alloc<T>(ln_mean));
+          : reinterpret_cast<float *>(dev_ctx.template Alloc<float>(ln_mean));
 
   float *ln_var_ptr =
       (ln_var == nullptr)
           ? (nullptr)
-          : reinterpret_cast<float *>(dev_ctx.template Alloc<T>(ln_var));
+          : reinterpret_cast<float *>(dev_ctx.template Alloc<float>(ln_var));
 
   XPUTypeT *ln_out_ptr =
       (ln_out == nullptr)

From 55c4eb8a6c58a3d72d92b6b7c797037681e2a417 Mon Sep 17 00:00:00 2001
From: mhy-666 <57670156+mhy-666@users.noreply.github.com>
Date: Wed, 26 Apr 2023 17:21:59 +0800
Subject: [PATCH 085/405] =?UTF-8?q?=E3=80=90prim=E3=80=91scatter=5Fnd=5Fad?=
 =?UTF-8?q?d=5Fgrad=20(#52469)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add scatter_nd_add comp

* add scatter_nd_add prim

* fix

* fix

* add public_python_api in TestScatterNdAddSimpleOp setup function

* fix composite_backward_api.h

* fix composite_backward

* add test cases

* fix composite_backward_api.h, unittest
---
 .../composite_backward_api.h                  |  16 ++
 paddle/phi/api/yaml/backward.yaml             |   1 +
 .../tests/unittests/test_scatter_nd_op.py     | 247 ++++++++++--------
 3 files changed, 153 insertions(+), 111 deletions(-)

diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
index 5e1e490c1b73a..099ebc81b900b 100644
--- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
+++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
@@ -1805,5 +1805,21 @@ void roll_grad(const Tensor& x,
     set_output<T>(x_grad_output, x_grad);
   }
 }
+
+template <typename T>
+void scatter_nd_add_grad(const Tensor& index,
+                         const Tensor& updates,
+                         const Tensor& out_grad,
+                         Tensor* x_grad,
+                         Tensor* updates_grad) {
+  if (x_grad) {
+    by_pass<T>(out_grad, x_grad);
+  }
+  if (updates_grad) {
+    // Gradient by Gather: dUpdates = dO[Ids]
+    auto tmp_updates_grad = gather_nd<T>(out_grad, index);
+    set_output<T>(tmp_updates_grad, updates_grad);
+  }
+}
 }  // namespace prim
 }  // namespace paddle
diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index 2394182ee4bd1..6faf2d0ba7a49 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -1454,6 +1454,7 @@
   kernel :
     func : scatter_nd_add_grad
   no_need_buffer : updates
+  composite: scatter_nd_add_grad(index, updates, out_grad, x_grad, updates_grad)
 
 - backward_op : segment_pool_grad
   forward : segment_pool (Tensor x, Tensor segment_ids, str pooltype="SUM") -> Tensor(out), Tensor(summed_ids)
diff --git a/python/paddle/fluid/tests/unittests/test_scatter_nd_op.py b/python/paddle/fluid/tests/unittests/test_scatter_nd_op.py
index 0d09e0af5c32a..66799466c59e4 100644
--- a/python/paddle/fluid/tests/unittests/test_scatter_nd_op.py
+++ b/python/paddle/fluid/tests/unittests/test_scatter_nd_op.py
@@ -69,6 +69,8 @@ class TestScatterNdAddSimpleOp(OpTest):
     def setUp(self):
         self.op_type = "scatter_nd_add"
         self.python_api = paddle.scatter_nd_add
+        self.public_python_api = paddle.scatter_nd_add
+        self.prim_op_type = "prim"
         self._set_dtype()
         if self.dtype == np.float64:
             target_dtype = "float64"
@@ -94,7 +96,7 @@ def test_check_output(self):
         self.check_output()
 
     def test_check_grad(self):
-        self.check_grad(['X', 'Updates'], 'Out')
+        self.check_grad(['X', 'Updates'], 'Out', check_prim=True)
 
 
 class TestScatterNdAddSimpleFP16Op(TestScatterNdAddSimpleOp):
@@ -127,7 +129,9 @@ def test_check_output(self):
     def test_check_grad(self):
         if core.is_compiled_with_cuda():
             place = core.CUDAPlace(0)
-            self.check_grad_with_place(place, ['X', 'Updates'], 'Out')
+            self.check_grad_with_place(
+                place, ['X', 'Updates'], 'Out', check_prim=True
+            )
 
 
 class TestScatterNdAddWithEmptyIndex(OpTest):
@@ -138,6 +142,8 @@ class TestScatterNdAddWithEmptyIndex(OpTest):
     def setUp(self):
         self.op_type = "scatter_nd_add"
         self.python_api = paddle.scatter_nd_add
+        self.public_python_api = paddle.scatter_nd_add
+        self.prim_op_type = "prim"
         self._set_dtype()
         if self.dtype == np.float64:
             target_dtype = "float64"
@@ -166,7 +172,7 @@ def test_check_output(self):
         self.check_output()
 
     def test_check_grad(self):
-        self.check_grad(['X', 'Updates'], 'Out')
+        self.check_grad(['X', 'Updates'], 'Out', check_prim=True)
 
 
 class TestScatterNdAddWithEmptyIndexFP16(TestScatterNdAddWithEmptyIndex):
@@ -199,7 +205,9 @@ def test_check_output(self):
     def test_check_grad(self):
         if core.is_compiled_with_cuda():
             place = core.CUDAPlace(0)
-            self.check_grad_with_place(place, ['X', 'Updates'], 'Out')
+            self.check_grad_with_place(
+                place, ['X', 'Updates'], 'Out', check_prim=True
+            )
 
 
 class TestScatterNdAddWithHighRankSame(OpTest):
@@ -210,6 +218,8 @@ class TestScatterNdAddWithHighRankSame(OpTest):
     def setUp(self):
         self.op_type = "scatter_nd_add"
         self.python_api = paddle.scatter_nd_add
+        self.public_python_api = paddle.scatter_nd_add
+        self.prim_op_type = "prim"
         self._set_dtype()
         if self.dtype == np.float64:
             target_dtype = "float64"
@@ -241,7 +251,7 @@ def test_check_output(self):
         self.check_output()
 
     def test_check_grad(self):
-        self.check_grad(['X', 'Updates'], 'Out')
+        self.check_grad(['X', 'Updates'], 'Out', check_prim=True)
 
 
 class TestScatterNdAddWithHighRankSameFP16(TestScatterNdAddWithHighRankSame):
@@ -274,7 +284,9 @@ def test_check_output(self):
     def test_check_grad(self):
         if core.is_compiled_with_cuda():
             place = core.CUDAPlace(0)
-            self.check_grad_with_place(place, ['X', 'Updates'], 'Out')
+            self.check_grad_with_place(
+                place, ['X', 'Updates'], 'Out', check_prim=True
+            )
 
 
 class TestScatterNdAddWithHighRankDiff(OpTest):
@@ -285,6 +297,8 @@ class TestScatterNdAddWithHighRankDiff(OpTest):
     def setUp(self):
         self.op_type = "scatter_nd_add"
         self.python_api = paddle.scatter_nd_add
+        self.public_python_api = paddle.scatter_nd_add
+        self.prim_op_type = "prim"
         shape = (8, 2, 2, 1, 10)
         ref_np = np.random.rand(*shape).astype("double")
         index = np.vstack([np.random.randint(0, s, size=500) for s in shape]).T
@@ -300,7 +314,7 @@ def test_check_output(self):
         self.check_output()
 
     def test_check_grad(self):
-        self.check_grad(['X', 'Updates'], 'Out')
+        self.check_grad(['X', 'Updates'], 'Out', check_prim=True)
 
 
 # Test Python API
@@ -310,70 +324,76 @@ class TestScatterNdOpAPI(unittest.TestCase):
     """
 
     def testcase1(self):
-        ref1 = paddle.static.data(
-            name='ref1',
-            shape=[10, 9, 8, 1, 3],
-            dtype='float32',
-        )
-        index1 = paddle.static.data(
-            name='index1',
-            shape=[5, 5, 8, 5],
-            dtype='int32',
-        )
-        updates1 = paddle.static.data(
-            name='update1',
-            shape=[5, 5, 8],
-            dtype='float32',
-        )
-        output1 = paddle.scatter_nd_add(ref1, index1, updates1)
+        with paddle.fluid.framework._static_guard():
+            ref1 = paddle.static.data(
+                name='ref1',
+                shape=[10, 9, 8, 1, 3],
+                dtype='float32',
+            )
+            index1 = paddle.static.data(
+                name='index1',
+                shape=[5, 5, 8, 5],
+                dtype='int32',
+            )
+            updates1 = paddle.static.data(
+                name='update1',
+                shape=[5, 5, 8],
+                dtype='float32',
+            )
+            output1 = paddle.scatter_nd_add(ref1, index1, updates1)
 
     def testcase2(self):
-        ref2 = paddle.static.data(
-            name='ref2',
-            shape=[10, 9, 8, 1, 3],
-            dtype='double',
-        )
-        index2 = paddle.static.data(
-            name='index2',
-            shape=[5, 8, 5],
-            dtype='int32',
-        )
-        updates2 = paddle.static.data(
-            name='update2',
-            shape=[5, 8],
-            dtype='double',
-        )
-        output2 = paddle.scatter_nd_add(
-            ref2, index2, updates2, name="scatter_nd_add"
-        )
+        with paddle.fluid.framework._static_guard():
+            ref2 = paddle.static.data(
+                name='ref2',
+                shape=[10, 9, 8, 1, 3],
+                dtype='double',
+            )
+            index2 = paddle.static.data(
+                name='index2',
+                shape=[5, 8, 5],
+                dtype='int32',
+            )
+            updates2 = paddle.static.data(
+                name='update2',
+                shape=[5, 8],
+                dtype='double',
+            )
+            output2 = paddle.scatter_nd_add(
+                ref2, index2, updates2, name="scatter_nd_add"
+            )
 
     def testcase3(self):
-        shape3 = [10, 9, 8, 1, 3]
-        index3 = paddle.static.data(
-            name='index3',
-            shape=[5, 5, 8, 5],
-            dtype='int32',
-        )
-        updates3 = paddle.static.data(
-            name='update3',
-            shape=[5, 5, 8],
-            dtype='float32',
-        )
-        output3 = paddle.scatter_nd(index3, updates3, shape3)
+        with paddle.fluid.framework._static_guard():
+            shape3 = [10, 9, 8, 1, 3]
+            index3 = paddle.static.data(
+                name='index3',
+                shape=[5, 5, 8, 5],
+                dtype='int32',
+            )
+            updates3 = paddle.static.data(
+                name='update3',
+                shape=[5, 5, 8],
+                dtype='float32',
+            )
+            output3 = paddle.scatter_nd(index3, updates3, shape3)
 
     def testcase4(self):
-        shape4 = [10, 9, 8, 1, 3]
-        index4 = paddle.static.data(
-            name='index4',
-            shape=[5, 5, 8, 5],
-            dtype='int32',
-        )
-        updates4 = paddle.static.data(
-            name='update4',
-            shape=[5, 5, 8],
-            dtype='double',
-        )
-        output4 = paddle.scatter_nd(index4, updates4, shape4, name='scatter_nd')
+        with paddle.fluid.framework._static_guard():
+            shape4 = [10, 9, 8, 1, 3]
+            index4 = paddle.static.data(
+                name='index4',
+                shape=[5, 5, 8, 5],
+                dtype='int32',
+            )
+            updates4 = paddle.static.data(
+                name='update4',
+                shape=[5, 5, 8],
+                dtype='double',
+            )
+            output4 = paddle.scatter_nd(
+                index4, updates4, shape4, name='scatter_nd'
+            )
 
     def testcase5(self):
         if not fluid.core.is_compiled_with_cuda():
@@ -430,60 +450,65 @@ def test_static_graph():
 class TestScatterNdOpRaise(unittest.TestCase):
     def test_check_raise(self):
         def check_raise_is_test():
-            try:
-                ref5 = paddle.static.data(
-                    name='ref5', shape=[-1, 3, 4, 5], dtype='float32'
-                )
-                index5 = paddle.static.data(
-                    name='index5', shape=[-1, 2, 10], dtype='int32'
-                )
-                updates5 = paddle.static.data(
-                    name='updates5', shape=[-1, 2, 10], dtype='float32'
-                )
-                output5 = paddle.scatter_nd_add(ref5, index5, updates5)
-            except Exception as e:
-                t = "The last dimension of Input(Index)'s shape should be no greater "
-                if t in str(e):
-                    raise IndexError
+            with paddle.fluid.framework._static_guard():
+                try:
+                    ref5 = paddle.static.data(
+                        name='ref5', shape=[-1, 3, 4, 5], dtype='float32'
+                    )
+                    index5 = paddle.static.data(
+                        name='index5', shape=[-1, 2, 10], dtype='int32'
+                    )
+                    updates5 = paddle.static.data(
+                        name='updates5', shape=[-1, 2, 10], dtype='float32'
+                    )
+                    output5 = paddle.scatter_nd_add(ref5, index5, updates5)
+                except Exception as e:
+                    t = "The last dimension of Input(Index)'s shape should be no greater "
+                    if t in str(e):
+                        raise IndexError
 
         self.assertRaises(IndexError, check_raise_is_test)
 
     def test_check_raise2(self):
         with self.assertRaises(ValueError):
-            ref6 = paddle.static.data(
-                name='ref6',
-                shape=[10, 9, 8, 1, 3],
-                dtype='double',
-            )
-            index6 = paddle.static.data(
-                name='index6',
-                shape=[5, 8, 5],
-                dtype='int32',
-            )
-            updates6 = paddle.static.data(
-                name='update6',
-                shape=[5, 8],
-                dtype='float32',
-            )
-            output6 = paddle.scatter_nd_add(ref6, index6, updates6)
-
-    def test_check_raise3(self):
-        def check_raise_is_test():
-            try:
-                shape = [3, 4, 5]
-                index7 = paddle.static.data(
-                    name='index7', shape=[-1, 2, 1], dtype='int32'
+            with paddle.fluid.framework._static_guard():
+                ref6 = paddle.static.data(
+                    name='ref6',
+                    shape=[10, 9, 8, 1, 3],
+                    dtype='double',
+                )
+                index6 = paddle.static.data(
+                    name='index6',
+                    shape=[5, 8, 5],
+                    dtype='int32',
                 )
-                updates7 = paddle.static.data(
-                    name='updates7', shape=[-1, 2, 4, 5, 20], dtype='float32'
+                updates6 = paddle.static.data(
+                    name='update6',
+                    shape=[5, 8],
+                    dtype='float32',
                 )
-                output7 = paddle.scatter_nd(index7, updates7, shape)
-            except Exception as e:
-                t = "Updates has wrong shape"
-                if t in str(e):
-                    raise ValueError
+                output6 = paddle.scatter_nd_add(ref6, index6, updates6)
 
-        self.assertRaises(ValueError, check_raise_is_test)
+    def test_check_raise3(self):
+        def check_raise_is_test():
+            with paddle.fluid.framework._static_guard():
+                try:
+                    shape = [3, 4, 5]
+                    index7 = paddle.static.data(
+                        name='index7', shape=[-1, 2, 1], dtype='int32'
+                    )
+                    updates7 = paddle.static.data(
+                        name='updates7',
+                        shape=[-1, 2, 4, 5, 20],
+                        dtype='float32',
+                    )
+                    output7 = paddle.scatter_nd(index7, updates7, shape)
+                except Exception as e:
+                    t = "Updates has wrong shape"
+                    if t in str(e):
+                        raise ValueError
+
+            self.assertRaises(ValueError, check_raise_is_test)
 
 
 class TestDygraph(unittest.TestCase):

From 0e30d56a5330a13e5d5590d5c52e7ff05b487772 Mon Sep 17 00:00:00 2001
From: huangjiyi <43315610+huangjiyi@users.noreply.github.com>
Date: Wed, 26 Apr 2023 19:42:05 +0800
Subject: [PATCH 086/405] Add CMake option WITH_CPP_DIST for installing C++
 distribution (#53282)

* rm paddle_install_dir

* update test=document_fix

* update test=document_fix

* update

* add test

* update

* update

* update

* update

* update

* update

* update

* update

* update

* add todo commont
---
 CMakeLists.txt                         |  16 +++
 cmake/PaddleConfig.cmake.in            |  33 +++++
 cmake/inference_lib.cmake              | 161 +------------------------
 cmake/paddle_lib.cmake                 |  24 ++++
 cmake/version.cmake                    |  43 +++++++
 python/env_dict.py.in                  |   5 +-
 python/setup.py.in                     |  82 ++++++++++---
 setup.py                               |  57 +++++++++
 test/CMakeLists.txt                    |   9 ++
 test/paddle_lib/CMakeLists.txt.in      |  14 +++
 test/paddle_lib/test_paddle_lib.cc     |  27 +++++
 test/paddle_lib/test_paddle_lib_gpu.cc |  25 ++++
 12 files changed, 321 insertions(+), 175 deletions(-)
 create mode 100644 cmake/PaddleConfig.cmake.in
 create mode 100644 cmake/paddle_lib.cmake
 create mode 100644 test/paddle_lib/CMakeLists.txt.in
 create mode 100644 test/paddle_lib/test_paddle_lib.cc
 create mode 100644 test/paddle_lib/test_paddle_lib_gpu.cc

diff --git a/CMakeLists.txt b/CMakeLists.txt
index db92b63b41c7a..34110edff3038 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -246,6 +246,7 @@ option(WITH_DISTRIBUTE "Compile with distributed support" OFF)
 option(WITH_BRPC_RDMA "Use brpc rdma as the rpc protocal" OFF)
 option(ON_INFER "Turn on inference optimization and inference-lib generation"
        ON)
+option(WITH_CPP_DIST "Install PaddlePaddle C++ distribution" OFF)
 ################################ Internal Configurations #######################################
 option(WITH_NV_JETSON "Compile PaddlePaddle with NV JETSON" OFF)
 option(WITH_PROFILER "Compile PaddlePaddle with GPU profiler and gperftools"
@@ -662,6 +663,21 @@ if(WITH_STRIP)
   endif()
 endif()
 
+if(WITH_CPP_DIST)
+  # TODO(huangjiyi): Separate installing C++ distribution from python package
+  # installation and support for installing C++ distribution on more platforms.
+  if(NOT LINUX OR NOT WITH_PYTHON)
+    set(WITH_CPP_DIST
+        OFF
+        CACHE
+          STRING
+          "Currently C++ Distribution Generation is only available on Linux and compiling WITH_PYTHON=ON."
+          FORCE)
+  else()
+    include(paddle_lib)
+  endif()
+endif()
+
 add_subdirectory(paddle)
 if(WITH_PYTHON)
   add_subdirectory(python)
diff --git a/cmake/PaddleConfig.cmake.in b/cmake/PaddleConfig.cmake.in
new file mode 100644
index 0000000000000..d32c23f6f6edd
--- /dev/null
+++ b/cmake/PaddleConfig.cmake.in
@@ -0,0 +1,33 @@
+# Paddle CMake configuration file
+# -------
+#
+# Finds the Paddle library
+#
+# This will define the following variables:
+#
+#   PADDLE_FOUND        -- True if the system has the Paddle library
+#   PADDLE_INCLUDE_DIRS -- The include directories for Paddle
+#   PADDLE_LIBRARIES    -- Libraries to link against
+
+get_filename_component(PADDLE_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_FILE}/../.." ABSOLUTE)
+
+# include directories
+set(PADDLE_INCLUDE_DIRS 
+    ${PADDLE_INSTALL_PREFIX}/include
+    ${PADDLE_INSTALL_PREFIX}/include/third_party
+)
+
+# Library dependencies.
+set(PADDLE_LIBRARIES_DIRS ${PADDLE_INSTALL_PREFIX}/lib)
+link_directories(${PADDLE_LIBRARIES_DIRS})
+
+file(GLOB PADDLE_LIBRARIES ${PADDLE_LIBRARIES_DIRS}/lib*)
+
+find_package(PythonLibs @PY_VERSION@ REQUIRED)
+list(APPEND PADDLE_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS})
+list(APPEND PADDLE_LIBRARIES ${PYTHON_LIBRARIES})
+
+if(@WITH_GPU@)
+    find_package(CUDA @CUDA_VERSION@ REQUIRED)
+    list(APPEND PADDLE_LIBRARIES ${CUDA_LIBRARIES})
+endif()
diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake
index f5fc9b8b9cf8f..cb1fdf72b8e2a 100755
--- a/cmake/inference_lib.cmake
+++ b/cmake/inference_lib.cmake
@@ -12,11 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# make package for paddle fluid shared and static library
-set(PADDLE_INSTALL_DIR
-    "${CMAKE_BINARY_DIR}/paddle_install_dir"
-    CACHE STRING "A path setting paddle shared and static libraries")
-
+# make package for paddle inference shared and static library
 set(PADDLE_INFERENCE_INSTALL_DIR
     "${CMAKE_BINARY_DIR}/paddle_inference_install_dir"
     CACHE STRING "A path setting paddle inference shared and static libraries")
@@ -214,7 +210,7 @@ endfunction()
 # inference library for only inference
 set(inference_lib_deps third_party paddle_inference paddle_inference_c
                        paddle_inference_shared paddle_inference_c_shared)
-add_custom_target(inference_lib_dist DEPENDS ${inference_lib_deps})
+add_custom_target(inference_lib_dist ALL DEPENDS ${inference_lib_deps})
 
 set(dst_dir "${PADDLE_INFERENCE_INSTALL_DIR}/third_party/threadpool")
 copy(
@@ -387,158 +383,5 @@ if(WITH_STRIP AND NOT WIN32)
   )
 endif()
 
-# fluid library for both train and inference
-set(fluid_lib_deps inference_lib_dist)
-add_custom_target(fluid_lib_dist ALL DEPENDS ${fluid_lib_deps})
-
-set(dst_dir "${PADDLE_INSTALL_DIR}/paddle/fluid")
-set(module "inference")
-if(WIN32)
-  copy(
-    fluid_lib_dist
-    SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/api/paddle_*.h
-         ${paddle_inference_lib}
-    DSTS ${dst_dir}/${module} ${dst_dir}/${module} ${dst_dir}/${module}
-         ${dst_dir}/${module})
-else()
-  copy(
-    fluid_lib_dist
-    SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/api/paddle_*.h
-         ${paddle_inference_lib}
-    DSTS ${dst_dir}/${module} ${dst_dir}/${module} ${dst_dir}/${module})
-endif()
-
-set(module "framework")
-set(framework_lib_deps framework_proto data_feed_proto trainer_desc_proto)
-add_dependencies(fluid_lib_dist ${framework_lib_deps})
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/*.h
-       ${src_dir}/${module}/details/*.h
-       ${PADDLE_BINARY_DIR}/paddle/fluid/framework/trainer_desc.pb.h
-       ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.h
-       ${PADDLE_BINARY_DIR}/paddle/fluid/framework/data_feed.pb.h
-       ${src_dir}/${module}/ir/memory_optimize_pass/*.h
-       ${src_dir}/${module}/ir/*.h
-       ${src_dir}/${module}/fleet/*.h
-  DSTS ${dst_dir}/${module}
-       ${dst_dir}/${module}/details
-       ${dst_dir}/${module}
-       ${dst_dir}/${module}
-       ${dst_dir}/${module}
-       ${dst_dir}/${module}/ir/memory_optimize_pass
-       ${dst_dir}/${module}/ir
-       ${dst_dir}/${module}/fleet)
-
-set(module "operators")
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/reader/blocking_queue.h
-  DSTS ${dst_dir}/${module}/reader/)
-
-set(module "memory")
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/allocation/*.h
-  DSTS ${dst_dir}/${module}/allocation)
-
-set(module "platform")
-set(platform_lib_deps phi_profiler_proto errors)
-if(WITH_GPU)
-  set(platform_lib_deps ${platform_lib_deps} external_error_proto)
-endif()
-
-add_dependencies(fluid_lib_dist ${platform_lib_deps})
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/dynload/*.h
-       ${PADDLE_BINARY_DIR}/paddle/phi/api/profiler/*.pb.h
-  DSTS ${dst_dir}/${module} ${dst_dir}/${module}/dynload ${dst_dir}/${module})
-
-set(module "string")
-copy(
-  fluid_lib_dist
-  SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/${module}/*.h
-       ${PADDLE_SOURCE_DIR}/paddle/utils/${module}/tinyformat/*.h
-  DSTS ${dst_dir}/${module} ${dst_dir}/${module}/tinyformat)
-
-set(module "imperative")
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/jit/*.h
-  DSTS ${dst_dir}/${module} ${dst_dir}/${module}/jit)
-
-set(module "pybind")
-copy(
-  fluid_lib_dist
-  SRCS ${CMAKE_CURRENT_BINARY_DIR}/paddle/fluid/${module}/pybind.h
-  DSTS ${dst_dir}/${module})
-
-set(dst_dir "${PADDLE_INSTALL_DIR}/third_party/eigen3")
-copy(
-  inference_lib_dist
-  SRCS ${EIGEN_INCLUDE_DIR}/Eigen/Core ${EIGEN_INCLUDE_DIR}/Eigen/src
-       ${EIGEN_INCLUDE_DIR}/unsupported/Eigen
-  DSTS ${dst_dir}/Eigen ${dst_dir}/Eigen ${dst_dir}/unsupported)
-
-set(dst_dir "${PADDLE_INSTALL_DIR}/third_party/dlpack")
-copy(
-  inference_lib_dist
-  SRCS ${DLPACK_INCLUDE_DIR}/dlpack
-  DSTS ${dst_dir})
-
-set(dst_dir "${PADDLE_INSTALL_DIR}/third_party/install/zlib")
-copy(
-  inference_lib_dist
-  SRCS ${ZLIB_INCLUDE_DIR} ${ZLIB_LIBRARIES}
-  DSTS ${dst_dir} ${dst_dir}/lib)
-
-# CMakeCache Info
-copy(
-  fluid_lib_dist
-  SRCS ${PADDLE_INFERENCE_INSTALL_DIR}/third_party
-       ${CMAKE_CURRENT_BINARY_DIR}/CMakeCache.txt
-  DSTS ${PADDLE_INSTALL_DIR} ${PADDLE_INSTALL_DIR})
-
-# paddle fluid version
-function(version version_file)
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} log --pretty=format:%H -1
-    WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}
-    OUTPUT_VARIABLE PADDLE_GIT_COMMIT)
-  file(
-    WRITE ${version_file}
-    "GIT COMMIT ID: ${PADDLE_GIT_COMMIT}\n" "WITH_MKL: ${WITH_MKL}\n"
-    "WITH_MKLDNN: ${WITH_MKLDNN}\n" "WITH_GPU: ${WITH_GPU}\n"
-    "WITH_ROCM: ${WITH_ROCM}\n" "WITH_IPU: ${WITH_IPU}\n")
-  if(WITH_GPU)
-    file(APPEND ${version_file}
-         "CUDA version: ${CUDA_VERSION}\n"
-         "CUDNN version: v${CUDNN_MAJOR_VERSION}.${CUDNN_MINOR_VERSION}\n")
-  endif()
-  if(WITH_ROCM)
-    file(APPEND ${version_file}
-         "HIP version: v${HIP_MAJOR_VERSION}.${HIP_MINOR_VERSION}\n"
-         "MIOpen version: v${MIOPEN_MAJOR_VERSION}.${MIOPEN_MINOR_VERSION}\n")
-  endif()
-  if(WITH_IPU)
-    file(APPEND ${version_file} "PopART version: ${POPART_VERSION}\n")
-  endif()
-  file(APPEND ${version_file}
-       "CXX compiler version: ${CMAKE_CXX_COMPILER_VERSION}\n")
-  if(TENSORRT_FOUND)
-    file(
-      APPEND ${version_file}
-      "WITH_TENSORRT: ${TENSORRT_FOUND}\n"
-      "TensorRT version: v${TENSORRT_MAJOR_VERSION}.${TENSORRT_MINOR_VERSION}.${TENSORRT_PATCH_VERSION}.${TENSORRT_BUILD_VERSION}\n"
-    )
-  endif()
-  if(WITH_LITE)
-    file(APPEND ${version_file} "WITH_LITE: ${WITH_LITE}\n"
-                                "LITE_GIT_TAG: ${LITE_GIT_TAG}\n")
-  endif()
-
-endfunction()
-version(${PADDLE_INSTALL_DIR}/version.txt)
 version(${PADDLE_INFERENCE_INSTALL_DIR}/version.txt)
 version(${PADDLE_INFERENCE_C_INSTALL_DIR}/version.txt)
diff --git a/cmake/paddle_lib.cmake b/cmake/paddle_lib.cmake
new file mode 100644
index 0000000000000..535878b5b3a7c
--- /dev/null
+++ b/cmake/paddle_lib.cmake
@@ -0,0 +1,24 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# make package for paddle shared library
+set(PADDLE_INSTALL_DIR ${PADDLE_BINARY_DIR}/paddle_install_dir)
+set(PADDLE_LIB_TEST_DIR ${PADDLE_BINARY_DIR}/test/paddle_lib)
+
+configure_file(${PADDLE_SOURCE_DIR}/cmake/PaddleConfig.cmake.in
+               ${PADDLE_INSTALL_DIR}/cmake/PaddleConfig.cmake @ONLY)
+configure_file(${PADDLE_SOURCE_DIR}/test/paddle_lib/CMakeLists.txt.in
+               ${PADDLE_BINARY_DIR}/test/paddle_lib/CMakeLists.txt @ONLY)
+
+version(${PADDLE_INSTALL_DIR}/version.txt)
diff --git a/cmake/version.cmake b/cmake/version.cmake
index 83bd3f1b1bc4a..e6707665a3851 100644
--- a/cmake/version.cmake
+++ b/cmake/version.cmake
@@ -71,3 +71,46 @@ math(EXPR PADDLE_VERSION_INTEGER "${PADDLE_MAJOR_VER} * 1000000
 add_definitions(-DPADDLE_VERSION=${PADDLE_VERSION})
 add_definitions(-DPADDLE_VERSION_INTEGER=${PADDLE_VERSION_INTEGER})
 message(STATUS "Paddle version is ${PADDLE_VERSION}")
+
+# write paddle version
+function(version version_file)
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} log --pretty=format:%H -1
+    WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}
+    OUTPUT_VARIABLE PADDLE_GIT_COMMIT)
+  file(
+    WRITE ${version_file}
+    "Paddle version: ${PADDLE_VERSION}\n"
+    "GIT COMMIT ID: ${PADDLE_GIT_COMMIT}\n"
+    "WITH_MKL: ${WITH_MKL}\n"
+    "WITH_MKLDNN: ${WITH_MKLDNN}\n"
+    "WITH_GPU: ${WITH_GPU}\n"
+    "WITH_ROCM: ${WITH_ROCM}\n"
+    "WITH_IPU: ${WITH_IPU}\n")
+  if(WITH_GPU)
+    file(APPEND ${version_file}
+         "CUDA version: ${CUDA_VERSION}\n"
+         "CUDNN version: v${CUDNN_MAJOR_VERSION}.${CUDNN_MINOR_VERSION}\n")
+  endif()
+  if(WITH_ROCM)
+    file(APPEND ${version_file}
+         "HIP version: v${HIP_MAJOR_VERSION}.${HIP_MINOR_VERSION}\n"
+         "MIOpen version: v${MIOPEN_MAJOR_VERSION}.${MIOPEN_MINOR_VERSION}\n")
+  endif()
+  if(WITH_IPU)
+    file(APPEND ${version_file} "PopART version: ${POPART_VERSION}\n")
+  endif()
+  file(APPEND ${version_file}
+       "CXX compiler version: ${CMAKE_CXX_COMPILER_VERSION}\n")
+  if(TENSORRT_FOUND)
+    file(
+      APPEND ${version_file}
+      "WITH_TENSORRT: ${TENSORRT_FOUND}\n"
+      "TensorRT version: v${TENSORRT_MAJOR_VERSION}.${TENSORRT_MINOR_VERSION}.${TENSORRT_PATCH_VERSION}.${TENSORRT_BUILD_VERSION}\n"
+    )
+  endif()
+  if(WITH_LITE)
+    file(APPEND ${version_file} "WITH_LITE: ${WITH_LITE}\n"
+                                "LITE_GIT_TAG: ${LITE_GIT_TAG}\n")
+  endif()
+endfunction()
diff --git a/python/env_dict.py.in b/python/env_dict.py.in
index 00ca04dc56cde..d8ae5f9144b65 100644
--- a/python/env_dict.py.in
+++ b/python/env_dict.py.in
@@ -75,5 +75,8 @@ env_dict={
     'PYBIND_INCLUDE_DIR':'@PYBIND_INCLUDE_DIR@',
     'WITH_PYTHON':'@WITH_PYTHON@',
     'WITH_CINN':'@WITH_CINN@',
-    'CINN_SOURCE_DIR':'@CINN_SOURCE_DIR@'
+    'CINN_SOURCE_DIR':'@CINN_SOURCE_DIR@',
+    'WITH_CPP_DIST':'@WITH_CPP_DIST@',
+    'PADDLE_INSTALL_DIR':'@PADDLE_INSTALL_DIR@',
+    'PADDLE_LIB_TEST_DIR':'@PADDLE_LIB_TEST_DIR@'
 }
diff --git a/python/setup.py.in b/python/setup.py.in
index c87a5923f5c6a..1c59c4aaa4746 100644
--- a/python/setup.py.in
+++ b/python/setup.py.in
@@ -782,6 +782,24 @@ if '${WITH_XPU}' == 'ON':
 
 headers += list(find_files('*.h', '${PYBIND_INCLUDE_DIR}', True)) # pybind headers
 
+def get_header_install_dir(header):
+    if 'pb.h' in header:
+        install_dir = re.sub('${PADDLE_BINARY_DIR}/', '', header)
+    elif 'third_party' not in header:
+        # paddle headers
+        install_dir = re.sub('@PADDLE_SOURCE_DIR@/', '', header)
+        print('install_dir: ', install_dir)
+        if 'fluid/jit' in install_dir:
+            install_dir = re.sub('fluid/jit', 'jit', install_dir)
+            print('fluid/jit install_dir: ', install_dir)
+    else:
+        # third_party
+        install_dir = re.sub('${THIRD_PARTY_PATH}', 'third_party', header)
+        patterns = ['install/mkldnn/include', 'pybind/src/extern_pybind/include', 'third_party/xpu/src/extern_xpu/xpu/include/']
+        for pattern in patterns:
+            install_dir = re.sub(pattern, '', install_dir)
+    return install_dir
+
 class InstallCommand(InstallCommandBase):
     def finalize_options(self):
         ret = InstallCommandBase.finalize_options(self)
@@ -813,21 +831,7 @@ class InstallHeaders(Command):
                                    ('force', 'force'))
 
     def mkdir_and_copy_file(self, header):
-        if 'pb.h' in header:
-            install_dir = re.sub('${PADDLE_BINARY_DIR}/', '', header)
-        elif 'third_party' not in header:
-            # paddle headers
-            install_dir = re.sub('@PADDLE_SOURCE_DIR@/', '', header)
-            print('install_dir: ', install_dir)
-            if 'fluid/jit' in install_dir:
-                install_dir = re.sub('fluid/jit', 'jit', install_dir)
-                print('fluid/jit install_dir: ', install_dir)
-        else:
-            # third_party
-            install_dir = re.sub('${THIRD_PARTY_PATH}', 'third_party', header)
-            patterns = ['install/mkldnn/include', 'pybind/src/extern_pybind/include', 'third_party/xpu/src/extern_xpu/xpu/include/']
-            for pattern in patterns:
-                install_dir = re.sub(pattern, '', install_dir)
+        install_dir = get_header_install_dir(header)
         install_dir = os.path.join(self.install_dir, os.path.dirname(install_dir))
         if not os.path.exists(install_dir):
             self.mkpath(install_dir)
@@ -885,6 +889,54 @@ if '${WITH_STRIP}' == 'ON':
     if os.system(command) != 0:
         raise Exception("strip *.so failed, command: %s" % command)
 
+def install_cpp_dist_and_build_test(paddle_install_dir, paddle_lib_test_dir):
+    """install cpp distribution and build test target
+
+    TODO(huangjiyi): 
+    1. This function will be moved when seperating C++ distribution
+    installation from python package installation.
+    2. Reduce the header and library files to be installed.
+    """
+    if '${CMAKE_BUILD_TYPE}' != 'Release':
+        return
+    os.makedirs(paddle_install_dir, exist_ok=True)
+    # install C++ header files
+    for header in headers: 
+        install_dir = get_header_install_dir(header)
+        install_dir = os.path.join(
+            paddle_install_dir, 'include', os.path.dirname(install_dir)
+        )
+        os.makedirs(install_dir, exist_ok=True)
+        shutil.copy(header, install_dir)
+
+    # install C++ shared libraries
+    lib_install_dir = os.path.join(paddle_install_dir, 'lib')
+    os.makedirs(lib_install_dir, exist_ok=True)
+    # install libpaddle.ext
+    paddle_libs = glob.glob('${PADDLE_BINARY_DIR}/paddle/fluid/pybind/${FLUID_CORE_NAME}.*')
+    for lib in paddle_libs:
+        shutil.copy(lib, lib_install_dir)
+    # install dependent libraries
+    libs_path = package_dir['paddle.libs']
+    for lib in package_data['paddle.libs']:
+        lib_path = os.path.join(libs_path, lib)
+        shutil.copy(lib_path, lib_install_dir)
+    
+    # build test target
+    cmake_args = ["cmake", paddle_lib_test_dir, "-B", paddle_lib_test_dir]
+    if os.getenv("GENERATOR") == "Ninja":
+        cmake_args.append("-GNinja")
+    subprocess.check_call(cmake_args)
+    subprocess.check_call(["cmake", "--build", paddle_lib_test_dir])
+
+
+# install cpp distribution
+if '${WITH_CPP_DIST}' == 'ON':
+    paddle_install_dir = '${PADDLE_INSTALL_DIR}'
+    paddle_lib_test_dir = '${PADDLE_LIB_TEST_DIR}'
+    install_cpp_dist_and_build_test(paddle_install_dir, paddle_lib_test_dir)
+    
+
 with redirect_stdout():
     setup(name='${PACKAGE_NAME}',
         version='${PADDLE_VERSION}',
diff --git a/setup.py b/setup.py
index 9e617b5e340fc..6a305243bbe3d 100644
--- a/setup.py
+++ b/setup.py
@@ -1545,6 +1545,52 @@ def check_build_dependency():
             raise RuntimeError(missing_modules.format(dependency=dependency))
 
 
+def install_cpp_dist_and_build_test(install_dir, lib_test_dir, headers, libs):
+    """install cpp distribution and build test target
+
+    TODO(huangjiyi):
+    1. This function will be moved when seperating C++ distribution
+    installation from python package installation.
+    2. Reduce the header and library files to be installed.
+    """
+    if env_dict.get("CMAKE_BUILD_TYPE") != 'Release':
+        return
+    os.makedirs(install_dir, exist_ok=True)
+    # install C++ header files
+    for header in headers:
+        header_install_dir = get_header_install_dir(header)
+        header_install_dir = os.path.join(
+            install_dir, 'include', os.path.dirname(header_install_dir)
+        )
+        os.makedirs(header_install_dir, exist_ok=True)
+        shutil.copy(header, header_install_dir)
+
+    # install C++ shared libraries
+    lib_install_dir = os.path.join(install_dir, 'lib')
+    os.makedirs(lib_install_dir, exist_ok=True)
+    # install libpaddle.ext
+    paddle_libs = glob.glob(
+        paddle_binary_dir
+        + '/paddle/fluid/pybind/'
+        + env_dict.get("FLUID_CORE_NAME")
+        + '.*'
+    )
+    for lib in paddle_libs:
+        shutil.copy(lib, lib_install_dir)
+    # install dependent libraries
+    libs_path = paddle_binary_dir + '/python/paddle/libs'
+    for lib in libs:
+        lib_path = os.path.join(libs_path, lib)
+        shutil.copy(lib_path, lib_install_dir)
+
+    # build test target
+    cmake_args = [CMAKE, lib_test_dir, "-B", lib_test_dir]
+    if os.getenv("GENERATOR") == "Ninja":
+        cmake_args.append("-GNinja")
+    subprocess.check_call(cmake_args)
+    subprocess.check_call([CMAKE, "--build", lib_test_dir])
+
+
 def main():
     # Parse the command line and check arguments before we proceed with building steps and setup
     parse_input_command(filter_args_list)
@@ -1617,6 +1663,17 @@ def main():
         if os.system(command) != 0:
             raise Exception("strip *.so failed, command: %s" % command)
 
+    # install cpp distribution
+    if env_dict.get("WITH_CPP_DIST") == 'ON':
+        paddle_install_dir = env_dict.get("PADDLE_INSTALL_DIR")
+        paddle_lib_test_dir = env_dict.get("PADDLE_LIB_TEST_DIR")
+        install_cpp_dist_and_build_test(
+            paddle_install_dir,
+            paddle_lib_test_dir,
+            headers,
+            package_data['paddle.libs'],
+        )
+
     setup(
         name=package_name,
         version=paddle_version,
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 066875945a79c..96bd6ca3c263d 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -135,6 +135,15 @@ if(WITH_TESTING)
   endif()
 endif()
 
+if(WITH_CPP_DIST)
+  add_test(NAME test_paddle_lib
+           COMMAND ${PADDLE_BINARY_DIR}/test/paddle_lib/test_paddle_lib)
+  if(WITH_GPU)
+    add_test(NAME test_paddle_lib_gpu
+             COMMAND ${PADDLE_BINARY_DIR}/test/paddle_lib/test_paddle_lib_gpu)
+  endif()
+endif()
+
 get_property(test_srcs GLOBAL PROPERTY TEST_SRCS)
 get_property(test_names GLOBAL PROPERTY TEST_NAMES)
 
diff --git a/test/paddle_lib/CMakeLists.txt.in b/test/paddle_lib/CMakeLists.txt.in
new file mode 100644
index 0000000000000..ec0c4c00f1905
--- /dev/null
+++ b/test/paddle_lib/CMakeLists.txt.in
@@ -0,0 +1,14 @@
+cmake_minimum_required(VERSION 3.15)
+project(test_paddle_lib)
+
+list(APPEND CMAKE_PREFIX_PATH "@PADDLE_BINARY_DIR@/paddle_install_dir")
+find_package(Paddle REQUIRED)
+include_directories(${PADDLE_INCLUDE_DIRS})
+
+add_executable(test_paddle_lib test_paddle_lib.cc)
+target_link_libraries(test_paddle_lib ${PADDLE_LIBRARIES})
+
+if(@WITH_GPU@)
+  add_executable(test_paddle_lib_gpu test_paddle_lib_gpu.cc)
+  target_link_libraries(test_paddle_lib_gpu ${PADDLE_LIBRARIES})
+endif()
diff --git a/test/paddle_lib/test_paddle_lib.cc b/test/paddle_lib/test_paddle_lib.cc
new file mode 100644
index 0000000000000..9c32d7e2c5b60
--- /dev/null
+++ b/test/paddle_lib/test_paddle_lib.cc
@@ -0,0 +1,27 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cassert>
+
+#include "paddle/extension.h"
+
+int main() {
+  int data[] = {1, 2, 3, 4};
+  auto tensor = paddle::from_blob(data, {2, 2}, phi::DataType::INT32);
+
+  assert(tensor.numel() == 4);
+  assert(tensor.dtype() == phi::DataType::INT32);
+  assert(tensor.is_cpu());
+  assert(tensor.template data<int>() == data);
+}
diff --git a/test/paddle_lib/test_paddle_lib_gpu.cc b/test/paddle_lib/test_paddle_lib_gpu.cc
new file mode 100644
index 0000000000000..4cd27c53f186d
--- /dev/null
+++ b/test/paddle_lib/test_paddle_lib_gpu.cc
@@ -0,0 +1,25 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cassert>
+
+#include "paddle/extension.h"
+
+int main() {
+  float data[] = {1., 2., 3., 4.};
+  auto tensor = paddle::from_blob(data, {2, 2}, phi::DataType::FLOAT32);
+  auto gpu_tensor =
+      paddle::experimental::copy_to(tensor, phi::GPUPlace(), false);
+  assert(gpu_tensor.is_gpu());
+}

From 3650c4a85dd2a0e52885897cfa4a3303a7f45645 Mon Sep 17 00:00:00 2001
From: liuzhenhai93 <liuzhenhai93@outlook.com>
Date: Wed, 26 Apr 2023 19:59:48 +0800
Subject: [PATCH 087/405] =?UTF-8?q?=20pp=20=E7=AD=96=E7=95=A5=E8=B0=83?=
 =?UTF-8?q?=E6=95=B4=E5=90=8E=EF=BC=8C=E6=A8=A1=E5=9E=8B=E8=BD=AC=E6=8D=A2?=
 =?UTF-8?q?=EF=BC=8C=E4=BB=A5=E4=BE=BF=E6=A8=A1=E5=9E=8B=E7=83=AD=E5=90=AF?=
 =?UTF-8?q?=20(#52927)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish
---
 .../parallel_layers/pp_layers.py              |   4 +-
 .../fleet/utils/pp_parallel_adaptor.py        | 612 ++++++++++++++++++
 .../unittests/collective/fleet/CMakeLists.txt |  13 +
 .../fleet/hybrid_parallel_pp_transformer.py   |   4 +-
 .../hybrid_parallel_pp_transformer_save.py    | 101 +++
 ..._pp_transformer_save_with_virtual_stage.py | 105 +++
 ...allel_pp_transformer_with_virtual_stage.py |   5 +-
 .../fleet/test_parallel_dygraph_pp_adaptor.py | 163 +++++
 .../unittests/collective/fleet/testslist.csv  |   1 +
 python/paddle/framework/io.py                 |   9 +-
 test/auto_parallel/CMakeLists.txt             |   2 +-
 test/distributed_passes/CMakeLists.txt        |   2 +-
 test/ir/inference/CMakeLists.txt              |   2 +-
 13 files changed, 1012 insertions(+), 11 deletions(-)
 create mode 100644 python/paddle/distributed/fleet/utils/pp_parallel_adaptor.py
 create mode 100644 python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_save.py
 create mode 100644 python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_save_with_virtual_stage.py
 create mode 100644 python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pp_adaptor.py

diff --git a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py
index f3be9894a9cfe..bc403da76ba91 100755
--- a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py
+++ b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py
@@ -564,7 +564,7 @@ def _segment_network_for_interleave(self, seg_method):
         self.segment_parts = seg.do_segment()
 
         logger.info(
-            "segment result:"
+            f"segment with method: {seg_method}; result: "
             + ", ".join(str(arg) for arg in self.segment_parts)
         )
 
@@ -594,7 +594,7 @@ def _segment_network(self, seg_method):
         self.segment_parts = seg.do_segment()
 
         logger.info(
-            "segment result:"
+            f"segment with method: {seg_method}; result: "
             + ", ".join(str(arg) for arg in self.segment_parts)
         )
 
diff --git a/python/paddle/distributed/fleet/utils/pp_parallel_adaptor.py b/python/paddle/distributed/fleet/utils/pp_parallel_adaptor.py
new file mode 100644
index 0000000000000..9fa1caf35a547
--- /dev/null
+++ b/python/paddle/distributed/fleet/utils/pp_parallel_adaptor.py
@@ -0,0 +1,612 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import math
+import re
+import shutil
+from collections import OrderedDict
+
+import paddle
+
+
+class ParallelConfig:
+    def __init__(self, mp: int, pp: int, vpp: int = 1, sharding: int = 1):
+        self.mp = mp
+        self.pp = pp
+        self.vpp = vpp
+        self.sharding = sharding
+
+    def pipe_parallel_group(self, i: int, j: int):
+        ans = []
+        for k in range(self.pp):
+            ans.append((i, j, k))
+        return ans
+
+
+class LayerReNamingHelper:
+    def __init__(self, template: str):
+        self._template = template
+        self._i = -1
+        self._last_old_layer_name = None
+
+    def get_new_layer_name(self, old_layer_name: str):
+        old_layer_name = old_layer_name.split(".")[0]
+        if (
+            self._last_old_layer_name is None
+            or old_layer_name != self._last_old_layer_name
+        ):
+            self._i = self._i + 1
+            self._last_old_layer_name = old_layer_name
+        return self._template.format(self._i)
+
+
+class LayerReNamingManager:
+    def __init__(self):
+        self._renaming_helpers = OrderedDict()
+        self._renaming_helpers["linear"] = LayerReNamingHelper("linear_{}")
+        self._renaming_helpers["layer_norm"] = LayerReNamingHelper(
+            "layer_norm_{}"
+        )
+        self._renaming_helpers["embedding"] = LayerReNamingHelper(
+            "embedding_{}"
+        )
+
+    def get_new_layer_name(self, old_name: str):
+        layer_name = ""
+        for (k, v) in self._renaming_helpers.items():
+            if old_name.startswith(k):
+                layer_name = v.get_new_layer_name(old_name)
+                break
+        return layer_name
+
+    def get_new_param_name(self, old_name: str):
+        names = old_name.split(".")
+        layer_name = self.get_new_layer_name(names[0])
+        assert layer_name, f"can not rename layer {names[0]}"
+        names[0] = layer_name
+        return ".".join(names)
+
+
+class PipeLineModelAdaptor:
+    def __init__(
+        self,
+        src_parallel_config: ParallelConfig,
+        dst_parallel_config: ParallelConfig,
+        transformer_layer_num: int,
+        segment_method: str = "layer",
+    ):
+        self._src_parallel_config = src_parallel_config
+        self._dst_parallel_config = dst_parallel_config
+        self._transformer_layer_num = transformer_layer_num
+        self._segment_method = segment_method
+
+    def apply(self, src_model_path: str, dst_model_path: str):
+        for i in range(self._src_parallel_config.mp):
+            for j in range(self._src_parallel_config.sharding):
+                # TODO(liuzhenhai): use multiple processs
+                layers = []
+
+                # 1、extract layers in the same pp group
+                group = self._src_parallel_config.pipe_parallel_group(i, j)
+                src_dirs = [
+                    "{}/mp_{:0>2d}_sharding_{:0>2d}_pp_{:0>2d}".format(
+                        src_model_path, *e
+                    )
+                    for e in group
+                ]
+                # first rank extract shared layer
+                with_shared = True
+                for dir in src_dirs:
+                    print("extract layer params in dir %s" % dir)
+                    layers.extend(self.extract_layers(dir, with_shared))
+                    with_shared = False
+                # 2、sort and unique layers
+                layers = self.sort_layers(layers)
+
+                # 3、resplit layers among pp group according new pp config
+                layer_segments = self.segment_layers(
+                    layers, self._dst_parallel_config, self._segment_method
+                )
+                dst_group = self._dst_parallel_config.pipe_parallel_group(i, j)
+                dst_dirs = [
+                    "{}/mp_{:0>2d}_sharding_{:0>2d}_pp_{:0>2d}".format(
+                        dst_model_path, *e
+                    )
+                    for e in dst_group
+                ]
+
+                # 4、merge layers belonging to the same node
+                for (layer_segment, dir_) in zip(layer_segments, dst_dirs):
+                    print(f"merge {len(layer_segment)} layers to {dir_}")
+                    self.merge_layers(layer_segment, dir_)
+
+                # 5、copy meta_state.pdopt
+                for (src_dir, dst_dir) in zip(src_dirs, dst_dirs):
+                    shutil.copyfile(
+                        f"{src_dir}/meta_state.pdopt",
+                        f"{dst_dir}/meta_state.pdopt",
+                    )
+
+    def peek_model(self, model_dir: str):
+        for i in range(self._src_parallel_config.mp):
+            for j in range(self._src_parallel_config.sharding):
+                group = self._src_parallel_config.pipe_parallel_group(i, j)
+                dirs = [
+                    "{}/mp_{:0>2d}_sharding_{:0>2d}_pp_{:0>2d}".format(
+                        model_dir, *e
+                    )
+                    for e in group
+                ]
+                for dir in dirs:
+                    print(f"peek partial model in {dir}:")
+                    self.peek_partial_model(dir)
+
+    def peek_partial_model(self, sub_dir: str):
+        state_dict = paddle.load(f"{sub_dir}/model.pdparams")
+        for (k, v) in state_dict.items():
+            print(f"\t{k} -> {v.name}")
+
+    def extract_layers(self, dir: str, with_shared: bool):
+        opt = paddle.load(dir + "/model_state.pdopt")
+        params = paddle.load(dir + "/model.pdparams")
+        shared_layer_parsed = False
+        # tname -> (layer, param_name)
+        tname_to_layer_and_pname = {}
+        for (k, v) in params.items():
+            layer = self._extract_layer_name(k)
+            assert layer
+            # special treatment for embedding layer, skip duplicated shared layer
+            # shared layer may exist or not, if it exist it share weight with _layers.0
+            # _layers.shared_layers.embed.word_embeddings.weight -> embedding_0.w_0
+            # _layers.shared_layers.embed.position_embeddings.weight -> embedding_1.w_0
+            # _layers.0.word_embeddings.weight -> embedding_0.w_0
+            # _layers.0.position_embeddings.weight -> embedding_1.w_0
+            shared_layer_parsed = shared_layer_parsed or (
+                "_layers.shared_layers" in layer
+            )
+            if (
+                "_layers.shared_layers" not in layer
+                and ("word_embeddings" in k or "position_embeddings" in k)
+                and shared_layer_parsed
+            ):
+                continue
+            tname_to_layer_and_pname[v.name] = (layer, k)
+
+            # get opt-> param mapping
+        tensor_names = list(tname_to_layer_and_pname.keys())
+        opt_names = [
+            e for e in opt.keys() if e not in ["master_weights", "LR_Scheduler"]
+        ]
+        opt_to_t = self._opt_name_to_tname(tensor_names, opt_names)
+        # gather tensors belonging to one layer togather
+        layers = OrderedDict()
+        for (k, v) in params.items():
+            layer, p = tname_to_layer_and_pname[v.name]
+            if layer not in layers:
+                layers[layer] = {}
+                layers[layer]["opt"] = OrderedDict()
+                layers[layer]["params"] = OrderedDict()
+                layers[layer]["master_weights"] = OrderedDict()
+            layers[layer]["params"][p] = v
+
+        for (k, v) in opt.items():
+            if k in ["master_weights", "LR_Scheduler"]:
+                continue
+            layer, _ = tname_to_layer_and_pname[opt_to_t[v.name]]
+            layers[layer]["opt"][k] = v
+
+        if "master_weights" in opt:
+            for (k, v) in opt["master_weights"].items():
+                layer, _ = tname_to_layer_and_pname[k]
+                layers[layer]["master_weights"][k] = v
+
+        if "LR_Scheduler" in opt:
+            for layer in layers:
+                layers[layer]["LR_Scheduler"] = opt["LR_Scheduler"]
+
+        ans = []
+
+        for (layer_name, layer) in layers.items():
+            # special treatment for embedding layer
+            if (not with_shared) and "shared_layers" in layer_name:
+                continue
+            file_name = f"./tmp_layer_files/{layer_name}.tmp"
+            paddle.save(layer, file_name)
+            ans.append((layer_name, file_name))
+            print(f"save layer {layer_name} to {file_name}")
+        return ans
+
+    def sort_layers(self, layers: list):
+        def priority(elem):
+            layer_name = elem[0]
+            if "shared_layers" in layer_name:
+                return -float(0.5)
+            match = re.search(
+                r"^_layers((\.\d+)+|(\.shared_layers\.[^\.]+))", layer_name
+            )
+            assert match, f"{layer_name} not a valid layer name"
+            return float(match.group(1).lstrip("."))
+
+        # strictly sort layers
+        print("before sort %s" % ("|".join([e[0] for e in layers])))
+        layers.sort(key=priority)
+        # unique
+        unique_layers = []
+        for e in layers:
+            if unique_layers and e[0] == unique_layers[-1][0]:
+                continue
+            unique_layers.append(e)
+        print("after sort %s " % ("|".join([e[0] for e in unique_layers])))
+        return unique_layers
+
+    def segment_layers(
+        self,
+        layers: list,
+        config: ParallelConfig,
+        segment_method: str = "layer",
+    ):
+        layer_num = len(layers)
+        stage_num = config.pp * config.vpp
+
+        # segment by weights
+        def segment_by_layer():
+            # assume model is of the structure below
+            # embedding -> n*(transformer layer) -> [optional output layer]
+            # segment index
+            weights = [0 for _ in range(layer_num)]
+            non_zero_layers = range(1, layer_num - 1)
+            # input layer is embedding
+            if self._transformer_layer_num:
+                assert self._transformer_layer_num < layer_num
+                non_zero_layers = range(1, 1 + self._transformer_layer_num)
+            for i in non_zero_layers:
+                weights[i] = 1
+
+            part_size = sum(weights) // stage_num
+            result = [0 for _ in range(stage_num + 1)]
+            memory_counter = 0
+            result_idx = 1
+            for idx, weight in enumerate(weights):
+                memory_counter += weight
+                if memory_counter == part_size:
+                    result[result_idx] = idx + 1
+                    result_idx += 1
+                    memory_counter = 0
+            result[stage_num] = layer_num
+            return result
+
+        def segment_uniform():
+            result = [0 for _ in range(stage_num + 1)]
+            part_size = math.floor(layer_num / stage_num)
+            extra_layers = layer_num % stage_num
+            for i in range(1, stage_num):
+                offset = 1 if i > (stage_num - extra_layers) else 0
+                result[i] = int(
+                    min(result[i - 1] + part_size + offset, layer_num)
+                )
+            result[stage_num] = layer_num
+            return result
+
+        result = (
+            segment_uniform()
+            if (segment_method == "uniform")
+            else segment_by_layer()
+        )
+        index_segments = [[] for _ in range(config.pp)]
+        for i in range(stage_num):
+            index_segments[i % config.pp].append((result[i], result[i + 1]))
+
+        # name layers
+        segments = [[] for i in range(config.pp)]
+        for i in range(config.pp):
+            for (start, end) in index_segments[i]:
+                for j in range(start, end):
+                    if config.vpp > 1:
+                        segments[i].append(
+                            (
+                                [f"_layers.{start}.{j - start}"],
+                                layers[j][1],
+                            )
+                        )
+                    else:
+                        segments[i].append(([f"_layers.{j}"], layers[j][1]))
+
+        shared_layer_exist = any(
+            "_layers.shared_layers" in e[0] for e in layers
+        )
+        if shared_layer_exist:
+            # special treatment for shared layer
+            if config.vpp > 1:
+                segments[0] = [
+                    ([layers[0][0], segments[0][0][0][0]], layers[0][1])
+                ] + segments[0][1:]
+            else:
+                segments[0] = [([layers[0][0]], layers[0][1])] + segments[0][1:]
+
+            for i in range(1, config.pp):
+                segments[i] = [([layers[0][0]], layers[0][1])] + segments[i]
+
+        for (pp_rank, segs) in enumerate(segments):
+            print(f"segmentment result for pp_rank {pp_rank}:")
+            print(50 * "=")
+            for seg in segs:
+                print(f"{seg[0]} => {seg[1]}")
+        return segments
+
+    def merge_layers(self, layers_segment: list, save_dir: str):
+        params = OrderedDict()
+        opt = OrderedDict()
+        master_weights = OrderedDict()
+        renaming_manager = LayerReNamingManager()
+
+        def merge(src, dst, map_k=None):
+            for (k, v) in src.items():
+                k = map_k(k) if map_k is not None else k
+                dst[k] = v
+
+        lr_scheduler = None
+        for (layer_names, file_path) in layers_segment:
+            print("load %s" % file_path)
+            layer = paddle.load(file_path)
+
+            def get_param_name_mapper(layer_name):
+                # replace layer name
+                def map_param_name(param_name):
+                    layer_pre = self._extract_layer_name(param_name)
+                    return layer_name + param_name[len(layer_pre) :]
+
+                return map_param_name
+
+            (
+                layer_params,
+                layer_opt,
+                layer_master_weight,
+            ) = self._map_tensor_names(
+                layer["params"],
+                layer["opt"],
+                layer["master_weights"],
+                renaming_manager,
+            )
+            for layer_name in layer_names:
+                merge(layer_params, params, get_param_name_mapper(layer_name))
+            merge(layer_opt, opt)
+            merge(layer_master_weight, master_weights)
+            lr_scheduler = layer["LR_Scheduler"]
+
+        opt = self._pack_opt_state_dict(opt, master_weights, lr_scheduler)
+        paddle.save(params, save_dir + "/model.pdparams")
+        paddle.save(opt, save_dir + "/model_state.pdopt")
+
+    def _pack_opt_state_dict(self, opt, master_weights, lr_scheduler):
+        opt["master_weights"] = master_weights
+        opt["LR_Scheduler"] = lr_scheduler
+        return opt
+
+    def _extract_layer_name(self, param_name: str):
+        match = re.search(
+            r"^_layers((\.\d+)+|(\.shared_layers\.[^\.]+))", param_name
+        )
+        layer_name = ""
+        return "" if (not match) else match.group()
+
+    # map opt names to tensor name
+    def _opt_name_to_tname(self, tensor_names, opt_names):
+        tensor_names = set(tensor_names)
+        all_names = []
+        all_names.extend(list(tensor_names))
+        all_names.extend(opt_names)
+        all_names.sort()
+        pre_t_name = ""
+        opt_to_t = {}
+        for n in all_names:
+            if n in tensor_names:
+                # we get a param
+                pre_t_name = n
+            else:
+                assert pre_t_name
+                opt_to_t[n] = pre_t_name
+        return opt_to_t
+
+    def _map_tensor_names(self, params, opt, master_weights, renaming_manager):
+        opt_renamed = OrderedDict()
+        master_weights_renamed = OrderedDict()
+        # old name to new name
+        t_name_mapping = {}
+        # map tensor names
+        for (k, v) in params.items():
+            t_name_mapping[v.name] = renaming_manager.get_new_param_name(v.name)
+            v.name = t_name_mapping[v.name]
+        # map opt names
+        opt_to_tname = self._opt_name_to_tname(
+            t_name_mapping.keys(), opt.keys()
+        )
+        for (k, v) in opt.items():
+            old_t_name = opt_to_tname[k]
+            t_name = t_name_mapping[old_t_name]
+            opt_name = t_name + k[len(old_t_name) :]
+            v.name = opt_name
+            opt_renamed[opt_name] = v
+
+        # map master names
+        for (k, v) in master_weights.items():
+            t_name = t_name_mapping[k]
+            v.name = t_name + v.name[len(k) :]
+            master_weights_renamed[t_name] = v
+        return (params, opt_renamed, master_weights_renamed)
+
+
+def parse_args():
+
+    parser = argparse.ArgumentParser(
+        prog='model converter', description='converter a model'
+    )
+    parser.add_argument(
+        '--src_path',
+        type=str,
+        default="./output/epoch_0_step_30",
+        help='path of the model to convert',
+    )
+
+    parser.add_argument(
+        '--dst_path',
+        type=str,
+        default="./test_adapt",
+        help='path to saved the converted model',
+    )
+
+    parser.add_argument(
+        '--src_mp',
+        type=int,
+        default=2,
+        help='mp degree of the origin triaing task that dumpped this model',
+    )
+
+    parser.add_argument(
+        '--src_pp',
+        type=int,
+        default=2,
+        help='pp degree of the origin triaing task that dumpped this model',
+    )
+
+    parser.add_argument(
+        '--src_vp',
+        type=int,
+        default=2,
+        help='vp degree of the origin triaing task that dumpped this model',
+    )
+
+    parser.add_argument(
+        '--dst_mp',
+        type=int,
+        default=None,
+        help='mp degree of the origin triaing task that dumpped this model',
+    )
+
+    parser.add_argument(
+        '--dst_pp',
+        type=int,
+        default=None,
+        help='pp degree of the expected triaing task that would recover this model',
+    )
+
+    parser.add_argument(
+        '--dst_vp',
+        type=int,
+        default=2,
+        help='vp degree of the expected triaing task that would recover this model',
+    )
+
+    parser.add_argument(
+        '--sharding',
+        type=int,
+        default=1,
+        help=" sharding degree of both the origin triaing task that dumpped this model and the expected triaing task that would recover this model",
+    )
+
+    parser.add_argument(
+        '--method',
+        type=str,
+        default="adapt_model",
+        help='vp degree of the expected triaing task that would recover this model',
+    )
+
+    parser.add_argument(
+        '--segment_method',
+        type=str,
+        default="layer",
+        help='method to segment layers to pp or vp stages',
+    )
+
+    parser.add_argument(
+        '--transformer_layer_num',
+        type=int,
+        default=0,
+        help='transformer_layer_num of the model',
+    )
+    # assume model is of the structure below
+    # embedding -> n*[transformer layer] -> optional output layer
+
+    args = parser.parse_args()
+
+    if args.dst_mp is None:
+        args.dst_mp = args.src_mp
+    if args.dst_pp is None:
+        args.dst_pp = args.src_pp
+
+    assert args.src_mp == args.dst_mp, "src mp {} dst mp {}".format(
+        args.src_mp, args.dst_mp
+    )
+
+    assert args.method in [
+        'peek_model',
+        'adapt_model',
+    ], "method should be in ['peek_model', 'adapt_model']"
+    assert args.segment_method in [
+        "uniform",
+        "layer",
+    ], "segment_method should be 'uniform' or 'layer"
+
+    print(
+        "adapt model dumped by task with pp degree:{}, vp degree:{}, mp degree:{} to task with pp degree:{}, vp degree:{}, mp degree:{}".format(
+            args.src_pp,
+            args.src_vp,
+            args.src_mp,
+            args.dst_pp,
+            args.dst_vp,
+            args.dst_mp,
+        )
+    )
+
+    return args
+
+
+def adaptor_from_args(args):
+    src_parallel_config = ParallelConfig(
+        args.src_mp, args.src_pp, args.src_vp, args.sharding
+    )
+
+    dst_parallel_config = ParallelConfig(
+        args.dst_mp, args.dst_pp, args.dst_vp, args.sharding
+    )
+
+    adaptor = PipeLineModelAdaptor(
+        src_parallel_config,
+        dst_parallel_config,
+        args.transformer_layer_num,
+        args.segment_method,
+    )
+    return adaptor
+
+
+def main():
+
+    args = parse_args()
+    adaptor = adaptor_from_args(args)
+    if args.method == "peek_model":
+        adaptor.peek_model(args.dst_path)
+    elif args.method == "adapt_model":
+        adaptor.apply(args.src_path, args.dst_path)
+
+
+if __name__ == "__main__":
+    """
+    Usage:
+        python pp_parallel_adaptor.py --src_mp xxx --src_path xxx --method \
+          adapt_model/peek_model --dst_path xxx --sharding xxx --segment_method xxx --transformer_layer_num xxx
+
+    for the meaning of a specific arg, please use:
+        python  pp_parallel_adaptor.py  -h
+    """
+    main()
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt b/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt
index 637dffe42898a..dad5f1d4b5b6b 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt
@@ -168,6 +168,19 @@ if((WITH_GPU) AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_pipeline_parallel_with_virtual_stage
     PROPERTIES TIMEOUT "500")
 endif()
+if((WITH_GPU) AND LOCAL_ALL_PLAT)
+  bash_test_modules(
+    test_parallel_dygraph_pp_adaptor
+    START_BASH
+    ../../dist_test.sh
+    LABELS
+    "RUN_TYPE=DIST"
+    ENVS
+    "PADDLE_DIST_UT_PORT=21976;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
+  )
+  set_tests_properties(test_parallel_dygraph_pp_adaptor PROPERTIES TIMEOUT
+                                                                   "500")
+endif()
 if((WITH_GPU OR WITH_XPU) AND (LINUX))
   py_test_modules(
     test_fleet_localsgd_meta_optimizer MODULES
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer.py
index 6c1ae3eac44e2..216f37796daf0 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer.py
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer.py
@@ -121,11 +121,11 @@ def forward(self, out, label):
 
 
 class ModelPipe(PipelineLayer):
-    def __init__(self, topology):
+    def __init__(self, topology, transformer_layer_num: int = 6):
         self.descs = []
         self.descs.append(LayerDesc(EmbeddingPipe))
 
-        for x in range(6):
+        for x in range(transformer_layer_num):
             self.descs.append(LayerDesc(TransformerNetPipe))
 
         self.descs.append(lambda x: x[0])
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_save.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_save.py
new file mode 100644
index 0000000000000..a8cf970f73d22
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_save.py
@@ -0,0 +1,101 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+
+import numpy as np
+from hybrid_parallel_pp_transformer import ModelPipe, set_random_seed
+
+import paddle
+import paddle.distributed as dist
+from paddle.distributed import fleet
+
+batch_size = 8
+length = 8
+micro_batch_size = 2
+vocab_size = 128
+transformer_layer_num = 8
+
+
+class TestDistPPSaveTraning(unittest.TestCase):
+    def setUp(self):
+        strategy = fleet.DistributedStrategy()
+        self.model_parallel_size = 1
+        self.data_parallel_size = 1
+        self.pipeline_parallel_size = 2
+        strategy.hybrid_configs = {
+            "dp_degree": self.data_parallel_size,
+            "mp_degree": self.model_parallel_size,
+            "pp_degree": self.pipeline_parallel_size,
+        }
+        strategy.pipeline_configs = {
+            "accumulate_steps": batch_size // micro_batch_size,
+            "micro_batch_size": micro_batch_size,
+        }
+        fleet.init(is_collective=True, strategy=strategy)
+
+    def test_pp_model(self):
+        print(f"pwd {os.getcwd()}")
+        hcg = fleet.get_hybrid_communicate_group()
+        word_size = hcg.get_model_parallel_world_size()
+        dp_id = hcg.get_data_parallel_rank()
+        pp_id = hcg.get_stage_id()
+        rank_id = dist.get_rank()
+        topology = hcg.topology()
+        set_random_seed(1024, dp_id, rank_id)
+
+        model = ModelPipe(topology, transformer_layer_num=transformer_layer_num)
+        scheduler = paddle.optimizer.lr.PiecewiseDecay(
+            boundaries=[2], values=[0.001, 0.002], verbose=True
+        )
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=scheduler, parameters=model.parameters()
+        )
+
+        model = fleet.distributed_model(model)
+        optimizer = fleet.distributed_optimizer(optimizer)
+        output_dir = "{}/mp_00_sharding_00_pp_{:0>2d}".format(
+            "./pp_transformer", pp_id
+        )
+        try:
+            os.makedirs(output_dir)
+        except:
+            # dir is already created, do nothing
+            pass
+        for step_id in range(2):
+            x_data = np.random.randint(0, vocab_size, size=[batch_size, length])
+            x = paddle.to_tensor(x_data)
+            x.stop_gradient = True
+            loss = model.train_batch([x, x], optimizer, scheduler)
+
+        paddle.save(
+            model.state_dict(),
+            os.path.join(output_dir, "model.pdparams"),
+        )
+
+        paddle.save(
+            optimizer.state_dict(),
+            os.path.join(output_dir, "model_state.pdopt"),
+        )
+        meta_dict = {
+            "epoch": 0,
+            "step": 2,
+            "cuda_rng_state": paddle.get_cuda_rng_state(),
+        }
+        paddle.save(meta_dict, os.path.join(output_dir, "meta_state.pdopt"))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_save_with_virtual_stage.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_save_with_virtual_stage.py
new file mode 100644
index 0000000000000..372cbe7f48d93
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_save_with_virtual_stage.py
@@ -0,0 +1,105 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+
+import numpy as np
+from hybrid_parallel_pp_transformer_with_virtual_stage import (
+    ModelPipe,
+    set_random_seed,
+)
+
+import paddle
+import paddle.distributed as dist
+from paddle.distributed import fleet
+
+batch_size = 8
+length = 8
+micro_batch_size = 2
+vocab_size = 128
+
+transformer_layer_num = 8
+
+
+class TestDistPPSaveTraning(unittest.TestCase):
+    def setUp(self):
+        strategy = fleet.DistributedStrategy()
+        self.model_parallel_size = 1
+        self.data_parallel_size = 1
+        self.pipeline_parallel_size = 2
+        strategy.hybrid_configs = {
+            "dp_degree": self.data_parallel_size,
+            "mp_degree": self.model_parallel_size,
+            "pp_degree": self.pipeline_parallel_size,
+        }
+        strategy.pipeline_configs = {
+            "accumulate_steps": batch_size // micro_batch_size,
+            "micro_batch_size": micro_batch_size,
+        }
+        fleet.init(is_collective=True, strategy=strategy)
+
+    def test_pp_model(self):
+        print(f"pwd {os.getcwd()}")
+        hcg = fleet.get_hybrid_communicate_group()
+        word_size = hcg.get_model_parallel_world_size()
+        dp_id = hcg.get_data_parallel_rank()
+        pp_id = hcg.get_stage_id()
+        rank_id = dist.get_rank()
+        topology = hcg.topology()
+        set_random_seed(1024, dp_id, rank_id)
+
+        model = ModelPipe(topology, transformer_layer_num=transformer_layer_num)
+        scheduler = paddle.optimizer.lr.PiecewiseDecay(
+            boundaries=[2], values=[0.001, 0.002], verbose=True
+        )
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=scheduler, parameters=model.parameters()
+        )
+
+        model = fleet.distributed_model(model)
+        optimizer = fleet.distributed_optimizer(optimizer)
+
+        output_dir = "{}/mp_00_sharding_00_pp_{:0>2d}".format(
+            "./pp_transformer_vp", pp_id
+        )
+        try:
+            os.makedirs(output_dir)
+        except:
+            # dir is already created, do nothing
+            pass
+        for step_id in range(2):
+            x_data = np.random.randint(0, vocab_size, size=[batch_size, length])
+            x = paddle.to_tensor(x_data)
+            x.stop_gradient = True
+            loss = model.train_batch([x, x], optimizer, scheduler)
+
+        paddle.save(
+            model.state_dict(),
+            os.path.join(output_dir, "model.pdparams"),
+        )
+        paddle.save(
+            optimizer.state_dict(),
+            os.path.join(output_dir, "model_state.pdopt"),
+        )
+        meta_dict = {
+            "epoch": 0,
+            "step": 2,
+            "cuda_rng_state": paddle.get_cuda_rng_state(),
+        }
+        paddle.save(meta_dict, os.path.join(output_dir, "meta_state.pdopt"))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py
index 21cc9134e4d3d..9f43fc4c9ef8e 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py
@@ -120,11 +120,10 @@ def forward(self, out, label):
 
 
 class ModelPipe(PipelineLayer):
-    def __init__(self, topology):
+    def __init__(self, topology, transformer_layer_num: int = 8):
         self.descs = []
         self.descs.append(LayerDesc(EmbeddingPipe))
-
-        for x in range(8):
+        for x in range(transformer_layer_num):
             self.descs.append(LayerDesc(TransformerNetPipe))
 
         self.descs.append(lambda x: x[0])
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pp_adaptor.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pp_adaptor.py
new file mode 100644
index 0000000000000..3b11210991d3f
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pp_adaptor.py
@@ -0,0 +1,163 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import unittest
+
+from test_parallel_dygraph_dataparallel import TestMultipleGpus
+
+import paddle
+from paddle.distributed.fleet.utils.pp_parallel_adaptor import (
+    ParallelConfig,
+    PipeLineModelAdaptor,
+    adaptor_from_args,
+    parse_args,
+)
+
+
+class TestPPAdaptor(TestMultipleGpus):
+    def test_parse_args(self):
+        args = parse_args()
+        self.assertEqual(args.src_mp, args.dst_mp)
+        adaptor = adaptor_from_args(args)
+        self.assertTrue(adaptor is not None)
+
+    def test_hybrid_parallel_transformer_unbalanced_data(self):
+        print(f"pwd {os.getcwd()}")
+        self.run_mnist_2gpu('hybrid_parallel_pp_transformer_save.py')
+        self.run_mnist_2gpu(
+            'hybrid_parallel_pp_transformer_save_with_virtual_stage.py'
+        )
+        # test pp adaptor
+        dir1 = "./pp_transformer"
+        p_config1 = ParallelConfig(mp=1, pp=2, vpp=1, sharding=1)
+        dir2 = "./pp_transformer_vp"
+        p_config2 = ParallelConfig(mp=1, pp=2, vpp=2, sharding=1)
+
+        pp_to_vp = PipeLineModelAdaptor(
+            src_parallel_config=p_config1,
+            dst_parallel_config=p_config2,
+            transformer_layer_num=8,
+            segment_method="layer",
+        )
+        vp_to_pp = PipeLineModelAdaptor(
+            src_parallel_config=p_config2,
+            dst_parallel_config=p_config1,
+            transformer_layer_num=8,
+            segment_method="layer",
+        )
+
+        def check_converted_model(converted_model_dir, expected_model_dir):
+            # for compatibility, converted_model_dir may contain more key than
+            # expected model, which does not hinder model recovering
+            for i in range(p_config1.pp):
+                sub_converted_model_dir = (
+                    "{}/mp_00_sharding_00_pp_{:0>2d}".format(
+                        converted_model_dir, i
+                    )
+                )
+                sub_expected_model_dir = (
+                    "{}/mp_00_sharding_00_pp_{:0>2d}".format(
+                        expected_model_dir, i
+                    )
+                )
+                print(
+                    f"converted_model_dir: {sub_converted_model_dir}; expected_model_dir: {sub_expected_model_dir}"
+                )
+
+                def check_names(dict_1, dict_2):
+                    for (k, v) in dict_2.items():
+                        self.assertTrue(k in dict_1)
+                        self.assertEqual(
+                            getattr(v, "name", ""),
+                            getattr(dict_1[k], "name", ""),
+                        )
+
+                # check param
+                params_1 = paddle.load(
+                    f"{sub_converted_model_dir}/model.pdparams"
+                )
+                params_2 = paddle.load(
+                    f"{sub_expected_model_dir}/model.pdparams"
+                )
+                check_names(params_1, params_2)
+                del params_1
+                del params_2
+                # check opt
+                opt_1 = paddle.load(
+                    f"{sub_converted_model_dir}/model_state.pdopt"
+                )
+                opt_2 = paddle.load(
+                    f"{sub_expected_model_dir}/model_state.pdopt"
+                )
+                check_names(opt_1, opt_2)
+                # check master wieghts
+                if "master_weights" in opt_2:
+                    self.assertTrue("master_weights" in opt_1)
+                    check_names(
+                        opt_2["master_weights"], opt_1["master_weights"]
+                    )
+
+        def create_dir_if_nonexist(dir: str):
+            if not os.path.exists(dir):
+                os.makedirs(dir)
+
+        # check pp to vp
+        tmp_dir1 = "./tmp_pp_to_vp"
+        create_dir_if_nonexist(tmp_dir1)
+        pp_to_vp.apply(dir1, tmp_dir1)
+        # browse the converted model
+        pp_to_vp.peek_model(tmp_dir1)
+        # check
+        check_converted_model(tmp_dir1, dir2)
+
+        # check vp to pp
+        tmp_dir2 = "./tmp_vp_to_pp"
+        create_dir_if_nonexist(tmp_dir2)
+        vp_to_pp.apply(dir2, tmp_dir2)
+        vp_to_pp.peek_model(tmp_dir2)
+        check_converted_model(tmp_dir2, dir1)
+
+        # check uniform segment
+        tmp_dir3 = "./tmp_vp_to_pp_uniform"
+        create_dir_if_nonexist(tmp_dir3)
+        vp_to_pp_uniform = PipeLineModelAdaptor(
+            src_parallel_config=p_config2,
+            dst_parallel_config=p_config1,
+            transformer_layer_num=8,
+            segment_method="uniform",
+        )
+        vp_to_pp_uniform.apply(dir2, tmp_dir3)
+        vp_to_pp_uniform.peek_model(tmp_dir3)
+
+        tmp_dir4 = "./tmp_pp_to_pp_uniform"
+        create_dir_if_nonexist(tmp_dir4)
+        pp_to_pp_uniform = PipeLineModelAdaptor(
+            src_parallel_config=p_config1,
+            dst_parallel_config=p_config1,
+            transformer_layer_num=8,
+            segment_method="uniform",
+        )
+        pp_to_pp_uniform.apply(dir1, tmp_dir4)
+        pp_to_pp_uniform.peek_model(tmp_dir4)
+        check_converted_model(tmp_dir3, tmp_dir4)
+
+        # rm dirs
+        for d in [dir1, dir2, tmp_dir1, tmp_dir2, tmp_dir3, tmp_dir4]:
+            shutil.rmtree(d, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv b/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv
index 459a3e39df8e8..688b5e759b1db 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv
@@ -14,6 +14,7 @@ test_dygraph_sharding_stage3_for_eager,,,350,DIST,../../dist_test.sh,2,,http_pro
 test_communicator_half_async,,,120,DIST,test_runner.py,2,,FLAGS_communicator_send_queue_size=1;FLAGS_communicator_max_merge_var_num=1;http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL
 test_parallel_dygraph_pipeline_parallel,,GPU,500,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_pipeline_parallel_with_virtual_stage,,GPU,500,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_parallel_dygraph_pp_adaptor,,GPU,500,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_localsgd_meta_optimizer,LINUX,GPU;XPU,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_class_center_sample,,GPU,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL
 test_pipeline,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
diff --git a/python/paddle/framework/io.py b/python/paddle/framework/io.py
index a482d8bed4150..e5d011453f29c 100644
--- a/python/paddle/framework/io.py
+++ b/python/paddle/framework/io.py
@@ -1071,11 +1071,18 @@ def load(path, **configs):
                     # paddle2.0: paddle.save/load
                     if "StructuredToParameterName@@" in load_result:
 
-                        for key in load_result["StructuredToParameterName@@"]:
+                        for (key, name) in load_result[
+                            "StructuredToParameterName@@"
+                        ].items():
                             if isinstance(load_result[key], np.ndarray):
                                 load_result[key] = _ndarray_to_tensor(
                                     load_result[key], config.return_numpy
                                 )
+                                # default name is "generatedxxx" which is set in Tensor init, if not set
+                                if not config.return_numpy and getattr(
+                                    load_result[key], "name", ""
+                                ):
+                                    load_result[key].name = name
 
                         if (
                             not config.keep_name_table
diff --git a/test/auto_parallel/CMakeLists.txt b/test/auto_parallel/CMakeLists.txt
index 74b2fdb47a17e..e8660fc7b02c5 100644
--- a/test/auto_parallel/CMakeLists.txt
+++ b/test/auto_parallel/CMakeLists.txt
@@ -76,7 +76,7 @@ if(WITH_DISTRIBUTE AND WITH_GPU)
   py_test_modules(test_pass_quantization MODULES test_pass_quantization)
   set_tests_properties(test_pass_quantization PROPERTIES TIMEOUT 60)
   py_test_modules(test_tuning_recompute MODULES test_tuning_recompute)
-  set_tests_properties(test_tuning_recompute PROPERTIES TIMEOUT 240)
+  set_tests_properties(test_tuning_recompute PROPERTIES TIMEOUT 300)
   py_test_modules(test_fused_linear_pass MODULES test_fused_linear_pass)
   set_tests_properties(test_fused_linear_pass PROPERTIES TIMEOUT 20)
   py_test_modules(test_align_tool MODULES test_align_tool)
diff --git a/test/distributed_passes/CMakeLists.txt b/test/distributed_passes/CMakeLists.txt
index 83d2f0fb52994..e2b8697fc8591 100644
--- a/test/distributed_passes/CMakeLists.txt
+++ b/test/distributed_passes/CMakeLists.txt
@@ -29,6 +29,6 @@ endif()
 foreach(TEST_OP ${TEST_OPS})
   py_test_modules(${TEST_OP} MODULES ${TEST_OP})
   list(APPEND DIST_TEST_OPS ${TEST_OP})
-  set_tests_properties(${TEST_OP} PROPERTIES TIMEOUT 120)
+  set_tests_properties(${TEST_OP} PROPERTIES TIMEOUT 200)
   set_tests_properties(${TEST_OP} PROPERTIES LABELS "RUN_TYPE=DIST")
 endforeach()
diff --git a/test/ir/inference/CMakeLists.txt b/test/ir/inference/CMakeLists.txt
index b3bb181ea4a81..90620dab0fd3d 100755
--- a/test/ir/inference/CMakeLists.txt
+++ b/test/ir/inference/CMakeLists.txt
@@ -197,7 +197,7 @@ if(WITH_GPU AND TENSORRT_FOUND)
   set_tests_properties(test_trt_fc_fuse_quant_dequant_pass PROPERTIES TIMEOUT
                                                                       100)
   set_tests_properties(test_trt_conv_quant_dequant_pass PROPERTIES TIMEOUT 100)
-  set_tests_properties(test_trt_matmul_quant_dequant PROPERTIES TIMEOUT 100)
+  set_tests_properties(test_trt_matmul_quant_dequant PROPERTIES TIMEOUT 180)
   set_tests_properties(test_trt_conv3d_op PROPERTIES TIMEOUT 60)
   set_tests_properties(test_trt_conv3d_transpose_op PROPERTIES TIMEOUT 60)
   set_tests_properties(test_trt_nearest_interp_v2_op PROPERTIES TIMEOUT 30)

From 0b6dd5355e5327912c2476e167277618079d9fce Mon Sep 17 00:00:00 2001
From: zhwesky2010 <1183042833@qq.com>
Date: Wed, 26 Apr 2023 21:19:18 +0800
Subject: [PATCH 088/405] [Zero-Dim] distributed scatter/all_to_all support
 input 0D tensor (#53186)

---
 .../fluid/operators/collective/c_allgather_op.cc  |  9 +++++++--
 .../operators/collective/c_allgather_op.cu.cc     |  6 +-----
 .../fluid/operators/collective/c_allgather_op.h   |  5 +----
 .../operators/collective/c_allgather_op_xpu.cc    |  5 +----
 paddle/phi/core/ddim.cc                           |  4 ++--
 .../paddle/distributed/communication/scatter.py   |  6 ++----
 .../communication/stream/all_gather.py            |  6 +++++-
 .../communication/stream/all_to_all.py            | 15 +++++++++++++--
 .../distributed/communication/stream/scatter.py   |  6 +++++-
 9 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/paddle/fluid/operators/collective/c_allgather_op.cc b/paddle/fluid/operators/collective/c_allgather_op.cc
index 04019756ffe3e..b78d9504dafb7 100644
--- a/paddle/fluid/operators/collective/c_allgather_op.cc
+++ b/paddle/fluid/operators/collective/c_allgather_op.cc
@@ -31,8 +31,13 @@ class CAllGatherOp : public framework::OperatorWithKernel {
                       platform::errors::InvalidArgument(
                           "The value of nranks should be >=2."));
     framework::DDim dim = ctx->GetInputDim("X");
-    dim[0] = dim[0] * nranks;
-    if (dim[0] < 0) dim[0] = -1;
+    // 0D use stack/unstack while others use concat/split
+    if (dim.size() == 0) {
+      dim = phi::make_ddim({nranks});
+    } else {
+      dim[0] = dim[0] * nranks;
+      if (dim[0] < 0) dim[0] = -1;
+    }
     ctx->SetOutputDim("Out", dim);
   }
 };
diff --git a/paddle/fluid/operators/collective/c_allgather_op.cu.cc b/paddle/fluid/operators/collective/c_allgather_op.cu.cc
index 93be43a1a324a..70b7d70dc93b3 100644
--- a/paddle/fluid/operators/collective/c_allgather_op.cu.cc
+++ b/paddle/fluid/operators/collective/c_allgather_op.cu.cc
@@ -57,13 +57,9 @@ class CAllGatherOpCUDAKernel : public framework::OpKernel<T> {
         platform::errors::InvalidArgument(
             "nranks: %s should equal to %s", nranks, comm->nranks()));
 
-    framework::DDim out_dims = in->dims();
-    out_dims[0] *= nranks;
-    out->mutable_data<T>(out_dims, place);
-
     int64_t send_numel = in->numel();
     const T* send_buff = in->data<T>();
-    T* recv_buff = out->data<T>();
+    T* recv_buff = out->mutable_data<T>(place);
 
     gpuStream_t stream = nullptr;
     if (ctx.Attr<bool>("use_calc_stream")) {
diff --git a/paddle/fluid/operators/collective/c_allgather_op.h b/paddle/fluid/operators/collective/c_allgather_op.h
index e896f96ead532..c5373bf130438 100644
--- a/paddle/fluid/operators/collective/c_allgather_op.h
+++ b/paddle/fluid/operators/collective/c_allgather_op.h
@@ -39,15 +39,12 @@ class CAllGatherOpCPUKernel : public framework::OpKernel<T> {
 #if defined(PADDLE_WITH_GLOO)
     auto in = ctx.Input<phi::DenseTensor>("X");
     auto out = ctx.Output<phi::DenseTensor>("Out");
-    framework::DDim out_dims = in->dims();
     auto place = ctx.GetPlace();
-
     auto gloo = paddle::framework::GlooWrapper::GetInstance();
     auto nranks = gloo->Size();
-    out_dims[0] *= nranks;
     int64_t send_numel = in->numel();
     const T* send_buff = in->data<T>();
-    T* recv_buff = out->mutable_data<T>(out_dims, place);
+    T* recv_buff = out->mutable_data<T>(place);
 
     PADDLE_ENFORCE_EQ(
         gloo->IsInitialized(),
diff --git a/paddle/fluid/operators/collective/c_allgather_op_xpu.cc b/paddle/fluid/operators/collective/c_allgather_op_xpu.cc
index 1e7d3f3a9fec1..c4fdb0fdf290e 100644
--- a/paddle/fluid/operators/collective/c_allgather_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_allgather_op_xpu.cc
@@ -43,12 +43,9 @@ class CAllGatherOpXPUKernel : public framework::OpKernel<T> {
         platform::errors::InvalidArgument(
             "nranks: %s should equal to %s", nranks, comm->nranks()));
 
-    framework::DDim out_dims = in->dims();
-    out_dims[0] *= nranks;
-
     size_t numel = in->numel();
     const void* sendbuff = in->data<T>();
-    void* recvbuff = out->mutable_data<T>(out_dims, place);
+    void* recvbuff = out->mutable_data<T>(place);
 
     XPUStream stream = nullptr;
     if (ctx.Attr<bool>("use_calc_stream")) {
diff --git a/paddle/phi/core/ddim.cc b/paddle/phi/core/ddim.cc
index 3256458e02be9..05ca29843b42f 100644
--- a/paddle/phi/core/ddim.cc
+++ b/paddle/phi/core/ddim.cc
@@ -154,7 +154,7 @@ DDim flatten_to_1d(const DDim& src) { return DDim({product(src)}); }
 DDim stride(const DDim& ddim) {
   DDim strides;
   strides.rank_ = ddim.size();
-  strides[ddim.size() - 1] = 1;
+  if (ddim.size() > 0) strides[ddim.size() - 1] = 1;
   for (int i = ddim.size() - 2; i >= 0; --i) {
     strides[i] = strides[i + 1] * ddim[i + 1];
   }
@@ -164,7 +164,7 @@ DDim stride(const DDim& ddim) {
 DDim stride_numel(const DDim& ddim) {
   DDim strides;
   strides.rank_ = ddim.size();
-  strides[ddim.size() - 1] = ddim[ddim.size() - 1];
+  if (ddim.size() > 0) strides[ddim.size() - 1] = ddim[ddim.size() - 1];
   for (int i = ddim.size() - 2; i >= 0; --i) {
     strides[i] = strides[i + 1] * ddim[i];
   }
diff --git a/python/paddle/distributed/communication/scatter.py b/python/paddle/distributed/communication/scatter.py
index a8ba94e49c814..2826779d55738 100644
--- a/python/paddle/distributed/communication/scatter.py
+++ b/python/paddle/distributed/communication/scatter.py
@@ -122,8 +122,7 @@ def scatter_object_list(
             in_obj_sizes.append(obj_size)
         max_obj_size_tensor = max(in_obj_sizes)
     else:
-        # NOTE: shape can be [] after 0D tensor support
-        max_obj_size_tensor = paddle.empty([1], dtype="int64")
+        max_obj_size_tensor = paddle.empty([], dtype="int64")
     stream.broadcast(max_obj_size_tensor, src)
     max_obj_size = int(max_obj_size_tensor.item())
 
@@ -137,8 +136,7 @@ def scatter_object_list(
     out_tensor = paddle.empty([max_obj_size], dtype="uint8")
     scatter(out_tensor, in_tensor_list if rank == src else None, src, group)
 
-    # NOTE: shape can be [] after 0D tensor support
-    out_tensor_size = paddle.empty([1], dtype="int64")
+    out_tensor_size = paddle.empty([], dtype="int64")
     scatter(out_tensor_size, in_obj_sizes if rank == src else None, src, group)
 
     out_object_list.clear()
diff --git a/python/paddle/distributed/communication/stream/all_gather.py b/python/paddle/distributed/communication/stream/all_gather.py
index 4d02753a1a634..69d9c5d52e080 100644
--- a/python/paddle/distributed/communication/stream/all_gather.py
+++ b/python/paddle/distributed/communication/stream/all_gather.py
@@ -108,7 +108,11 @@ def _all_gather_in_static_mode(tensor_list, tensor, group, sync_op):
         },
     )
     tensor_list.clear()
-    tensor_list.extend(paddle.split(out, nranks, 0))
+    # 0D use stack/unstack while others use concat/split
+    if len(tensor.shape) == 0:
+        tensor_list.extend(paddle.unstack(out, 0))
+    else:
+        tensor_list.extend(paddle.split(out, nranks, 0))
 
 
 def all_gather(
diff --git a/python/paddle/distributed/communication/stream/all_to_all.py b/python/paddle/distributed/communication/stream/all_to_all.py
index d8793601f729a..38b1d2fcb3e82 100644
--- a/python/paddle/distributed/communication/stream/all_to_all.py
+++ b/python/paddle/distributed/communication/stream/all_to_all.py
@@ -78,7 +78,12 @@ def _all_to_all_in_static_mode(
     if isinstance(in_tensor_or_tensor_list, list):
         if len(in_tensor_or_tensor_list) == 0:
             raise RuntimeError("The input tensor_list should not be empty.")
-        in_tensor = paddle.concat(in_tensor_or_tensor_list, axis=0)
+        # 0D use stack/unstack while others use concat/split
+        if len(in_tensor_or_tensor_list[0].shape) == 0:
+            in_tensor = paddle.stack(in_tensor_or_tensor_list, axis=0)
+        else:
+            in_tensor = paddle.concat(in_tensor_or_tensor_list, axis=0)
+
     out_tensor = out_tensor_or_tensor_list
     if isinstance(out_tensor_or_tensor_list, list):
         if len(out_tensor_or_tensor_list) != 0:
@@ -110,7 +115,13 @@ def _all_to_all_in_static_mode(
     if isinstance(out_tensor_or_tensor_list, list):
         if not sync_op:
             dist.wait(out_tensor, use_calc_stream=False)
-        out_tensor_or_tensor_list.extend(paddle.split(out_tensor, nranks, 0))
+        # 0D use stack/unstack while others use concat/split
+        if len(in_tensor_or_tensor_list[0].shape) == 0:
+            out_tensor_or_tensor_list.extend(paddle.unstack(out_tensor, 0))
+        else:
+            out_tensor_or_tensor_list.extend(
+                paddle.split(out_tensor, nranks, 0)
+            )
 
     return None
 
diff --git a/python/paddle/distributed/communication/stream/scatter.py b/python/paddle/distributed/communication/stream/scatter.py
index c4a6a66afbcd4..c112516a1fc10 100644
--- a/python/paddle/distributed/communication/stream/scatter.py
+++ b/python/paddle/distributed/communication/stream/scatter.py
@@ -91,7 +91,11 @@ def _scatter_in_static_mode(
                 )
         else:
             tensor_list = [tensor for _ in range(nranks)]
-        input_tensor = paddle.concat(tensor_list, axis=0)
+        # 0D use stack/unstack while others use concat/split
+        if len(tensor_list[0].shape) == 0:
+            input_tensor = paddle.stack(tensor_list, axis=0)
+        else:
+            input_tensor = paddle.concat(tensor_list, axis=0)
 
     ring_id = 0 if group is None else group.id
 

From bfeedd29eafd8aeb623061d9d59c821cba8b7c2d Mon Sep 17 00:00:00 2001
From: mengziheng <121283369+mengziheng@users.noreply.github.com>
Date: Thu, 27 Apr 2023 09:49:56 +0800
Subject: [PATCH 089/405] Pad grad  (#53374)

* add pad op

* add_some_code

* modify some code

* add some code

* add some code

* modify some code

* add some code

* modify some code

* Update composite_backward_api.h

* modify some code

* add some code

* add some code

* add some code
---
 paddle/fluid/operators/pad_op.cc              |  25 ++++
 .../composite_backward_api.h                  |  27 ++++
 .../prim/api/manual_prim/eager_prim_api.cc    |  11 ++
 .../prim/api/manual_prim/prim_manual_api.h    |   8 ++
 .../prim/api/manual_prim/static_prim_api.cc   |  27 ++++
 paddle/phi/api/yaml/legacy_backward.yaml      |   1 +
 .../fluid/tests/unittests/test_pad_op.py      | 134 ++++++++++--------
 7 files changed, 176 insertions(+), 57 deletions(-)

diff --git a/paddle/fluid/operators/pad_op.cc b/paddle/fluid/operators/pad_op.cc
index fd23f57793970..e2a0b3e025381 100644
--- a/paddle/fluid/operators/pad_op.cc
+++ b/paddle/fluid/operators/pad_op.cc
@@ -17,6 +17,9 @@ limitations under the License. */
 #include "paddle/fluid/framework/infershape_utils.h"
 #include "paddle/fluid/framework/op_registry.h"
 #include "paddle/fluid/platform/complex.h"
+#include "paddle/fluid/prim/api/composite_backward/composite_backward_api.h"
+#include "paddle/fluid/prim/utils/static/composite_grad_desc_maker.h"
+#include "paddle/fluid/prim/utils/static/desc_tensor.h"
 #include "paddle/phi/infermeta/unary.h"
 
 namespace paddle {
@@ -129,6 +132,27 @@ class PadOpGradMaker : public framework::SingleGradOpMaker<T> {
   }
 };
 
+class PadCompositeGradOpMaker : public prim::CompositeGradOpMakerBase {
+  using prim::CompositeGradOpMakerBase::CompositeGradOpMakerBase;
+
+ public:
+  void Apply() override {
+    paddle::Tensor x = this->GetSingleForwardInput("X");
+    paddle::Tensor out_grad = this->GetSingleOutputGrad("Out");
+    paddle::Tensor x_grad = this->GetSingleInputGrad("X");
+    auto* dx_ptr = this->GetOutputPtr(&x_grad);
+    std::string dx_name = this->GetOutputName(x_grad);
+
+    std::vector<int> paddings =
+        static_cast<std::vector<int>>(this->Attr<std::vector<int>>("paddings"));
+    float pad_value = static_cast<float>(this->Attr<float>("pad_value"));
+    VLOG(6) << "Runing add_grad composite func";
+
+    prim::pad_grad<prim::DescTensor>(x, out_grad, paddings, pad_value, dx_ptr);
+    this->RecoverOutputName(x_grad, dx_name);
+  }
+};
+
 template <typename T>
 class PadOpDoubleGradMaker : public framework::SingleGradOpMaker<T> {
  public:
@@ -155,6 +179,7 @@ REGISTER_OPERATOR(pad,
                   ops::PadOpMaker,
                   ops::PadOpGradMaker<paddle::framework::OpDesc>,
                   ops::PadOpGradMaker<paddle::imperative::OpBase>,
+                  ops::PadCompositeGradOpMaker,
                   PadInferShapeFunctor);
 REGISTER_OPERATOR(pad_grad,
                   ops::PadOpGrad,
diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
index 099ebc81b900b..9790f36ec590a 100644
--- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
+++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
@@ -1806,6 +1806,32 @@ void roll_grad(const Tensor& x,
   }
 }
 
+template <typename T>
+void pad_grad(const Tensor& input,
+              const Tensor& out_grad,
+              const std::vector<int>& paddings,
+              const Scalar& pad_value,
+              Tensor* input_grad) {
+  if (input_grad) {
+    size_t rank = input.dims().size();
+    auto out_dims = out_grad.dims();
+
+    std::vector<int> starts(rank, 0);
+    std::vector<int64_t> ends(rank, 0);
+    std::vector<int64_t> axes(rank, 0);
+    std::vector<int64_t> infer_flags(rank, 1);
+    std::vector<int64_t> decrease_axis({});
+    for (size_t i = 0; i < rank; ++i) {
+      starts.push_back(static_cast<int>(paddings[2 * i]));
+      ends.push_back(static_cast<int64_t>(out_dims[i] - paddings[2 * i + 1]));
+      axes.push_back(i);
+    }
+    auto out_tmp =
+        slice<T>(out_grad, axes, starts, ends, infer_flags, decrease_axis);
+    set_output<T>(out_tmp, input_grad);
+  }
+}
+
 template <typename T>
 void scatter_nd_add_grad(const Tensor& index,
                          const Tensor& updates,
@@ -1821,5 +1847,6 @@ void scatter_nd_add_grad(const Tensor& index,
     set_output<T>(tmp_updates_grad, updates_grad);
   }
 }
+
 }  // namespace prim
 }  // namespace paddle
diff --git a/paddle/fluid/prim/api/manual_prim/eager_prim_api.cc b/paddle/fluid/prim/api/manual_prim/eager_prim_api.cc
index 852add94fed6f..d667f0fabd71e 100644
--- a/paddle/fluid/prim/api/manual_prim/eager_prim_api.cc
+++ b/paddle/fluid/prim/api/manual_prim/eager_prim_api.cc
@@ -33,5 +33,16 @@ Tensor cast<Tensor>(const Tensor& x, DataType dtype) {
   return ::cast_ad_func(x, dtype);
 }
 
+template <>
+Tensor slice<Tensor>(const Tensor& input,
+                     const std::vector<int64_t>& axes,
+                     const IntArray& starts,
+                     const IntArray& ends,
+                     const std::vector<int64_t>& infer_flags,
+                     const std::vector<int64_t>& decrease_axis) {
+  VLOG(4) << "Eager Prim API slice_ad_func call";
+  return ::slice_ad_func(input, axes, starts, ends, infer_flags, decrease_axis);
+}
+
 }  // namespace prim
 }  // namespace paddle
diff --git a/paddle/fluid/prim/api/manual_prim/prim_manual_api.h b/paddle/fluid/prim/api/manual_prim/prim_manual_api.h
index 383ded54f548d..2146edba60db8 100644
--- a/paddle/fluid/prim/api/manual_prim/prim_manual_api.h
+++ b/paddle/fluid/prim/api/manual_prim/prim_manual_api.h
@@ -38,5 +38,13 @@ Tensor full(const IntArray& shape,
 template <typename T>
 Tensor cast(const Tensor& x, DataType dtype);
 
+template <typename T>
+Tensor slice(const Tensor& input,
+             const std::vector<int64_t>& axes,
+             const IntArray& starts,
+             const IntArray& ends,
+             const std::vector<int64_t>& infer_flags,
+             const std::vector<int64_t>& decrease_axis);
+
 }  // namespace prim
 }  // namespace paddle
diff --git a/paddle/fluid/prim/api/manual_prim/static_prim_api.cc b/paddle/fluid/prim/api/manual_prim/static_prim_api.cc
index efe5e53d173c3..0db4497e6289b 100644
--- a/paddle/fluid/prim/api/manual_prim/static_prim_api.cc
+++ b/paddle/fluid/prim/api/manual_prim/static_prim_api.cc
@@ -127,5 +127,32 @@ Tensor cast<DescTensor>(const Tensor& x, DataType dtype) {
   return out;
 }
 
+template <>
+Tensor slice<DescTensor>(const Tensor& input,
+                         const std::vector<int64_t>& axes,
+                         const IntArray& starts,
+                         const IntArray& ends,
+                         const std::vector<int64_t>& infer_flags,
+                         const std::vector<int64_t>& decrease_axis) {
+  framework::BlockDesc* block = StaticCompositeContext::Instance().GetBlock();
+  framework::OpDesc* op = block->AppendOp();
+  op->SetType("slice");
+  op->SetInput(
+      "Input",
+      {std::static_pointer_cast<prim::DescTensor>(input.impl())->Name()});
+  auto out = empty<DescTensor>({}, phi::DataType::FLOAT32, paddle::Place());
+  op->SetOutput(
+      "Out", {std::static_pointer_cast<prim::DescTensor>(out.impl())->Name()});
+  op->SetAttr("axes", unsafe_vector_cast<int64_t, int>(axes));
+  op->SetAttr("starts", unsafe_vector_cast<int64_t, int>(starts.GetData()));
+  op->SetAttr("ends", unsafe_vector_cast<int64_t, int>(ends.GetData()));
+  op->SetAttr("infer_flags", unsafe_vector_cast<int64_t, int>(infer_flags));
+  op->SetAttr("decrease_axis", unsafe_vector_cast<int64_t, int>(decrease_axis));
+  op->CheckAttrs();
+  op->InferVarType(block);
+  op->InferShape(*block);
+  return out;
+}
+
 }  // namespace prim
 }  // namespace paddle
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index 3f4acc31e6a99..c80e79e3ff207 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -722,6 +722,7 @@
     func : pad_grad
     param: [out_grad, paddings, pad_value]
   no_need_buffer : x
+  composite : pad_grad(x, out_grad, paddings, pad_value, x_grad)
   backward : pad_double_grad
 
 - backward_op : pool2d_double_grad
diff --git a/python/paddle/fluid/tests/unittests/test_pad_op.py b/python/paddle/fluid/tests/unittests/test_pad_op.py
index a40fd7a7409bc..5fd186338444d 100644
--- a/python/paddle/fluid/tests/unittests/test_pad_op.py
+++ b/python/paddle/fluid/tests/unittests/test_pad_op.py
@@ -24,7 +24,9 @@
 
 
 def pad_wrapper(x, paddings, pad_value):
-    return paddle._C_ops.pad(x, paddings, float(pad_value))
+    return paddle.nn.functional.pad(
+        x, pad=list(paddings), mode='constant', value=pad_value
+    )
 
 
 class TestPadOp(OpTest):
@@ -37,7 +39,7 @@ def setUp(self):
             'X': np.random.random(self.shape).astype(self.dtype),
         }
         self.attrs = {}
-        self.attrs['paddings'] = np.array(self.paddings).flatten()
+        self.attrs['paddings'] = list(np.array(self.paddings).flatten())
         self.attrs['pad_value'] = self.pad_value
         self.outputs = {
             'Out': np.pad(
@@ -47,6 +49,9 @@ def setUp(self):
                 constant_values=self.pad_value,
             )
         }
+        self.prim_op_type = "prim"
+        self.public_python_api = pad_wrapper
+        self.enable_cinn = False
 
     def get_dtype(self):
         return np.float64
@@ -55,7 +60,7 @@ def test_check_output(self):
         self.check_output()
 
     def test_check_grad_normal(self):
-        self.check_grad(['X'], 'Out')
+        self.check_grad(['X'], 'Out', check_prim=True)
 
     def initTestCase(self):
         self.shape = (16, 16)
@@ -111,16 +116,19 @@ def test_check_grad_normal(self):
 
 class TestPadOpError(unittest.TestCase):
     def test_errors(self):
-        with program_guard(Program(), Program()):
-            input_data = np.random.random((2, 2)).astype("float32")
+        with paddle.fluid.framework._static_guard():
+            with program_guard(Program(), Program()):
+                input_data = np.random.random((2, 2)).astype("float32")
 
-            def test_Variable():
-                paddle.nn.functional.pad(x=input_data, pad=[1, 1, 1, 1])
+                def test_Variable():
+                    paddle.nn.functional.pad(x=input_data, pad=[1, 1, 1, 1])
 
-            self.assertRaises(TypeError, test_Variable)
+                self.assertRaises(TypeError, test_Variable)
 
-            data = paddle.static.data(name='data', shape=[4], dtype='float16')
-            paddle.nn.functional.pad(x=data, pad=[0, 1])
+                data = paddle.static.data(
+                    name='data', shape=[4], dtype='float16'
+                )
+                paddle.nn.functional.pad(x=data, pad=[0, 1])
 
 
 class TestPaddingValueTensor(UnittestBase):
@@ -129,34 +137,40 @@ def init_info(self):
         self.save_path = os.path.join(self.temp_dir.name, self.path_prefix())
 
     def test_static(self):
-        main_prog = Program()
-        starup_prog = Program()
-        with program_guard(main_prog, starup_prog):
-            fc = paddle.nn.Linear(4, 10)
-            x = paddle.randn([2, 4])
-            x.stop_gradient = False
-            feat = fc(x)  # [2,3,10]
-
-            out = self.call_func(feat)
-
-            sgd = paddle.optimizer.SGD()
-            sgd.minimize(paddle.mean(out))
-            self.assertTrue(self.var_prefix() in str(main_prog))
-
-            exe = paddle.static.Executor()
-            exe.run(starup_prog)
-            res = exe.run(fetch_list=[feat, out])
-            gt = np.pad(res[0], [1, 1], 'constant', constant_values=[1.0, 1.0])
-            np.testing.assert_allclose(res[1], gt)
-            paddle.static.save_inference_model(
-                self.save_path, [x], [feat, out], exe
-            )
-            # Test for Inference Predictor
-            infer_outs = self.infer_prog()
-            gt = np.pad(
-                infer_outs[0], [1, 1], 'constant', constant_values=[1.0, 1.0]
-            )
-            np.testing.assert_allclose(infer_outs[1], gt)
+        with paddle.fluid.framework._static_guard():
+            main_prog = Program()
+            starup_prog = Program()
+            with program_guard(main_prog, starup_prog):
+                fc = paddle.nn.Linear(4, 10)
+                x = paddle.randn([2, 4])
+                x.stop_gradient = False
+                feat = fc(x)  # [2,3,10]
+
+                out = self.call_func(feat)
+
+                sgd = paddle.optimizer.SGD()
+                sgd.minimize(paddle.mean(out))
+                self.assertTrue(self.var_prefix() in str(main_prog))
+
+                exe = paddle.static.Executor()
+                exe.run(starup_prog)
+                res = exe.run(fetch_list=[feat, out])
+                gt = np.pad(
+                    res[0], [1, 1], 'constant', constant_values=[1.0, 1.0]
+                )
+                np.testing.assert_allclose(res[1], gt)
+                paddle.static.save_inference_model(
+                    self.save_path, [x], [feat, out], exe
+                )
+                # Test for Inference Predictor
+                infer_outs = self.infer_prog()
+                gt = np.pad(
+                    infer_outs[0],
+                    [1, 1],
+                    'constant',
+                    constant_values=[1.0, 1.0],
+                )
+                np.testing.assert_allclose(infer_outs[1], gt)
 
     def path_prefix(self):
         return 'padding_value'
@@ -183,23 +197,26 @@ def call_func(self, x):
 
 class TestPaddingValueTensor3(unittest.TestCase):
     def test_static(self):
-        np_x = np.random.random((16, 16)).astype('float32')
-        main_prog = Program()
-        starup_prog = Program()
-        with program_guard(main_prog, starup_prog):
-            x = paddle.assign(np_x).astype('float32')
-            pad_value = paddle.assign([0.0]).astype('float64')
-            y = paddle.nn.functional.pad(x, [0, 1, 2, 3], value=pad_value)
-            loss = y.sum()
-            optimize_ops, params_grads = paddle.optimizer.SGD(0.01).minimize(
-                loss
+        with paddle.fluid.framework._static_guard():
+            np_x = np.random.random((16, 16)).astype('float32')
+            main_prog = Program()
+            starup_prog = Program()
+            with program_guard(main_prog, starup_prog):
+                x = paddle.assign(np_x).astype('float32')
+                pad_value = paddle.assign([0.0]).astype('float64')
+                y = paddle.nn.functional.pad(x, [0, 1, 2, 3], value=pad_value)
+                loss = y.sum()
+                optimize_ops, params_grads = paddle.optimizer.SGD(
+                    0.01
+                ).minimize(loss)
+
+            exe = paddle.static.Executor(paddle.CPUPlace())
+            res = exe.run(
+                main_prog, fetch_list=[y] + [g for p, g in params_grads]
             )
-
-        exe = paddle.static.Executor(paddle.CPUPlace())
-        res = exe.run(main_prog, fetch_list=[y] + [g for p, g in params_grads])
-        pd_out = res[0]
-        np_out = np.pad(np_x, [(0, 1), (2, 3)], constant_values=0.0)
-        np.testing.assert_allclose(pd_out, np_out)
+            pd_out = res[0]
+            np_out = np.pad(np_x, [(0, 1), (2, 3)], constant_values=0.0)
+            np.testing.assert_allclose(pd_out, np_out)
 
 
 @unittest.skipIf(
@@ -215,13 +232,16 @@ def setUp(self):
         self.python_api = pad_wrapper
         x = np.random.random(self.shape).astype(np.float32)
         self.attrs = {}
-        self.attrs['paddings'] = np.array(self.paddings).flatten()
+        self.attrs['paddings'] = list(np.array(self.paddings).flatten())
         self.attrs['pad_value'] = self.pad_value
         out = np.pad(
             x, self.paddings, mode='constant', constant_values=self.pad_value
         )
         self.inputs = {'X': convert_float_to_uint16(x)}
         self.outputs = {'Out': convert_float_to_uint16(out)}
+        self.enable_cinn = False
+        self.prim_op_type = "prim"
+        self.public_python_api = pad_wrapper
 
     def initTestCase(self):
         self.shape = (16, 16)
@@ -234,9 +254,9 @@ def test_check_output(self):
 
     def test_check_grad(self):
         place = core.CUDAPlace(0)
-        self.check_grad_with_place(place, ['X'], 'Out')
+        self.check_grad_with_place(place, ['X'], 'Out', check_prim=True)
 
 
 if __name__ == '__main__':
-    paddle.enable_static()
+    # paddle.enable_static()
     unittest.main()

From 82ac39132c49ca2792a37c2166f9d9242069d63d Mon Sep 17 00:00:00 2001
From: cyberslack_lee <luhputu0815@gmail.com>
Date: Thu, 27 Apr 2023 10:15:50 +0800
Subject: [PATCH 090/405] =?UTF-8?q?=E3=80=90Hackathon4=E3=80=91No5=20nexta?=
 =?UTF-8?q?fter=20=20(#52544)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 paddle/phi/api/yaml/ops.yaml                  |  10 ++
 paddle/phi/kernels/cpu/nextafter_kernel.cc    |  22 ++++
 paddle/phi/kernels/gpu/nextafter_kernel.cu    |  22 ++++
 .../phi/kernels/impl/nextafter_kernel_impl.h  |  84 +++++++++++++
 paddle/phi/kernels/nextafter_kernel.h         |  28 +++++
 python/paddle/__init__.py                     |   2 +
 .../tests/unittests/test_nextafter_op.py      | 118 ++++++++++++++++++
 python/paddle/tensor/__init__.py              |   2 +
 python/paddle/tensor/math.py                  |  36 ++++++
 9 files changed, 324 insertions(+)
 create mode 100644 paddle/phi/kernels/cpu/nextafter_kernel.cc
 create mode 100644 paddle/phi/kernels/gpu/nextafter_kernel.cu
 create mode 100644 paddle/phi/kernels/impl/nextafter_kernel_impl.h
 create mode 100644 paddle/phi/kernels/nextafter_kernel.h
 create mode 100644 python/paddle/fluid/tests/unittests/test_nextafter_op.py

diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index 507fef3309660..8fc8c4c9b081c 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -1421,6 +1421,16 @@
   data_transform :
     skip_transform : out_size, size_tensor, scale_tensor
 
+- op : nextafter
+  args : (Tensor x, Tensor y)
+  output : Tensor(out)
+  infer_meta :
+    func : ElementwiseInferMeta
+    param: [x, y]
+  kernel :
+    func : nextafter
+    data_type : x
+
 - op : nll_loss
   args : (Tensor input, Tensor label, Tensor weight, int64_t ignore_index = -100, str reduction = "mean")
   output : Tensor(out), Tensor(total_weight)
diff --git a/paddle/phi/kernels/cpu/nextafter_kernel.cc b/paddle/phi/kernels/cpu/nextafter_kernel.cc
new file mode 100644
index 0000000000000..ac4ab00a4d3fe
--- /dev/null
+++ b/paddle/phi/kernels/cpu/nextafter_kernel.cc
@@ -0,0 +1,22 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/nextafter_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/impl/nextafter_kernel_impl.h"
+
+PD_REGISTER_KERNEL(
+    nextafter, CPU, ALL_LAYOUT, phi::NextafterKernel, float, double) {}
diff --git a/paddle/phi/kernels/gpu/nextafter_kernel.cu b/paddle/phi/kernels/gpu/nextafter_kernel.cu
new file mode 100644
index 0000000000000..e0ac8212853c9
--- /dev/null
+++ b/paddle/phi/kernels/gpu/nextafter_kernel.cu
@@ -0,0 +1,22 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/nextafter_kernel.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/impl/nextafter_kernel_impl.h"
+
+PD_REGISTER_KERNEL(
+    nextafter, GPU, ALL_LAYOUT, phi::NextafterKernel, float, double) {}
diff --git a/paddle/phi/kernels/impl/nextafter_kernel_impl.h b/paddle/phi/kernels/impl/nextafter_kernel_impl.h
new file mode 100644
index 0000000000000..6d54009282528
--- /dev/null
+++ b/paddle/phi/kernels/impl/nextafter_kernel_impl.h
@@ -0,0 +1,84 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include <algorithm>
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/device_context.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
+#include "paddle/phi/kernels/funcs/math.h"
+#include "paddle/phi/kernels/nextafter_kernel.h"
+namespace phi {
+template <typename T>
+struct NextafterOut {
+  using type = T;
+};
+
+template <>
+struct NextafterOut<int32_t> {
+  using type = double;
+};
+
+template <>
+struct NextafterOut<int64_t> {
+  using type = double;
+};
+template <typename T>
+struct NextafterFunctor {
+  NextafterFunctor(const T* x,
+                   const T* y,
+                   typename NextafterOut<T>::type* out,
+                   int64_t numel)
+      : x_(x), y_(y), out_(out), numel_(numel) {}
+
+  HOSTDEVICE void operator()(int64_t idx) const {
+    out_[idx] = static_cast<typename NextafterOut<T>::type>(std::nextafter(
+        static_cast<float>(x_[idx]), static_cast<float>(y_[idx])));
+  }
+  const T* x_;
+  const T* y_;
+  typename NextafterOut<T>::type* out_;
+  int64_t numel_;
+};
+template <>
+struct NextafterFunctor<double> {
+  NextafterFunctor(const double* x, const double* y, double* out, int64_t numel)
+      : x_(x), y_(y), out_(out), numel_(numel) {}
+
+  HOSTDEVICE void operator()(int64_t idx) const {
+    out_[idx] = std::nextafter(x_[idx], y_[idx]);
+  }
+
+  const double* x_;
+  const double* y_;
+  double* out_;
+  int64_t numel_;
+};
+
+template <typename T, typename Context>
+void NextafterKernel(const Context& ctx,
+                     const DenseTensor& x,
+                     const DenseTensor& y,
+                     DenseTensor* out) {
+  auto* out_data = ctx.template Alloc<T>(out);
+  auto x_data = x.data<T>();
+  auto y_data = y.data<T>();
+  auto x_numel = x.numel();
+
+  phi::funcs::ForRange<Context> for_range(ctx, x_numel);
+  phi::NextafterFunctor<T> functor(x_data, y_data, out_data, x_numel);
+  for_range(functor);
+}
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/nextafter_kernel.h b/paddle/phi/kernels/nextafter_kernel.h
new file mode 100644
index 0000000000000..3a185e39bd940
--- /dev/null
+++ b/paddle/phi/kernels/nextafter_kernel.h
@@ -0,0 +1,28 @@
+
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void NextafterKernel(const Context& dev_ctx,
+                     const DenseTensor& x,
+                     const DenseTensor& y,
+                     DenseTensor* out);
+
+}  // namespace phi
diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py
index f319ab27c063a..d155dce987c08 100644
--- a/python/paddle/__init__.py
+++ b/python/paddle/__init__.py
@@ -299,6 +299,7 @@
 from .tensor.math import trapezoid  # noqa: F401
 from .tensor.math import cumulative_trapezoid  # noqa: F401
 from .tensor.math import vander  # noqa: F401
+from .tensor.math import nextafter  # noqa: F401
 
 from .tensor.random import bernoulli  # noqa: F401
 from .tensor.random import poisson  # noqa: F401
@@ -688,4 +689,5 @@
     'cumulative_trapezoid',
     'polar',
     'vander',
+    'nextafter',
 ]
diff --git a/python/paddle/fluid/tests/unittests/test_nextafter_op.py b/python/paddle/fluid/tests/unittests/test_nextafter_op.py
new file mode 100644
index 0000000000000..5048778e1b7b7
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_nextafter_op.py
@@ -0,0 +1,118 @@
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from eager_op_test import OpTest
+
+import paddle
+
+
+def ref_nextafter(x, y):
+    out = np.nextafter(x, y)
+    return out
+
+
+class TestNextafterAPI(unittest.TestCase):
+    def setUp(self):
+        self.x = np.random.rand(2, 3, 4, 5).astype('float32')
+        self.y = np.random.rand(2, 3, 4, 5).astype('float32')
+        self.x1 = np.array([0, 0, 10]).astype("float32")
+        self.y1 = np.array([np.inf, -np.inf, 10]).astype("float32")
+        self.x2 = np.random.rand(100).astype("float32")
+        self.y2 = np.random.rand(100).astype("float32")
+        self.place = (
+            paddle.CUDAPlace(0)
+            if paddle.is_compiled_with_cuda()
+            else paddle.CPUPlace()
+        )
+
+    def test_static_api(self):
+        paddle.enable_static()
+        with paddle.static.program_guard(paddle.static.Program()):
+            x = paddle.static.data(
+                name='x', shape=self.x.shape, dtype='float32'
+            )
+            y = paddle.static.data(
+                name='y', shape=self.y.shape, dtype='float32'
+            )
+            out = paddle.nextafter(x, y)
+            exe = paddle.static.Executor(self.place)
+            res = exe.run(feed={'x': self.x, 'y': self.y}, fetch_list=[out])
+        out_ref = ref_nextafter(self.x, self.y)
+        np.testing.assert_allclose(out_ref, res[0], rtol=1e-05)
+
+        with paddle.static.program_guard(paddle.static.Program()):
+            x1 = paddle.static.data(
+                name='x', shape=self.x1.shape, dtype='float32'
+            )
+            y1 = paddle.static.data(
+                name='y', shape=self.y1.shape, dtype='float32'
+            )
+            out = paddle.nextafter(x1, y1)
+            exe = paddle.static.Executor(self.place)
+            res = exe.run(feed={'x': self.x1, 'y': self.y1}, fetch_list=[out])
+        out_ref = ref_nextafter(self.x1, self.y1)
+        np.testing.assert_allclose(out_ref, res[0], rtol=1e-05)
+
+        with paddle.static.program_guard(paddle.static.Program()):
+            x2 = paddle.static.data(
+                name='x', shape=self.x2.shape, dtype='float32'
+            )
+            y2 = paddle.static.data(
+                name='y', shape=self.y2.shape, dtype='float32'
+            )
+            out = paddle.nextafter(x2, y2)
+            exe = paddle.static.Executor(self.place)
+            res = exe.run(feed={'x': self.x2, 'y': self.y2}, fetch_list=[out])
+        out_ref = ref_nextafter(self.x2, self.y2)
+        np.testing.assert_allclose(out_ref, res[0], rtol=1e-05)
+
+    def test_dygraph_api(self):
+        paddle.disable_static(self.place)
+        x = paddle.to_tensor(self.x)
+        y = paddle.to_tensor(self.y)
+        out = paddle.nextafter(x, y)
+        out_ref = ref_nextafter(self.x, self.y)
+        np.testing.assert_allclose(out_ref, out.numpy(), rtol=1e-05)
+        paddle.enable_static()
+
+
+class TestNextafterOP(OpTest):
+    def setUp(self):
+        self.op_type = "nextafter"
+        self.python_api = paddle.nextafter
+        self.init_dtype()
+
+        x = np.array([1, 2]).astype(self.dtype)
+        y = np.array([2, 1]).astype(self.dtype)
+        out = np.nextafter(x, y)
+        self.inputs = {'x': x, 'y': y}
+        self.outputs = {'out': out}
+
+    def test_check_output(self):
+        self.check_output()
+
+    def init_dtype(self):
+        self.dtype = np.float64
+
+
+class TestNextafterOPFP32(TestNextafterOP):
+    def init_dtype(self):
+        self.dtype = np.float32
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/python/paddle/tensor/__init__.py b/python/paddle/tensor/__init__.py
index b78ac0e57c22e..bea2fd7323d9f 100644
--- a/python/paddle/tensor/__init__.py
+++ b/python/paddle/tensor/__init__.py
@@ -251,6 +251,7 @@
 from .math import sigmoid  # noqa: F401
 from .math import sigmoid_  # noqa: F401
 from .math import vander  # noqa: F401
+from .math import nextafter  # noqa: F401
 
 from .random import multinomial  # noqa: F401
 from .random import standard_normal  # noqa: F401
@@ -540,6 +541,7 @@
     'sigmoid',
     'sigmoid_',
     'vander',
+    'nextafter',
 ]
 
 # this list used in math_op_patch.py for magic_method bind
diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py
index 2f94f0a7e2013..37519996a2acd 100644
--- a/python/paddle/tensor/math.py
+++ b/python/paddle/tensor/math.py
@@ -5486,3 +5486,39 @@ def vander(x, n=None, increasing=False, name=None):
         res[:, 1:] = paddle.cumprod(res[:, 1:], dim=-1)
     res = res[:, ::-1] if not increasing else res
     return res
+
+
+def nextafter(x, y, name=None):
+    r"""
+    Return the next floating-point value after input towards other, elementwise.
+    The shapes of input and other must be broadcastable.
+
+    Args:
+        x (Tensor): An N-D Tensor, the data type is float32, float64.
+        y (Tensor): An N-D Tensor, the data type is float32, float64.
+        name(str, optional):Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
+
+    Returns:
+        out (Tensor): An N-D Tensor, the shape and data type is the same with input.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+            out = paddle.nextafter(paddle.to_tensor([1.0,2.0]),paddle.to_tensor([2.0,1.0]))
+            print(out)
+            #Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #       [1.00000012, 1.99999988])
+    """
+    if in_dygraph_mode():
+        return _C_ops.nextafter(x, y)
+    else:
+        check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'nextafter')
+        check_variable_and_dtype(y, 'y', ['float32', 'float64'], 'nextafter')
+        op_type = "nextafter"
+        helper = LayerHelper(op_type, **locals())
+        inputs = {"x": x, "y": y}
+        out = helper.create_variable_for_type_inference(dtype=paddle.float32)
+        outputs = {"out": out}
+        helper.append_op(type=op_type, inputs=inputs, outputs=outputs)
+    return out

From 8a6ad6e5d353ab116c6c6c60245295fe71b2a391 Mon Sep 17 00:00:00 2001
From: superwinner1 <82640284+superwinner1@users.noreply.github.com>
Date: Thu, 27 Apr 2023 10:21:14 +0800
Subject: [PATCH 091/405] =?UTF-8?q?=E3=80=90Hackathon=20No.55=E3=80=91add?=
 =?UTF-8?q?=20fmax=20BF16=20test=20(#51925)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../phi/kernels/funcs/elementwise_functor.h   | 11 +++++++
 .../kernels/gpu/elementwise_grad_kernel.cu    |  1 +
 paddle/phi/kernels/kps/elementwise_kernel.cu  |  1 +
 .../fluid/tests/unittests/test_fmax_op.py     | 31 ++++++++++++++++++-
 4 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/paddle/phi/kernels/funcs/elementwise_functor.h b/paddle/phi/kernels/funcs/elementwise_functor.h
index dc78bd7098411..af0ae3b6447be 100644
--- a/paddle/phi/kernels/funcs/elementwise_functor.h
+++ b/paddle/phi/kernels/funcs/elementwise_functor.h
@@ -229,6 +229,17 @@ struct FMaxFunctor<dtype::float16> {
   }
 };
 
+template <>
+struct FMaxFunctor<dtype::bfloat16> {
+  inline HOSTDEVICE dtype::bfloat16 operator()(const dtype::bfloat16 a,
+                                               const dtype::bfloat16 b) const {
+    float float_a = static_cast<float>(a);
+    float float_b = static_cast<float>(b);
+    auto result = std::fmax(float_a, float_b);
+    return static_cast<dtype::bfloat16>(result);
+  }
+};
+
 template <>
 struct FMaxFunctor<int> {
   inline HOSTDEVICE int operator()(const int a, const int b) const {
diff --git a/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu b/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu
index b3ad0dacae37c..30e222663da10 100644
--- a/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu
@@ -97,6 +97,7 @@ PD_REGISTER_KERNEL(fmax_grad,
                    double,
                    int,
                    phi::dtype::float16,
+                   phi::dtype::bfloat16,
                    int64_t) {}
 
 PD_REGISTER_KERNEL(fmin_grad,
diff --git a/paddle/phi/kernels/kps/elementwise_kernel.cu b/paddle/phi/kernels/kps/elementwise_kernel.cu
index 80a969c4fabb4..cbf811a9830b0 100644
--- a/paddle/phi/kernels/kps/elementwise_kernel.cu
+++ b/paddle/phi/kernels/kps/elementwise_kernel.cu
@@ -155,6 +155,7 @@ PD_REGISTER_KERNEL(fmax,
                    double,
                    int,
                    float16,
+                   bfloat16,
                    int64_t) {}
 
 PD_REGISTER_KERNEL(fmin,
diff --git a/python/paddle/fluid/tests/unittests/test_fmax_op.py b/python/paddle/fluid/tests/unittests/test_fmax_op.py
index 19417919df248..0271854aebb72 100644
--- a/python/paddle/fluid/tests/unittests/test_fmax_op.py
+++ b/python/paddle/fluid/tests/unittests/test_fmax_op.py
@@ -15,7 +15,7 @@
 import unittest
 
 import numpy as np
-from eager_op_test import OpTest
+from eager_op_test import OpTest, convert_float_to_uint16
 
 import paddle
 from paddle.fluid import core
@@ -241,5 +241,34 @@ def test_check_grad_normal(self):
         self.check_grad(['X', 'Y'], 'Out')
 
 
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA and not support the bfloat16",
+)
+class TestFmaxBF16OP(OpTest):
+    def setUp(self):
+        self.op_type = "elementwise_fmax"
+        self.python_api = paddle.fmax
+        self.dtype = np.uint16
+        x = np.random.uniform(0.1, 1, [13, 17]).astype("float32")
+        sgn = np.random.choice([-1, 1], [13, 17]).astype("float32")
+        y = x + sgn * np.random.uniform(0.1, 1, [13, 17]).astype("float32")
+        out = np.fmax(x, y)
+        self.inputs = {
+            'X': convert_float_to_uint16(x),
+            'Y': convert_float_to_uint16(y),
+        }
+        self.outputs = {'Out': convert_float_to_uint16(out)}
+
+    def test_check_output(self):
+        place = core.CUDAPlace(0)
+        self.check_output_with_place(place)
+
+    def test_check_grad(self):
+        place = core.CUDAPlace(0)
+        self.check_grad_with_place(place, ['X', 'Y'], 'Out')
+
+
 if __name__ == "__main__":
     unittest.main()

From 166964b109a7bbbfbc2dea040610eb1f38c9602e Mon Sep 17 00:00:00 2001
From: risemeup1 <62429225+risemeup1@users.noreply.github.com>
Date: Thu, 27 Apr 2023 10:21:40 +0800
Subject: [PATCH 092/405] update cmake3.16 to 3.18 (#53288)

* update cmake3.16 to 3.18

* test

* Update Dockerfile.ubuntu
---
 cmake/external/dlpack.cmake          | 1 +
 tools/dockerfile/Dockerfile.ubuntu   | 4 ++--
 tools/dockerfile/Dockerfile.ubuntu18 | 4 ++--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/cmake/external/dlpack.cmake b/cmake/external/dlpack.cmake
index 727202a434683..fc631c1d184aa 100644
--- a/cmake/external/dlpack.cmake
+++ b/cmake/external/dlpack.cmake
@@ -17,6 +17,7 @@ include(ExternalProject)
 set(DLPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/dlpack)
 
 set(DLPACK_REPOSITORY ${GIT_URL}/dmlc/dlpack.git)
+
 set(DLPACK_TAG v0.4)
 
 set(DLPACK_INCLUDE_DIR ${THIRD_PARTY_PATH}/dlpack/src/extern_dlpack/include)
diff --git a/tools/dockerfile/Dockerfile.ubuntu b/tools/dockerfile/Dockerfile.ubuntu
index fba5365f8b25e..fd7e476f710ab 100644
--- a/tools/dockerfile/Dockerfile.ubuntu
+++ b/tools/dockerfile/Dockerfile.ubuntu
@@ -42,8 +42,8 @@ RUN wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellch
 
 # install cmake
 WORKDIR /home
-RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz
-ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH
+RUN wget -q https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.18.0-Linux-x86_64.tar.gz && rm cmake-3.18.0-Linux-x86_64.tar.gz
+ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH
 
 # Install Python3.7
 RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \
diff --git a/tools/dockerfile/Dockerfile.ubuntu18 b/tools/dockerfile/Dockerfile.ubuntu18
index 35397967f283f..41f8f97db0018 100644
--- a/tools/dockerfile/Dockerfile.ubuntu18
+++ b/tools/dockerfile/Dockerfile.ubuntu18
@@ -50,8 +50,8 @@ ENV PATH=/usr/local/gcc-8.2/bin:$PATH
 
 # install cmake
 WORKDIR /home
-RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz
-ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH
+RUN wget -q https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.18.0-Linux-x86_64.tar.gz && rm cmake-3.18.0-Linux-x86_64.tar.gz
+ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH
 
 
 RUN apt-get update && \

From eee9c7887aaffb92cb2247c81a58fc88dcc2bd73 Mon Sep 17 00:00:00 2001
From: huangjiyi <43315610+huangjiyi@users.noreply.github.com>
Date: Thu, 27 Apr 2023 10:30:37 +0800
Subject: [PATCH 093/405] Register fluid xpu kerenls to phi [part 2] (#53188)

* update

* fix bug
---
 .../c_softmax_with_cross_entropy_op_xpu.cc    | 20 ++++++++-----
 .../operators/collective/c_split_op_xpu.cc    | 13 ++++----
 .../collective/c_sync_calc_stream_op_xpu.cc   |  4 +--
 .../collective/c_sync_comm_stream_op_xpu.cc   |  4 +--
 .../operators/controlflow/logical_op_xpu.h    |  7 ++++-
 .../controlflow/logicaland_op_xpu.cc          | 27 +++++++++++------
 .../controlflow/logicalnot_op_xpu.cc          | 19 +++++++-----
 .../operators/controlflow/logicalor_op_xpu.cc | 26 ++++++++++------
 .../detection/iou_similarity_op_xpu.cc        |  5 ++--
 .../fused/fused_gemm_epilogue_op_xpu.cc       | 30 ++++++++++---------
 paddle/fluid/operators/load_combine_op_xpu.cc | 17 ++++++-----
 paddle/fluid/operators/lod_reset_op.cc        | 16 +++++-----
 paddle/fluid/operators/log_loss_op_xpu.cc     | 13 ++++----
 .../operators/metrics/accuracy_op_xpu.cc      |  8 ++---
 .../optimizers/lars_momentum_op_xpu.cc        | 12 +++++---
 .../operators/reduce_ops/logsumexp_op_xpu.cc  |  4 +++
 16 files changed, 134 insertions(+), 91 deletions(-)

diff --git a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op_xpu.cc b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op_xpu.cc
index 83824c8108401..695c28d77554a 100644
--- a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op_xpu.cc
@@ -29,7 +29,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename DeviceContext, typename T>
+template <typename T, typename DeviceContext>
 class CSoftmaxWithCrossEntropyOp : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
@@ -468,7 +468,7 @@ struct CSoftmaxWithCrossEntropyFunctor<phi::XPUContext, T> {
   }
 };
 
-template <typename DeviceContext, typename T>
+template <typename T, typename DeviceContext>
 class CSoftmaxWithCrossEntropyGrad : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& context) const override {
@@ -535,9 +535,13 @@ class CSoftmaxWithCrossEntropyGrad : public framework::OpKernel<T> {
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-REGISTER_OP_XPU_KERNEL(c_softmax_with_cross_entropy,
-                       ops::CSoftmaxWithCrossEntropyOp<phi::XPUContext, float>);
-
-REGISTER_OP_XPU_KERNEL(
-    c_softmax_with_cross_entropy_grad,
-    ops::CSoftmaxWithCrossEntropyGrad<phi::XPUContext, float>);
+PD_REGISTER_STRUCT_KERNEL(c_softmax_with_cross_entropy,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::CSoftmaxWithCrossEntropyOp,
+                          float) {}
+PD_REGISTER_STRUCT_KERNEL(c_softmax_with_cross_entropy_grad,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::CSoftmaxWithCrossEntropyGrad,
+                          float) {}
diff --git a/paddle/fluid/operators/collective/c_split_op_xpu.cc b/paddle/fluid/operators/collective/c_split_op_xpu.cc
index bad920a11ff5e..d573a83d708c4 100644
--- a/paddle/fluid/operators/collective/c_split_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_split_op_xpu.cc
@@ -22,7 +22,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename T>
+template <typename T, typename DeviceContext>
 class CSplitOpXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
@@ -87,7 +87,10 @@ class CSplitOpXPUKernel : public framework::OpKernel<T> {
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-REGISTER_OP_XPU_KERNEL(c_split,
-                       ops::CSplitOpXPUKernel<float>,
-                       ops::CSplitOpXPUKernel<int>,
-                       ops::CSplitOpXPUKernel<plat::float16>);
+PD_REGISTER_STRUCT_KERNEL(c_split,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::CSplitOpXPUKernel,
+                          float,
+                          int,
+                          plat::float16) {}
diff --git a/paddle/fluid/operators/collective/c_sync_calc_stream_op_xpu.cc b/paddle/fluid/operators/collective/c_sync_calc_stream_op_xpu.cc
index 4917057059ffc..0b432cab281fc 100644
--- a/paddle/fluid/operators/collective/c_sync_calc_stream_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_sync_calc_stream_op_xpu.cc
@@ -17,5 +17,5 @@ limitations under the License. */
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-REGISTER_OP_XPU_KERNEL(c_sync_calc_stream,
-                       ops::CSyncCalcStreamKernel<float, plat::XPUPlace>)
+PD_REGISTER_STRUCT_KERNEL(
+    c_sync_calc_stream, XPU, ALL_LAYOUT, ops::CSyncCalcStreamKernel, float) {}
diff --git a/paddle/fluid/operators/collective/c_sync_comm_stream_op_xpu.cc b/paddle/fluid/operators/collective/c_sync_comm_stream_op_xpu.cc
index bbb3b62bf3cf6..ce2c20d57f0b3 100644
--- a/paddle/fluid/operators/collective/c_sync_comm_stream_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_sync_comm_stream_op_xpu.cc
@@ -17,5 +17,5 @@ limitations under the License. */
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-REGISTER_OP_XPU_KERNEL(c_sync_comm_stream,
-                       ops::CSyncCommStreamKernel<float, plat::XPUPlace>);
+PD_REGISTER_STRUCT_KERNEL(
+    c_sync_comm_stream, XPU, ALL_LAYOUT, ops::CSyncCommStreamKernel, float) {}
diff --git a/paddle/fluid/operators/controlflow/logical_op_xpu.h b/paddle/fluid/operators/controlflow/logical_op_xpu.h
index 8afefd6837449..614db61558f79 100644
--- a/paddle/fluid/operators/controlflow/logical_op_xpu.h
+++ b/paddle/fluid/operators/controlflow/logical_op_xpu.h
@@ -156,7 +156,12 @@ class BinaryLogicalOpXPUKernel : public framework::OpKernel<T> {
   }
 };
 
-template <typename T>
+#define DEFINE_BINARY_LOGICAL_OP_XPU_KERNEL(op_name, xpu_type) \
+  template <typename T, typename DeviceContext>                \
+  class BinaryLogical##op_name##CPUKernel                      \
+      : public CReduceOpCPUKernel<xpu_type, T> {};
+
+template <typename T, typename DeviceContext>
 class UnaryLogicalOpXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& context) const override {
diff --git a/paddle/fluid/operators/controlflow/logicaland_op_xpu.cc b/paddle/fluid/operators/controlflow/logicaland_op_xpu.cc
index 6248b6e0b0637..563317f209ebc 100644
--- a/paddle/fluid/operators/controlflow/logicaland_op_xpu.cc
+++ b/paddle/fluid/operators/controlflow/logicaland_op_xpu.cc
@@ -14,14 +14,23 @@ limitations under the License. */
 
 #ifdef PADDLE_WITH_XPU
 #include "paddle/fluid/operators/controlflow/logical_op_xpu.h"
+
+namespace paddle {
+namespace operators {
+DEFINE_BINARY_LOGICAL_OP_XPU_KERNEL(AND, XpuLogicalType::XPU_AND);
+}  // namespace operators
+}  // namespace paddle
+
 namespace ops = paddle::operators;
-REGISTER_OP_XPU_KERNEL(
-    logical_and,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_AND, bool>,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_AND, int8_t>,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_AND, int16_t>,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_AND, int>,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_AND, int64_t>,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_AND, float>,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_AND, double>);
+PD_REGISTER_STRUCT_KERNEL(logical_and,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::BinaryLogicalANDCPUKernel,
+                          bool,
+                          int8_t,
+                          int16_t,
+                          int,
+                          int64_t,
+                          float,
+                          double) {}
 #endif
diff --git a/paddle/fluid/operators/controlflow/logicalnot_op_xpu.cc b/paddle/fluid/operators/controlflow/logicalnot_op_xpu.cc
index b8336c7201c3b..1431816810b1e 100644
--- a/paddle/fluid/operators/controlflow/logicalnot_op_xpu.cc
+++ b/paddle/fluid/operators/controlflow/logicalnot_op_xpu.cc
@@ -15,12 +15,15 @@ limitations under the License. */
 #ifdef PADDLE_WITH_XPU
 #include "paddle/fluid/operators/controlflow/logical_op_xpu.h"
 namespace ops = paddle::operators;
-REGISTER_OP_XPU_KERNEL(logicalnot,
-                       ops::UnaryLogicalOpXPUKernel<bool>,
-                       ops::UnaryLogicalOpXPUKernel<int8_t>,
-                       ops::UnaryLogicalOpXPUKernel<int16_t>,
-                       ops::UnaryLogicalOpXPUKernel<int>,
-                       ops::UnaryLogicalOpXPUKernel<int64_t>,
-                       ops::UnaryLogicalOpXPUKernel<float>,
-                       ops::UnaryLogicalOpXPUKernel<double>);
+PD_REGISTER_STRUCT_KERNEL(logicalnot,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::UnaryLogicalOpXPUKernel,
+                          bool,
+                          int8_t,
+                          int16_t,
+                          int,
+                          int64_t,
+                          float,
+                          double) {}
 #endif
diff --git a/paddle/fluid/operators/controlflow/logicalor_op_xpu.cc b/paddle/fluid/operators/controlflow/logicalor_op_xpu.cc
index 126596841a29f..03c707222e44e 100644
--- a/paddle/fluid/operators/controlflow/logicalor_op_xpu.cc
+++ b/paddle/fluid/operators/controlflow/logicalor_op_xpu.cc
@@ -15,14 +15,22 @@ limitations under the License. */
 #ifdef PADDLE_WITH_XPU
 #include "paddle/fluid/operators/controlflow/logical_op_xpu.h"
 
+namespace paddle {
+namespace operators {
+DEFINE_BINARY_LOGICAL_OP_XPU_KERNEL(OR, XpuLogicalType::XPU_OR);
+}  // namespace operators
+}  // namespace paddle
+
 namespace ops = paddle::operators;
-REGISTER_OP_XPU_KERNEL(
-    logical_or,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_OR, bool>,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_OR, int8_t>,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_OR, int16_t>,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_OR, int>,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_OR, int64_t>,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_OR, float>,
-    ops::BinaryLogicalOpXPUKernel<ops::XpuLogicalType::XPU_OR, double>);
+PD_REGISTER_STRUCT_KERNEL(logical_or,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::BinaryLogicalORCPUKernel,
+                          bool,
+                          int8_t,
+                          int16_t,
+                          int,
+                          int64_t,
+                          float,
+                          double) {}
 #endif
diff --git a/paddle/fluid/operators/detection/iou_similarity_op_xpu.cc b/paddle/fluid/operators/detection/iou_similarity_op_xpu.cc
index 1dc669075b17a..27ffa64c2a892 100644
--- a/paddle/fluid/operators/detection/iou_similarity_op_xpu.cc
+++ b/paddle/fluid/operators/detection/iou_similarity_op_xpu.cc
@@ -19,7 +19,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename DeviceContext, typename T>
+template <typename T, typename DeviceContext>
 class XPUIOUSimilarityKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
@@ -57,6 +57,7 @@ class XPUIOUSimilarityKernel : public framework::OpKernel<T> {
 namespace ops = paddle::operators;
 using XPU = paddle::platform::XPUDeviceContext;
 
-REGISTER_OP_XPU_KERNEL(iou_similarity, ops::XPUIOUSimilarityKernel<XPU, float>);
+PD_REGISTER_STRUCT_KERNEL(
+    iou_similarity, XPU, ALL_LAYOUT, ops::XPUIOUSimilarityKernel, float) {}
 
 #endif
diff --git a/paddle/fluid/operators/fused/fused_gemm_epilogue_op_xpu.cc b/paddle/fluid/operators/fused/fused_gemm_epilogue_op_xpu.cc
index 82b437b943cb4..6594df2f5164f 100644
--- a/paddle/fluid/operators/fused/fused_gemm_epilogue_op_xpu.cc
+++ b/paddle/fluid/operators/fused/fused_gemm_epilogue_op_xpu.cc
@@ -22,7 +22,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename DeviceContext, typename T>
+template <typename T, typename DeviceContext>
 class FusedGemmEpilogueXPUKernel : public framework::OpKernel<T> {
   using XPUType = typename XPUTypeTrait<T>::Type;
 
@@ -102,7 +102,7 @@ class FusedGemmEpilogueXPUKernel : public framework::OpKernel<T> {
   }
 };
 
-template <typename DeviceContext, typename T>
+template <typename T, typename DeviceContext>
 class FusedGemmEpilogueXPUGradKernel : public framework::OpKernel<T> {
   using XPUType = typename XPUTypeTrait<T>::Type;
 
@@ -227,15 +227,17 @@ class FusedGemmEpilogueXPUGradKernel : public framework::OpKernel<T> {
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-
-REGISTER_OP_XPU_KERNEL(
-    fused_gemm_epilogue,
-    ops::FusedGemmEpilogueXPUKernel<phi::XPUContext, float>,
-    ops::FusedGemmEpilogueXPUKernel<phi::XPUContext,
-                                    paddle::platform::float16>);
-
-REGISTER_OP_XPU_KERNEL(
-    fused_gemm_epilogue_grad,
-    ops::FusedGemmEpilogueXPUGradKernel<phi::XPUContext, float>,
-    ops::FusedGemmEpilogueXPUGradKernel<phi::XPUContext,
-                                        paddle::platform::float16>);
+namespace plat = paddle::platform;
+
+PD_REGISTER_STRUCT_KERNEL(fused_gemm_epilogue,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::FusedGemmEpilogueXPUKernel,
+                          float,
+                          plat::float16) {}
+PD_REGISTER_STRUCT_KERNEL(fused_gemm_epilogue_grad,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::FusedGemmEpilogueXPUGradKernel,
+                          float,
+                          plat::float16) {}
diff --git a/paddle/fluid/operators/load_combine_op_xpu.cc b/paddle/fluid/operators/load_combine_op_xpu.cc
index 307c9042c543d..d285af37cda98 100644
--- a/paddle/fluid/operators/load_combine_op_xpu.cc
+++ b/paddle/fluid/operators/load_combine_op_xpu.cc
@@ -15,11 +15,12 @@ limitations under the License. */
 #include "paddle/fluid/operators/load_combine_op.h"
 
 namespace ops = paddle::operators;
-using XPUCtx = paddle::platform::XPUDeviceContext;
-
-REGISTER_OP_XPU_KERNEL(load_combine,
-                       ops::LoadCombineOpKernel<float, XPUCtx>,
-                       ops::LoadCombineOpKernel<double, XPUCtx>,
-                       ops::LoadCombineOpKernel<int, XPUCtx>,
-                       ops::LoadCombineOpKernel<int8_t, XPUCtx>,
-                       ops::LoadCombineOpKernel<int64_t, XPUCtx>);
+PD_REGISTER_STRUCT_KERNEL(load_combine,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::LoadCombineOpKernel,
+                          float,
+                          double,
+                          int,
+                          int8_t,
+                          int64_t) {}
diff --git a/paddle/fluid/operators/lod_reset_op.cc b/paddle/fluid/operators/lod_reset_op.cc
index aa5c6bc249cda..ae464e7b47161 100644
--- a/paddle/fluid/operators/lod_reset_op.cc
+++ b/paddle/fluid/operators/lod_reset_op.cc
@@ -259,13 +259,15 @@ PD_REGISTER_STRUCT_KERNEL(lod_reset,
                           int64_t) {}
 
 #ifdef PADDLE_WITH_XPU
-using XPUCtx = paddle::platform::XPUDeviceContext;
-REGISTER_OP_XPU_KERNEL(lod_reset,
-                       ops::LoDResetKernel<paddle::platform::float16, XPUCtx>,
-                       ops::LoDResetKernel<float, XPUCtx>,
-                       ops::LoDResetKernel<double, XPUCtx>,
-                       ops::LoDResetKernel<int, XPUCtx>,
-                       ops::LoDResetKernel<int64_t, XPUCtx>);
+PD_REGISTER_STRUCT_KERNEL(lod_reset,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::LoDResetKernel,
+                          plat::float16,
+                          float,
+                          double,
+                          int,
+                          int64_t) {}
 #endif
 
 PD_REGISTER_STRUCT_KERNEL(lod_reset_grad,
diff --git a/paddle/fluid/operators/log_loss_op_xpu.cc b/paddle/fluid/operators/log_loss_op_xpu.cc
index 87e6d42e98ad5..6c0c7f30d8e49 100644
--- a/paddle/fluid/operators/log_loss_op_xpu.cc
+++ b/paddle/fluid/operators/log_loss_op_xpu.cc
@@ -17,7 +17,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename DeviceContext, typename T, typename AttrType = T>
+template <typename T, typename DeviceContext, typename AttrType = T>
 class LogLossXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
@@ -37,7 +37,7 @@ class LogLossXPUKernel : public framework::OpKernel<T> {
     PADDLE_ENFORCE_XDNN_SUCCESS(r, "log_loss");
   }
 };
-template <typename DeviceContext, typename T, typename AttrType = T>
+template <typename T, typename DeviceContext, typename AttrType = T>
 class LogLossGradXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
@@ -67,10 +67,9 @@ class LogLossGradXPUKernel : public framework::OpKernel<T> {
 }  // namespace operators
 }  // namespace paddle
 namespace ops = paddle::operators;
-REGISTER_OP_XPU_KERNEL(
-    log_loss, ops::LogLossXPUKernel<paddle::platform::XPUDeviceContext, float>);
-REGISTER_OP_XPU_KERNEL(
-    log_loss_grad,
-    ops::LogLossGradXPUKernel<paddle::platform::XPUDeviceContext, float>);
 
+PD_REGISTER_STRUCT_KERNEL(
+    log_loss, XPU, ALL_LAYOUT, ops::LogLossXPUKernel, float) {}
+PD_REGISTER_STRUCT_KERNEL(
+    log_loss_grad, XPU, ALL_LAYOUT, ops::LogLossGradXPUKernel, float) {}
 #endif
diff --git a/paddle/fluid/operators/metrics/accuracy_op_xpu.cc b/paddle/fluid/operators/metrics/accuracy_op_xpu.cc
index 737228902b6e7..0ac30b3e87347 100644
--- a/paddle/fluid/operators/metrics/accuracy_op_xpu.cc
+++ b/paddle/fluid/operators/metrics/accuracy_op_xpu.cc
@@ -21,7 +21,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename DeviceContext, typename T>
+template <typename T, typename DeviceContext>
 class AccuracyXPUKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
@@ -73,8 +73,6 @@ class AccuracyXPUKernel : public framework::OpKernel<T> {
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OP_XPU_KERNEL(
-    accuracy,
-    ops::AccuracyXPUKernel<paddle::platform::XPUDeviceContext, float>);
-
+PD_REGISTER_STRUCT_KERNEL(
+    accuracy, XPU, ALL_LAYOUT, ops::AccuracyXPUKernel, float) {}
 #endif
diff --git a/paddle/fluid/operators/optimizers/lars_momentum_op_xpu.cc b/paddle/fluid/operators/optimizers/lars_momentum_op_xpu.cc
index 267e064b3065e..52b57252b0abe 100644
--- a/paddle/fluid/operators/optimizers/lars_momentum_op_xpu.cc
+++ b/paddle/fluid/operators/optimizers/lars_momentum_op_xpu.cc
@@ -20,7 +20,7 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-template <typename T>
+template <typename T, typename DeviceContext>
 class LarsMomentumOpXPUKernel : public framework::OpKernel<T> {
   using XPUType = typename XPUTypeTrait<T>::Type;
 
@@ -115,7 +115,11 @@ class LarsMomentumOpXPUKernel : public framework::OpKernel<T> {
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OP_XPU_KERNEL(lars_momentum,
-                       ops::LarsMomentumOpXPUKernel<paddle::platform::float16>,
-                       ops::LarsMomentumOpXPUKernel<float>);
+namespace plat = paddle::platform;
+PD_REGISTER_STRUCT_KERNEL(lars_momentum,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::LarsMomentumOpXPUKernel,
+                          float,
+                          plat::float16) {}
 #endif
diff --git a/paddle/fluid/operators/reduce_ops/logsumexp_op_xpu.cc b/paddle/fluid/operators/reduce_ops/logsumexp_op_xpu.cc
index e250b5585da06..b23fee1a012df 100644
--- a/paddle/fluid/operators/reduce_ops/logsumexp_op_xpu.cc
+++ b/paddle/fluid/operators/reduce_ops/logsumexp_op_xpu.cc
@@ -70,6 +70,10 @@ class XPULogsumexpKernel : public framework::OpKernel<T> {
 }  // namespace paddle
 
 namespace ops = paddle::operators;
+// This kernel can not be registered in phi, because op logsumexp should run
+// phi::LogsumexpKernel rather than XPULogsumexpKernel here. And if register
+// xpu logsumexp kernel in phi, op logsumexp will run XPULogsumexpKernel here
+// and raise error.
 REGISTER_OP_XPU_KERNEL(
     logsumexp,
     ops::XPULogsumexpKernel<paddle::platform::XPUDeviceContext, float>);

From 2e1ac529c67ba5a090230749683eb845bfb3afce Mon Sep 17 00:00:00 2001
From: houj04 <35131887+houj04@users.noreply.github.com>
Date: Thu, 27 Apr 2023 10:34:39 +0800
Subject: [PATCH 094/405] [XPU] remove scale_loss in parallel.py (#53337)

* [XPU] remove scale_loss in parallel.py

* [XPU] throw Unimplemented when using Reducer
---
 paddle/fluid/imperative/reducer.cc            |  4 ++-
 .../allocation/naive_best_fit_allocator.cc    |  4 +--
 .../fluid/operators/reader/buffered_reader.cc |  2 +-
 .../platform/device/xpu/xpu_resource_pool.cc  |  8 ++---
 paddle/phi/kernels/xpu/bmm_grad_kernel.cc     |  2 +-
 paddle/phi/kernels/xpu/bmm_kernel.cc          |  2 +-
 paddle/phi/kernels/xpu/xpu_api_wrapper.h      |  6 ++--
 python/paddle/distributed/parallel.py         | 17 +---------
 .../fluid/dygraph/tensor_patch_methods.py     | 33 +++++--------------
 9 files changed, 24 insertions(+), 54 deletions(-)

diff --git a/paddle/fluid/imperative/reducer.cc b/paddle/fluid/imperative/reducer.cc
index f90e1243d0f8d..33fbfc1d33746 100644
--- a/paddle/fluid/imperative/reducer.cc
+++ b/paddle/fluid/imperative/reducer.cc
@@ -61,7 +61,9 @@ void Group::DivNRanks(const platform::DeviceContext &context, int64_t nranks) {
     VLOG(4) << "after div 2" << *tensor;
   } else if (platform::is_xpu_place(tensor->place())) {
 #ifdef PADDLE_WITH_XPU_BKCL
-// TODO(liuyuhui) support xpu about div nranks in the future
+    PADDLE_THROW(
+        platform::errors::Unimplemented("DivNRanks is not supported on XPU / "
+                                        "XPU_BKCL, use EagerReducer instead."));
 #endif
   }
 }
diff --git a/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc b/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc
index 8c9eb889add6c..29ac488e9c52b 100644
--- a/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc
+++ b/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc
@@ -149,7 +149,7 @@ void *Alloc<platform::XPUPlace>(const platform::XPUPlace &place, size_t size) {
   VLOG(10) << "Allocate " << size << " bytes on " << platform::Place(place);
   void *p = nullptr;
 
-  platform::XPUDeviceGuard gurad(place.device);
+  platform::XPUDeviceGuard guard(place.device);
   int ret = xpu_malloc(reinterpret_cast<void **>(&p), size);
   if (ret != XPU_SUCCESS) {
     VLOG(10) << "xpu memory malloc(" << size << ") failed, try again";
@@ -182,7 +182,7 @@ void Free<platform::XPUPlace>(const platform::XPUPlace &place,
   VLOG(10) << "Free " << size << " bytes on " << platform::Place(place);
   VLOG(10) << "Free pointer=" << p << " on " << platform::Place(place);
 
-  platform::XPUDeviceGuard gurad(place.device);
+  platform::XPUDeviceGuard guard(place.device);
   xpu_free(p);
 #else
   PADDLE_THROW(
diff --git a/paddle/fluid/operators/reader/buffered_reader.cc b/paddle/fluid/operators/reader/buffered_reader.cc
index f0f54eafaa36b..b1e29ea69249d 100644
--- a/paddle/fluid/operators/reader/buffered_reader.cc
+++ b/paddle/fluid/operators/reader/buffered_reader.cc
@@ -268,7 +268,7 @@ void BufferedReader::ReadAsync(size_t i) {
         xpu_ptrs.emplace_back(xpu[i].mutable_data(place_, cpu[i].type()));
       }
 
-      platform::XPUDeviceGuard gurad(place_.device);
+      platform::XPUDeviceGuard guard(place_.device);
       int r = xpu_event_record(events_[i].get(), compute_stream_);
       PADDLE_ENFORCE_XDNN_SUCCESS(r, "xpu_event_record");
       r = xpu_stream_wait_event(stream_.get(), events_[i].get());
diff --git a/paddle/fluid/platform/device/xpu/xpu_resource_pool.cc b/paddle/fluid/platform/device/xpu/xpu_resource_pool.cc
index dccb8a9f295d6..5b6cbac603e5d 100644
--- a/paddle/fluid/platform/device/xpu/xpu_resource_pool.cc
+++ b/paddle/fluid/platform/device/xpu/xpu_resource_pool.cc
@@ -22,14 +22,14 @@ XpuStreamResourcePool::XpuStreamResourcePool() {
   pool_.reserve(dev_cnt);
   for (int dev_idx = 0; dev_idx < dev_cnt; ++dev_idx) {
     auto creator = [dev_idx] {
-      platform::XPUDeviceGuard gurad(dev_idx);
+      platform::XPUDeviceGuard guard(dev_idx);
       xpuStream stream;
       xpu_stream_create(&stream);
       return stream;
     };
 
     auto deleter = [dev_idx](xpuStream stream) {
-      platform::XPUDeviceGuard gurad(dev_idx);
+      platform::XPUDeviceGuard guard(dev_idx);
       xpu_stream_destroy(stream);
     };
 
@@ -63,14 +63,14 @@ XpuEventResourcePool::XpuEventResourcePool() {
   pool_.reserve(dev_cnt);
   for (int dev_idx = 0; dev_idx < dev_cnt; ++dev_idx) {
     auto creator = [dev_idx] {
-      platform::XPUDeviceGuard gurad(dev_idx);
+      platform::XPUDeviceGuard guard(dev_idx);
       xpuEventHandle event;
       xpu_event_create(&event);
       return event;
     };
 
     auto deleter = [dev_idx](xpuEventHandle event) {
-      platform::XPUDeviceGuard gurad(dev_idx);
+      platform::XPUDeviceGuard guard(dev_idx);
       xpu_event_destroy(event);
     };
 
diff --git a/paddle/phi/kernels/xpu/bmm_grad_kernel.cc b/paddle/phi/kernels/xpu/bmm_grad_kernel.cc
index 5f4a0d9a99d39..4c3b3dcd2c9b0 100644
--- a/paddle/phi/kernels/xpu/bmm_grad_kernel.cc
+++ b/paddle/phi/kernels/xpu/bmm_grad_kernel.cc
@@ -33,7 +33,7 @@ void MatMul(const Context& dev_ctx,
     MatMulXPUFunction<T, int32_t>(a, b, out, trans_a, trans_b, xpu_ctx);
   } else if (fccal_type == XPUFCCalcType::FC_FLOAT) {
     MatMulXPUFunction<T, float>(a, b, out, trans_a, trans_b, xpu_ctx);
-  } else if (fccal_type == XPUFCCalcType::FC_INT_WITH_LL) {
+  } else if (fccal_type == XPUFCCalcType::FC_INT32_WITH_LL) {
     MatMulXPUFunction<T, int_with_ll_t>(a, b, out, trans_a, trans_b, xpu_ctx);
   } else {
     MatMulXPUFunction<T, int16_t>(a, b, out, trans_a, trans_b, xpu_ctx);
diff --git a/paddle/phi/kernels/xpu/bmm_kernel.cc b/paddle/phi/kernels/xpu/bmm_kernel.cc
index b68a5fc3c0076..04afc3ef1b007 100644
--- a/paddle/phi/kernels/xpu/bmm_kernel.cc
+++ b/paddle/phi/kernels/xpu/bmm_kernel.cc
@@ -68,7 +68,7 @@ void BmmKernel(const Context& dev_ctx,
     MatMulXPUFunction<T, int32_t>(x, y, out, trans_x, trans_y, xpu_ctx);
   } else if (fccal_type == XPUFCCalcType::FC_FLOAT) {
     MatMulXPUFunction<T, float>(x, y, out, trans_x, trans_y, xpu_ctx);
-  } else if (fccal_type == XPUFCCalcType::FC_INT_WITH_LL) {
+  } else if (fccal_type == XPUFCCalcType::FC_INT32_WITH_LL) {
     MatMulXPUFunction<T, int_with_ll_t>(x, y, out, trans_x, trans_y, xpu_ctx);
   } else {
     MatMulXPUFunction<T, int16_t>(x, y, out, trans_x, trans_y, xpu_ctx);
diff --git a/paddle/phi/kernels/xpu/xpu_api_wrapper.h b/paddle/phi/kernels/xpu/xpu_api_wrapper.h
index 5bbe1163552ca..b75eaa1589323 100644
--- a/paddle/phi/kernels/xpu/xpu_api_wrapper.h
+++ b/paddle/phi/kernels/xpu/xpu_api_wrapper.h
@@ -30,7 +30,7 @@ enum XPUFCCalcType {
   FC_INT16 = 0,
   FC_INT32,
   FC_FLOAT,
-  FC_INT_WITH_LL,
+  FC_INT32_WITH_LL,
 };
 
 template <typename T>
@@ -42,8 +42,8 @@ XPUFCCalcType FCCalcType() {
     return XPUFCCalcType::FC_INT32;
   } else if (std::getenv("XPU_PADDLE_FC_LOCAL_INT16") != nullptr) {
     return XPUFCCalcType::FC_FLOAT;
-  } else if (std::getenv("XPU_PADDLE_FC_INT_WITH_LL") != nullptr) {
-    return XPUFCCalcType::FC_INT_WITH_LL;
+  } else if (std::getenv("XPU_PADDLE_FC_INT32_WITH_LL") != nullptr) {
+    return XPUFCCalcType::FC_INT32_WITH_LL;
   }
   return XPUFCCalcType::FC_INT16;
 }
diff --git a/python/paddle/distributed/parallel.py b/python/paddle/distributed/parallel.py
index 3fdf7cdcdd954..30a3e82eaf1b2 100644
--- a/python/paddle/distributed/parallel.py
+++ b/python/paddle/distributed/parallel.py
@@ -47,7 +47,7 @@
 # (TODO: GhostScreaming) It will be removed later.
 from paddle.framework import _set_expected_place
 from paddle.framework import base as imperative_base
-from paddle.framework import core, in_dygraph_mode, to_variable
+from paddle.framework import core, in_dygraph_mode
 from paddle.nn.layer import layers
 from paddle.utils import deprecated
 
@@ -117,21 +117,6 @@ def _split_tensors(coalesced_grads_and_grad_vars):
                 assert g_var.shape == g_shape
 
 
-def scale_loss(loss):
-    # TODO(liuyuhui) Currently only for xpu. Will be removed in the future.
-    if not paddle.distributed.ParallelEnv().world_size > 1:
-        return loss
-
-    loss_scale = to_variable(
-        np.array([paddle.distributed.ParallelEnv().world_size]).astype(
-            "float32"
-        )
-    )
-    loss_scale.stop_gradient = True
-    scaled_loss = loss / loss_scale
-    return scaled_loss
-
-
 @imperative_base.no_grad
 @framework.dygraph_only
 def build_groups(vars, group_size):
diff --git a/python/paddle/fluid/dygraph/tensor_patch_methods.py b/python/paddle/fluid/dygraph/tensor_patch_methods.py
index 882a333b5ebf7..90d896f7c7dc1 100644
--- a/python/paddle/fluid/dygraph/tensor_patch_methods.py
+++ b/python/paddle/fluid/dygraph/tensor_patch_methods.py
@@ -275,8 +275,6 @@ def backward(self, grad_tensor=None, retain_graph=False):
                 # 4: [5000.]
 
         """
-        from paddle.distributed.parallel import scale_loss
-
         if framework._non_static_mode():
             if in_profiler_mode():
                 record_event = profiler.RecordEvent(
@@ -306,30 +304,15 @@ def backward(self, grad_tensor=None, retain_graph=False):
             if _grad_scalar:
                 # When using amp with Fleet DistributedStrategy, we do loss scaling implicitly.
                 self = _grad_scalar.scale(self)
-            if paddle.is_compiled_with_xpu():
-                # TODO(liuyuhui): Currently only for xpu. Will be removed in the future.
-                scaled_loss = scale_loss(self)
-                if framework.global_var._in_eager_mode_:
-                    core.eager.run_backward(
-                        [scaled_loss], grad_tensor, retain_graph
-                    )
-                else:
-                    core.dygraph_run_backward(
-                        [scaled_loss],
-                        [grad_tensor],
-                        retain_graph,
-                        framework._dygraph_tracer(),
-                    )
+            if framework.global_var._in_eager_mode_:
+                core.eager.run_backward([self], grad_tensor, retain_graph)
             else:
-                if framework.global_var._in_eager_mode_:
-                    core.eager.run_backward([self], grad_tensor, retain_graph)
-                else:
-                    core.dygraph_run_backward(
-                        [self],
-                        [grad_tensor],
-                        retain_graph,
-                        framework._dygraph_tracer(),
-                    )
+                core.dygraph_run_backward(
+                    [self],
+                    [grad_tensor],
+                    retain_graph,
+                    framework._dygraph_tracer(),
+                )
             if in_profiler_mode():
                 record_event.end()
         else:

From 0fac328106f3d7062eceafc48bcc1e8ae92c464b Mon Sep 17 00:00:00 2001
From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com>
Date: Thu, 27 Apr 2023 10:45:15 +0800
Subject: [PATCH 095/405] remove some [-Wunused-parameter] warning (#53365)

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop
---
 paddle/fluid/distributed/collective/process_group.h    | 2 +-
 paddle/fluid/distributed/ps/service/brpc_ps_client.h   | 4 ++--
 paddle/fluid/eager/to_static/run_program_op_node.h     | 8 ++++----
 paddle/fluid/framework/data_set.h                      | 2 +-
 paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc          | 2 +-
 paddle/phi/kernels/empty_kernel.cc                     | 2 +-
 paddle/phi/kernels/flatten_kernel.cc                   | 6 +++---
 paddle/phi/kernels/funcs/activation_functor.h          | 4 ++--
 paddle/phi/kernels/funcs/compound_functors.h           | 8 +++++---
 paddle/phi/kernels/funcs/detail/activation_functions.h | 4 ++--
 paddle/phi/kernels/funcs/gather.h                      | 6 +++---
 paddle/phi/kernels/funcs/jit/helper.h                  | 2 +-
 paddle/phi/kernels/funcs/reduce_functor.h              | 2 +-
 paddle/phi/kernels/funcs/scatter.h                     | 2 +-
 paddle/phi/kernels/funcs/strided_memcpy.h              | 6 +++---
 paddle/phi/kernels/squeeze_grad_kernel.cc              | 2 +-
 paddle/phi/kernels/squeeze_kernel.cc                   | 4 ++--
 paddle/phi/kernels/unsqueeze_kernel.cc                 | 2 +-
 18 files changed, 35 insertions(+), 33 deletions(-)

diff --git a/paddle/fluid/distributed/collective/process_group.h b/paddle/fluid/distributed/collective/process_group.h
index 447fc5d1b3c7b..eff17c9d4e061 100644
--- a/paddle/fluid/distributed/collective/process_group.h
+++ b/paddle/fluid/distributed/collective/process_group.h
@@ -486,7 +486,7 @@ class ProcessGroup {
   virtual std::shared_ptr<ProcessGroup::Task> Reduce(
       std::vector<phi::DenseTensor>&,  // NOLINT
       std::vector<phi::DenseTensor>&,  // NOLINT
-      const ReduceOptions& opts) {
+      const ReduceOptions& opts UNUSED) {
     PADDLE_THROW(phi::errors::InvalidArgument(
         "ProcessGroup%s does not support reduce", GetBackendName()));
   }
diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_client.h b/paddle/fluid/distributed/ps/service/brpc_ps_client.h
index bbaecc498a80a..d902824bfd60c 100755
--- a/paddle/fluid/distributed/ps/service/brpc_ps_client.h
+++ b/paddle/fluid/distributed/ps/service/brpc_ps_client.h
@@ -30,7 +30,7 @@
 #include "paddle/fluid/framework/lod_tensor.h"
 #include "paddle/fluid/framework/scope.h"
 #include "paddle/fluid/framework/tensor_util.h"
-
+#include "paddle/phi/core/macros.h"
 namespace brpc {
 class Channel;
 class Controller;
@@ -63,7 +63,7 @@ class DownpourPsClientService : public PsService {
                        PsResponseMessage *response,
                        ::google::protobuf::Closure *done);
 
-  virtual void FLService(::google::protobuf::RpcController *controller,
+  virtual void FLService(::google::protobuf::RpcController *controller UNUSED,
                          const CoordinatorReqMessage *request,
                          CoordinatorResMessage *response,
                          ::google::protobuf::Closure *done) {
diff --git a/paddle/fluid/eager/to_static/run_program_op_node.h b/paddle/fluid/eager/to_static/run_program_op_node.h
index 6b3c73dbdb39b..57defbaee4dca 100644
--- a/paddle/fluid/eager/to_static/run_program_op_node.h
+++ b/paddle/fluid/eager/to_static/run_program_op_node.h
@@ -457,8 +457,8 @@ inline void RunProgramAPI(
 }
 
 inline void RunProgramGradAPI(
-    const std::vector<paddle::Tensor> &x,
-    const std::vector<paddle::Tensor> &params,
+    const std::vector<paddle::Tensor> &x UNUSED,
+    const std::vector<paddle::Tensor> &params UNUSED,
     const std::vector<paddle::Tensor> &out_grad,
     const std::vector<paddle::framework::Scope *> &step_scope,  // NOLINT
     const paddle::framework::AttributeMap &attrs,
@@ -610,8 +610,8 @@ class GradNodeRunProgram : public egr::GradNodeBase {
                                egr::kSlotSmallVectorSize>
   operator()(paddle::small_vector<std::vector<paddle::Tensor>,
                                   egr::kSlotSmallVectorSize> &grads,  // NOLINT
-             bool create_graph,
-             bool is_new_grad) override {
+             bool create_graph UNUSED,
+             bool is_new_grad UNUSED) override {
     VLOG(3) << "Running Eager Backward Node: GradNodeRunProgram";
     paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
         hooked_grads = GradNodeRunProgram::ApplyGradientHooks(grads);
diff --git a/paddle/fluid/framework/data_set.h b/paddle/fluid/framework/data_set.h
index 1bc60993e36a0..9af5fbfc6b4a5 100644
--- a/paddle/fluid/framework/data_set.h
+++ b/paddle/fluid/framework/data_set.h
@@ -58,7 +58,7 @@ class Dataset {
                          const uint16_t start_sample_layer UNUSED,
                          const bool with_hierachy UNUSED,
                          const uint16_t seed_ UNUSED,
-                         const uint16_t sample_slot) {}
+                         const uint16_t sample_slot UNUSED) {}
   // set file list
   virtual void SetFileList(const std::vector<std::string>& filelist) = 0;
   // set readers' num
diff --git a/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
index 2ba103ce0fcae..a5274c5f7ae7c 100644
--- a/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
@@ -44,7 +44,7 @@ class FCMKLDNNHandler
                   const phi::DenseTensor* x,
                   const phi::DenseTensor* weights,
                   const phi::DenseTensor* bias,
-                  phi::DenseTensor* out,
+                  phi::DenseTensor* out UNUSED,
                   const int in_num_col_dims,
                   dnnl::engine onednn_engine,
                   platform::Place cpu_place)
diff --git a/paddle/phi/kernels/empty_kernel.cc b/paddle/phi/kernels/empty_kernel.cc
index 0877a8e24468c..8df5e9a543eb2 100644
--- a/paddle/phi/kernels/empty_kernel.cc
+++ b/paddle/phi/kernels/empty_kernel.cc
@@ -30,7 +30,7 @@ void EmptyKernel(const Context& dev_ctx,
 
 template <typename T, typename Context>
 void EmptyLikeKernel(const Context& dev_ctx,
-                     const DenseTensor& x,
+                     const DenseTensor& x UNUSED,
                      DataType dtype UNUSED,
                      DenseTensor* out) {
   dev_ctx.template Alloc<T>(out);
diff --git a/paddle/phi/kernels/flatten_kernel.cc b/paddle/phi/kernels/flatten_kernel.cc
index 939e270613650..67d56f1c46aa3 100644
--- a/paddle/phi/kernels/flatten_kernel.cc
+++ b/paddle/phi/kernels/flatten_kernel.cc
@@ -25,8 +25,8 @@ namespace phi {
 template <typename T, typename Context>
 void FlattenInferKernel(const Context& dev_ctx,
                         const DenseTensor& x,
-                        int start_axis,
-                        int stop_axis,
+                        int start_axis UNUSED,
+                        int stop_axis UNUSED,
                         DenseTensor* out) {
   dev_ctx.Alloc(out, x.dtype());
   auto out_dims = out->dims();
@@ -43,7 +43,7 @@ void FlattenKernel(const Context& dev_ctx,
                    int start_axis,
                    int stop_axis,
                    DenseTensor* out,
-                   DenseTensor* xshape) {
+                   DenseTensor* xshape UNUSED) {
   FlattenInferKernel<T, Context>(dev_ctx, x, start_axis, stop_axis, out);
 }
 
diff --git a/paddle/phi/kernels/funcs/activation_functor.h b/paddle/phi/kernels/funcs/activation_functor.h
index 6a3554318e5e6..aefa653928633 100644
--- a/paddle/phi/kernels/funcs/activation_functor.h
+++ b/paddle/phi/kernels/funcs/activation_functor.h
@@ -1198,7 +1198,7 @@ struct TanhGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x UNUSED, Out out, dOut dout, dX dx) const {
     dx.device(d) = dout * (static_cast<T>(1) - out * out);
   }
 
@@ -1794,7 +1794,7 @@ struct SigmoidGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x UNUSED, Out out, dOut dout, dX dx) const {
     dx.device(d) = dout * out * (static_cast<T>(1) - out);
   }
 
diff --git a/paddle/phi/kernels/funcs/compound_functors.h b/paddle/phi/kernels/funcs/compound_functors.h
index 0fd3fd0e932fc..020c6dcd87f30 100644
--- a/paddle/phi/kernels/funcs/compound_functors.h
+++ b/paddle/phi/kernels/funcs/compound_functors.h
@@ -96,12 +96,12 @@ struct BinaryCompoundGradDyFunctor {
         unary_fun_(unary_fun),
         d_unary_fun_(d_unary_fun) {}
 
-  inline HOSTDEVICE T Recompute(T x, T y, T out, T dout) {
+  inline HOSTDEVICE T Recompute(T x, T y, T out UNUSED, T dout) {
     return dout * d_binary_fun_.Dy(x, unary_fun_(y)) * d_unary_fun_.UseX(y);
   }
 
   inline HOSTDEVICE T
-  UseIntermediateOut(T x, T y, T intermediate_out, T out, T dout) {
+  UseIntermediateOut(T x, T y, T intermediate_out, T out UNUSED, T dout) {
     if (InPlace) {
       return dout * d_binary_fun_.Dy(x, intermediate_out) *
              d_unary_fun_.UseOut(intermediate_out);
@@ -111,7 +111,9 @@ struct BinaryCompoundGradDyFunctor {
     }
   }
 
-  inline HOSTDEVICE T GetIntermediateOut(T x, T y) { return unary_fun_(y); }
+  inline HOSTDEVICE T GetIntermediateOut(T x UNUSED, T y) {
+    return unary_fun_(y);
+  }
 
  private:
   DBinaryFun d_binary_fun_;
diff --git a/paddle/phi/kernels/funcs/detail/activation_functions.h b/paddle/phi/kernels/funcs/detail/activation_functions.h
index 26be2a83280c3..f1352df226094 100644
--- a/paddle/phi/kernels/funcs/detail/activation_functions.h
+++ b/paddle/phi/kernels/funcs/detail/activation_functions.h
@@ -20,7 +20,7 @@ limitations under the License. */
 
 #include "paddle/phi/backends/cpu/cpu_info.h"
 #include "paddle/phi/core/hostdevice.h"
-
+#include "paddle/phi/core/macros.h"
 namespace phi {
 namespace funcs {
 namespace detail {
@@ -104,7 +104,7 @@ DEVICE T TanhV2(const T a) {
 namespace backward {
 
 template <typename T>
-DEVICE T Identity(const T a, const T b) {
+DEVICE T Identity(const T a, const T b UNUSED) {
   return a;
 }
 
diff --git a/paddle/phi/kernels/funcs/gather.h b/paddle/phi/kernels/funcs/gather.h
index f1ab1a16f1224..50f7f4fa0322c 100644
--- a/paddle/phi/kernels/funcs/gather.h
+++ b/paddle/phi/kernels/funcs/gather.h
@@ -21,8 +21,8 @@ limitations under the License. */
 #include "paddle/phi/common/place.h"
 #include "paddle/phi/core/ddim.h"
 #include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/macros.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
-
 namespace phi {
 namespace funcs {
 
@@ -34,7 +34,7 @@ namespace funcs {
  * return: output tensor
  */
 template <typename T, typename IndexT = int>
-void CPUGather(const phi::CPUContext& ctx,
+void CPUGather(const phi::CPUContext& ctx UNUSED,
                const DenseTensor& src,
                const DenseTensor& index,
                DenseTensor* output) {
@@ -95,7 +95,7 @@ void CPUGather(const phi::CPUContext& ctx,
 }
 
 template <typename T, typename IndexT = int>
-void CPUGatherNd(const phi::CPUContext& ctx,
+void CPUGatherNd(const phi::CPUContext& ctx UNUSED,
                  const DenseTensor& input,
                  const DenseTensor& index,
                  DenseTensor* output) {
diff --git a/paddle/phi/kernels/funcs/jit/helper.h b/paddle/phi/kernels/funcs/jit/helper.h
index ff233710a5cf3..7e3394dffd4a2 100644
--- a/paddle/phi/kernels/funcs/jit/helper.h
+++ b/paddle/phi/kernels/funcs/jit/helper.h
@@ -74,7 +74,7 @@ inline typename std::enable_if<
     !std::is_same<typename KernelTuple::data_type, float>::value ||
         !std::is_same<PlaceType, phi::CPUPlace>::value,
     const Kernel*>::type
-GetJitCode(const typename KernelTuple::attr_type& attr) {
+GetJitCode(const typename KernelTuple::attr_type& attr UNUSED) {
   return nullptr;
 }
 
diff --git a/paddle/phi/kernels/funcs/reduce_functor.h b/paddle/phi/kernels/funcs/reduce_functor.h
index f1d3772cc25a5..596be9bc33058 100644
--- a/paddle/phi/kernels/funcs/reduce_functor.h
+++ b/paddle/phi/kernels/funcs/reduce_functor.h
@@ -153,7 +153,7 @@ struct ProdGradFunctor {
                   DX* dx,
                   DY* dy,
                   const Dim& dim,
-                  int size) {
+                  int size UNUSED) {
     dx->device(place) = dy->broadcast(dim) * y->broadcast(dim) * x->inverse();
   }
 };
diff --git a/paddle/phi/kernels/funcs/scatter.h b/paddle/phi/kernels/funcs/scatter.h
index 7c23a35072cb8..d430588541837 100644
--- a/paddle/phi/kernels/funcs/scatter.h
+++ b/paddle/phi/kernels/funcs/scatter.h
@@ -47,7 +47,7 @@ elementwise_inner_add(const phi::CPUContext& ctx,
 
 template <typename T, typename IndexT = int>
 typename std::enable_if<!std::is_floating_point<T>::value>::type
-elementwise_inner_add(const phi::CPUContext& ctx,
+elementwise_inner_add(const phi::CPUContext& ctx UNUSED,
                       const T* src_pointer,
                       T* dst_pointer,
                       size_t src_index,
diff --git a/paddle/phi/kernels/funcs/strided_memcpy.h b/paddle/phi/kernels/funcs/strided_memcpy.h
index d80cf7b2e2421..19a542bd2b5e9 100644
--- a/paddle/phi/kernels/funcs/strided_memcpy.h
+++ b/paddle/phi/kernels/funcs/strided_memcpy.h
@@ -12,9 +12,9 @@ limitations under the License. */
 #pragma once
 #include <vector>
 
-#include "paddle/phi/kernels/funcs/detail/strided_memcpy.h"
-
 #include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/macros.h"
+#include "paddle/phi/kernels/funcs/detail/strided_memcpy.h"
 
 namespace phi {
 class CPUContext;
@@ -65,7 +65,7 @@ inline void CopyWithContext(const Context& ctx,
 }
 
 template <>
-inline void CopyWithContext<phi::CPUContext>(const phi::CPUContext& ctx,
+inline void CopyWithContext<phi::CPUContext>(const phi::CPUContext& ctx UNUSED,
                                              const Place& dst_place,
                                              void* dst,
                                              const Place& src_place,
diff --git a/paddle/phi/kernels/squeeze_grad_kernel.cc b/paddle/phi/kernels/squeeze_grad_kernel.cc
index 0fe2f79073430..473acf9d7a1d1 100644
--- a/paddle/phi/kernels/squeeze_grad_kernel.cc
+++ b/paddle/phi/kernels/squeeze_grad_kernel.cc
@@ -23,7 +23,7 @@ template <typename T, typename Context>
 void SqueezeGradKernel(const Context& dev_ctx,
                        const DenseTensor& xshape,
                        const DenseTensor& dout,
-                       const IntArray& axes,
+                       const IntArray& axes UNUSED,
                        DenseTensor* dx) {
   auto xshape_dims = xshape.dims();
   auto x_dims = phi::slice_ddim(xshape_dims, 1, xshape_dims.size());
diff --git a/paddle/phi/kernels/squeeze_kernel.cc b/paddle/phi/kernels/squeeze_kernel.cc
index a0b72381601d6..d495b040921b5 100644
--- a/paddle/phi/kernels/squeeze_kernel.cc
+++ b/paddle/phi/kernels/squeeze_kernel.cc
@@ -23,7 +23,7 @@ namespace phi {
 template <typename T, typename Context>
 void SqueezeInferKernel(const Context& dev_ctx,
                         const DenseTensor& x,
-                        const IntArray& axes,
+                        const IntArray& axes UNUSED,
                         DenseTensor* out) {
   auto out_dims = out->dims();
   dev_ctx.template Alloc<T>(out);
@@ -39,7 +39,7 @@ void SqueezeKernel(const Context& dev_ctx,
                    const DenseTensor& x,
                    const IntArray& axes,
                    DenseTensor* out,
-                   DenseTensor* xshape) {
+                   DenseTensor* xshape UNUSED) {
   SqueezeInferKernel<T, Context>(dev_ctx, x, axes, out);
 }
 
diff --git a/paddle/phi/kernels/unsqueeze_kernel.cc b/paddle/phi/kernels/unsqueeze_kernel.cc
index 4354b09c753b1..c08c31da4ef0c 100644
--- a/paddle/phi/kernels/unsqueeze_kernel.cc
+++ b/paddle/phi/kernels/unsqueeze_kernel.cc
@@ -44,7 +44,7 @@ void UnsqueezeKernel(const Context& dev_ctx,
                      const DenseTensor& x,
                      const IntArray& axes,
                      DenseTensor* out,
-                     DenseTensor* xshape) {
+                     DenseTensor* xshape UNUSED) {
   UnsqueezeInferKernel<T, Context>(dev_ctx, x, axes, out);
 }
 }  // namespace phi

From c50f5fa4e32b22acd3bf3e89ca0968ee65d0c5ea Mon Sep 17 00:00:00 2001
From: engineer1109 <jialiang.wang@xdxct.com>
Date: Thu, 27 Apr 2023 10:50:24 +0800
Subject: [PATCH 096/405] fix softmax assert error (#53360)

---
 paddle/phi/kernels/funcs/softmax_impl.h | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/paddle/phi/kernels/funcs/softmax_impl.h b/paddle/phi/kernels/funcs/softmax_impl.h
index 330ac331b6b8e..8f6b0fdd32c4b 100644
--- a/paddle/phi/kernels/funcs/softmax_impl.h
+++ b/paddle/phi/kernels/funcs/softmax_impl.h
@@ -82,12 +82,12 @@ class SoftmaxEigen {
       // axis != -1, class dimension split into (axis, remain), max and sum
       // should be calculated along axis dimension
       softmax.device(*context.eigen_device()) =
-          (logits.reshape(batch_axis_remain) - logits.reshape(batch_axis_remain)
-                                                   .maximum(along_axis)
-                                                   .eval()
-                                                   .reshape(batch_one_remain)
-                                                   .broadcast(one_axis_one)
-                                                   .reshape(batch_classes))
+          (logits.reshape(batch_classes) - logits.reshape(batch_axis_remain)
+                                               .maximum(along_axis)
+                                               .eval()
+                                               .reshape(batch_one_remain)
+                                               .broadcast(one_axis_one)
+                                               .reshape(batch_classes))
               .unaryExpr(ValueClip<T>());
     }
 

From 2c12abd7fe3f3329b920afbb222df7fad787f5b6 Mon Sep 17 00:00:00 2001
From: Wilber <jiweibo@baidu.com>
Date: Thu, 27 Apr 2023 10:56:43 +0800
Subject: [PATCH 097/405] revert pr
 https://github.com/PaddlePaddle/Paddle/pull/46779 (#53373)

---
 paddle/fluid/inference/tensorrt/op_teller.cc | 10 ----------
 test/ir/inference/test_trt_convert_gather.py |  2 +-
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index 78e300a8d730d..bb0fbdf6ca848 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -542,16 +542,6 @@ struct SimpleOpTypeSetTeller : public Teller {
                      "the pass.";
           return false;
         }
-
-        auto index_var_name = desc.Input("Index")[0];
-        auto* index_var_desc = block->FindVar(index_var_name);
-
-        // The index input must be int32 datatype.
-        if (index_var_desc->GetDataType() !=
-            paddle::framework::proto::VarType_Type::VarType_Type_INT32) {
-          VLOG(3) << "gather op Index input data type must be int32";
-          return false;
-        }
 #if !IS_TRT_VERSION_GE(7000)
         auto* x_var_desc = block->FindVar(desc.Input("X")[0]);
         const auto x_shape = x_var_desc->GetShape();
diff --git a/test/ir/inference/test_trt_convert_gather.py b/test/ir/inference/test_trt_convert_gather.py
index 3c25dd6eff1c9..69a2624b77e09 100644
--- a/test/ir/inference/test_trt_convert_gather.py
+++ b/test/ir/inference/test_trt_convert_gather.py
@@ -182,7 +182,7 @@ def generate_trt_nodes_num(dynamic_shape):
             if self.input_num == 3:
                 return 0, 5
             else:
-                if dynamic_shape and self.index_type_int32:
+                if dynamic_shape:
                     return 1, 3
                 else:
                     return 0, 4

From 1bd468e24dae9ae1b2793253232bc7a78ec840ef Mon Sep 17 00:00:00 2001
From: JYChen <zoooo0820@qq.com>
Date: Thu, 27 Apr 2023 11:00:26 +0800
Subject: [PATCH 098/405] Hack__getitem__ from 0-d to 1-d with FLAGS_set_to_1d
 (#53358)

---
 paddle/fluid/pybind/eager_method.cc    | 16 +++++++++++++---
 paddle/fluid/pybind/imperative.cc      | 13 +++++++++++++
 paddle/phi/kernels/funcs/slice_utils.h | 23 +++++++++++++++++++++--
 python/paddle/fft.py                   |  4 ++--
 python/paddle/fluid/variable_index.py  |  7 +++++++
 5 files changed, 56 insertions(+), 7 deletions(-)

diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc
index 1da7bd774531b..28a86b8db8a3c 100644
--- a/paddle/fluid/pybind/eager_method.cc
+++ b/paddle/fluid/pybind/eager_method.cc
@@ -136,17 +136,18 @@ static PyObject* tensor_method_numpy(TensorObject* self,
       }
     }
     if (set_to_1d) {
-      // 0D Tensor hack process to 1D numpy, will remove in future
+      // 0D Tensor hack process to 1D numpy, will remove in release 2.6
       VLOG(0)
           << "Warning:: 0D Tensor cannot be used as 'Tensor.numpy()[0]' . In "
              "order to avoid this problem, "
              "0D Tensor will be changed to 1D numpy currently, but it's not "
              "correct and will be "
-             "removed in future. For Tensor contain only one element, Please "
+             "removed in release 2.6. For Tensor contain only one element, "
+             "Please "
              "modify "
              " 'Tensor.numpy()[0]' to 'float(Tensor)' as soon as "
              "possible, "
-             "otherwise 'Tensor.numpy()[0]' will raise error in future.";
+             "otherwise 'Tensor.numpy()[0]' will raise error in release 2.6.";
       py_rank = 1;
       py_dims[0] = 1;
       py_strides[0] = sizeof_dtype * numel;
@@ -923,7 +924,16 @@ static PyObject* tensor__getitem_index_not_tensor(TensorObject* self,
     }
   }
 
+  bool set_to_1d = FLAGS_set_to_1d;
   if (!none_axes.empty()) {
+    if (set_to_1d) {
+      // NOTE(zoooo0820): When all axes are decreased, the output will be 1-D
+      // with FLAGS_set_to_1d=True. In this case, one `None` should be pop out,
+      // otherwise the output shape will be not correct.
+      if (static_cast<int>(decrease_axis.size()) == tensor->dims().size()) {
+        none_axes.pop_back();
+      }
+    }
     if (!none_axes.empty()) {
       paddle::Tensor new_out;
       {
diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc
index 39a28c7487c30..372fae12ec3bc 100644
--- a/paddle/fluid/pybind/imperative.cc
+++ b/paddle/fluid/pybind/imperative.cc
@@ -63,6 +63,7 @@ limitations under the License. */
 #include "paddle/phi/core/compat/arg_map_context.h"
 #include "paddle/phi/core/type_defs.h"
 
+PHI_DECLARE_bool(set_to_1d);
 namespace paddle {
 namespace pybind {
 
@@ -1064,7 +1065,19 @@ void BindImperative(py::module *m_ptr) {
                }
                tracer->TraceOp(op_type, ins, outs, std::move(attrs));
              }
+
+             bool set_to_1d = FLAGS_set_to_1d;
              if (!none_axes.empty()) {
+               if (set_to_1d) {
+                 // NOTE(zoooo0820): When all axes are decreased, the output
+                 // will be 1-D with FLAGS_set_to_1d=True. In this case, one
+                 // `None` should be pop out, otherwise the output shape will be
+                 // not correct.
+                 if (static_cast<int>(decrease_axis.size()) ==
+                     tensor->dims().size()) {
+                   none_axes.pop_back();
+                 }
+               }
                if (!none_axes.empty()) {
                  // Deal with cases that decrease_axes is not empty
                  // For example:
diff --git a/paddle/phi/kernels/funcs/slice_utils.h b/paddle/phi/kernels/funcs/slice_utils.h
index 9bbb7681dd888..81a35b6774ae2 100644
--- a/paddle/phi/kernels/funcs/slice_utils.h
+++ b/paddle/phi/kernels/funcs/slice_utils.h
@@ -13,10 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #pragma once
+#include <glog/logging.h>
 #include <paddle/phi/core/ddim.h>
-
 #include <string>
 #include <vector>
+#include "paddle/phi/core/flags.h"
+
+PHI_DECLARE_bool(set_to_1d);
 
 namespace phi {
 
@@ -202,7 +205,23 @@ inline DDim GetDecreasedDims(const DDim slice_dims,
         new_shape.push_back(decreased_dims[i]);
       }
     }
-
+    if (FLAGS_set_to_1d && new_shape.size() == 0) {
+      // NOTE(zoooo0820): Hack procssing to 1-D, when axes decrease to 0-D in
+      // slice. This will remove in release 2.6.
+      VLOG(0)
+          << "Warning:: In Tensor '__getitem__', if the number of scalar "
+             "elements "
+             "in the index is equal to the rank of the Tensor, the output "
+             "should "
+             "be 0-D. In order to be consistent with the behavior of previous "
+             "versions, it will be processed to 1-D. But it is not correct and "
+             "will be "
+             "removed in release 2.6. "
+             "If 1-D is still wanted, please modify the index element from "
+             "scalar to slice "
+             "(e.g. 'x[i]' => 'x[i:i+1]'). ";
+      new_shape.push_back(1);
+    }
     decreased_dims = phi::make_ddim(new_shape);
   }
   return decreased_dims;
diff --git a/python/paddle/fft.py b/python/paddle/fft.py
index 48c20f7fdafaf..438c65ae2f044 100644
--- a/python/paddle/fft.py
+++ b/python/paddle/fft.py
@@ -1371,7 +1371,7 @@ def fftshift(x, axes=None, name=None):
     elif isinstance(axes, int):
         shifts = shape[axes] // 2
     else:
-        shifts = paddle.stack([shape[ax] // 2 for ax in axes])
+        shifts = paddle.concat([shape[ax : ax + 1] // 2 for ax in axes])
     return paddle.roll(x, shifts, axes, name=name)
 
 
@@ -1416,7 +1416,7 @@ def ifftshift(x, axes=None, name=None):
     elif isinstance(axes, int):
         shifts = -shape[axes] // 2
     else:
-        shifts = paddle.stack([-shape[ax] // 2 for ax in axes])
+        shifts = paddle.concat([-shape[ax : ax + 1] // 2 for ax in axes])
     return paddle.roll(x, shifts, axes, name=name)
 
 
diff --git a/python/paddle/fluid/variable_index.py b/python/paddle/fluid/variable_index.py
index fe7833f048f04..86ba9d5d58dcd 100644
--- a/python/paddle/fluid/variable_index.py
+++ b/python/paddle/fluid/variable_index.py
@@ -574,6 +574,13 @@ def _getitem_impl_(var, item):
 
         out = reverse(out, axis=reverse_axes)
 
+    # NOTE(zoooo0820): When all axes are decreased, the output will be 1-D
+    # with FLAGS_set_to_1d=True. In this case, one `None` should be pop out,
+    # otherwise the output shape will be not correct.
+    set_to_1d = paddle.get_flags('FLAGS_set_to_1d')['FLAGS_set_to_1d']
+    if set_to_1d and len(decrease_axes) == len(var.shape):
+        none_axes = none_axes[1:]
+
     if len(none_axes) > 0:
         # Deal with cases that decrease_axes is not empty
         # For example:

From 35af5818afe85daf16c02fc4e79749236cce0fb8 Mon Sep 17 00:00:00 2001
From: wanghuancoder <wanghuan29@baidu.com>
Date: Thu, 27 Apr 2023 11:01:52 +0800
Subject: [PATCH 099/405] refine SynchronizeAllDevice (#53370)

---
 paddle/fluid/platform/profiler_helper.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/paddle/fluid/platform/profiler_helper.h b/paddle/fluid/platform/profiler_helper.h
index 0f058ad0f8df8..1d34d5fd27b3e 100644
--- a/paddle/fluid/platform/profiler_helper.h
+++ b/paddle/fluid/platform/profiler_helper.h
@@ -95,28 +95,34 @@ std::vector<std::vector<MemEvent>> GetMemEvents() {
 
 void SynchronizeAllDevice() {
 #ifdef PADDLE_WITH_CUDA
+  int pre_device_id = GetCurrentDeviceId();
   int count = GetGPUDeviceCount();
   for (int i = 0; i < count; i++) {
     SetDeviceId(i);
     PADDLE_ENFORCE_GPU_SUCCESS(cudaDeviceSynchronize());
   }
+  SetDeviceId(pre_device_id);
 #endif
 #ifdef PADDLE_WITH_HIP
+  int pre_device_id = GetCurrentDeviceId();
   int count = GetGPUDeviceCount();
   for (int i = 0; i < count; i++) {
     SetDeviceId(i);
     PADDLE_ENFORCE_GPU_SUCCESS(hipDeviceSynchronize());
   }
+  SetDeviceId(pre_device_id);
 #endif
 #ifdef PADDLE_WITH_CUSTOM_DEVICE
   auto dev_types = phi::DeviceManager::GetAllCustomDeviceTypes();
   for (const auto &dev_type : dev_types) {
+    int pre_device_id = phi::DeviceManager::GetDevice(dev_type);
     auto dev_cnt = phi::DeviceManager::GetDeviceCount(dev_type);
     for (size_t i = 0; i < dev_cnt; i++) {
       auto place = paddle::platform::CustomPlace(dev_type, i);
       phi::DeviceManager::SetDevice(place);
       phi::DeviceManager::SynchronizeDevice(place);
     }
+    phi::DeviceManager::SetDevice(dev_type, pre_device_id);
   }
 #endif
 }

From c0ee14f6767cda4f38566c2f23eb364e3354c935 Mon Sep 17 00:00:00 2001
From: hua-zi <83271073+hua-zi@users.noreply.github.com>
Date: Thu, 27 Apr 2023 11:12:24 +0800
Subject: [PATCH 100/405] updata Adamw.py (#52984)

* updata Adamw.py

out.backward()  -> loss.backward()

* Update adamw.py
---
 python/paddle/optimizer/adamw.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/paddle/optimizer/adamw.py b/python/paddle/optimizer/adamw.py
index a525ac194a12c..f8e00eabecf5e 100644
--- a/python/paddle/optimizer/adamw.py
+++ b/python/paddle/optimizer/adamw.py
@@ -90,7 +90,7 @@ class AdamW(Optimizer):
         name (str, optional): Normally there is no need for user to set this property.
             For more information, please refer to :ref:`api_guide_Name`.
             The default value is None.
-    **Notes**:
+    Notes:
         **Currently, AdamW doesn't support sparse parameter optimization.**
 
     Examples:
@@ -111,7 +111,7 @@ class AdamW(Optimizer):
                     beta1=beta1,
                     beta2=beta2,
                     weight_decay=0.01)
-            out.backward()
+            loss.backward()
             opt.step()
             opt.clear_grad()
 
@@ -135,7 +135,7 @@ class AdamW(Optimizer):
                 }],
                 weight_decay=0.01,
                 beta1=0.9)
-            out.backward()
+            loss.backward()
             opt.step()
             opt.clear_grad()
 

From 6768c6ecd8c30f3dd3cdf7e8628693ed9bb2c636 Mon Sep 17 00:00:00 2001
From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com>
Date: Thu, 27 Apr 2023 11:29:02 +0800
Subject: [PATCH 101/405] =?UTF-8?q?=E3=80=90prim=E3=80=91Concat=20bug=20(#?=
 =?UTF-8?q?53350)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* modify concat_grad add sum comp rule

* modify opcompat
---
 .../prim/api/composite_backward/composite_backward_api.h  | 2 +-
 paddle/phi/api/yaml/op_compat.yaml                        | 6 ++++++
 python/paddle/fluid/tests/unittests/CMakeLists.txt        | 3 ++-
 python/paddle/fluid/tests/unittests/test_sum_op.py        | 6 ++++--
 python/paddle/incubate/autograd/composite_rules.py        | 8 ++++++++
 5 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
index 9790f36ec590a..4784f2fb61751 100644
--- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
+++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
@@ -470,7 +470,7 @@ void concat_grad(const std::vector<Tensor>& x,
     sections.push_back(x[i].dims()[axis_value]);
   }
   std::vector<Tensor> x_grad_tmp =
-      split<T>(out_grad, phi::IntArray(sections), axis);
+      split<T>(out_grad, phi::IntArray(sections), axis_value);
   for (int i = 0; i < x_num; ++i) {
     set_output<T>(x_grad_tmp.at(i), x_grad.at(i));
   }
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index 8d29a77337320..68778a1c85602 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -95,6 +95,12 @@
     attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
+- op : add_n (sum)
+  inputs:
+    {inputs : X}
+  outputs:
+    {out : Out}
+
 - op : addmm
   backward : addmm_grad
   inputs :
diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt
index 670a48faf5c9b..a383e1b0c0624 100755
--- a/python/paddle/fluid/tests/unittests/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt
@@ -1113,7 +1113,8 @@ set(TEST_CINN_OPS
     test_dropout_op
     test_group_norm_op
     test_tile_op
-    test_roll_op)
+    test_roll_op
+    test_sum_op)
 
 foreach(TEST_CINN_OPS ${TEST_CINN_OPS})
   if(WITH_CINN)
diff --git a/python/paddle/fluid/tests/unittests/test_sum_op.py b/python/paddle/fluid/tests/unittests/test_sum_op.py
index 0f006eab26ff4..99406f4599c83 100644
--- a/python/paddle/fluid/tests/unittests/test_sum_op.py
+++ b/python/paddle/fluid/tests/unittests/test_sum_op.py
@@ -45,6 +45,8 @@ class TestSumOp(OpTest):
     def setUp(self):
         self.op_type = "sum"
         self.python_api = sum_wrapper
+        self.public_python_api = paddle.add_n
+        self.prim_op_type = "comp"
         self.init_kernel_type()
         self.use_mkldnn = False
         self.init_kernel_type()
@@ -60,10 +62,10 @@ def init_kernel_type(self):
         self.dtype = np.float64
 
     def test_check_output(self):
-        self.check_output()
+        self.check_output(check_prim=True)
 
     def test_check_grad(self):
-        self.check_grad(['x0'], 'Out')
+        self.check_grad(['x0'], 'Out', check_prim=True)
 
 
 class TestSelectedRowsSumOp(unittest.TestCase):
diff --git a/python/paddle/incubate/autograd/composite_rules.py b/python/paddle/incubate/autograd/composite_rules.py
index 3a1a3ea7d6751..0a47715c6062c 100644
--- a/python/paddle/incubate/autograd/composite_rules.py
+++ b/python/paddle/incubate/autograd/composite_rules.py
@@ -679,6 +679,14 @@ def group_norm_composite(x, scale, bias, epsilon, groups, data_layout):
     return out, ret_mean_, ret_var_
 
 
+@REGISTER_COMPOSITE('sum')
+def sum_composite(x):
+    ans = 0
+    for xi in x:
+        ans += xi
+    return ans
+
+
 @REGISTER_COMPOSITE('leaky_relu')
 def leaky_relu_composite(x, negative_slope):
     """define composite rule of op leaky_relu."""

From e8d296efecd95ecccd3fb71c2dbcd70245119b7d Mon Sep 17 00:00:00 2001
From: HydrogenSulfate <490868991@qq.com>
Date: Thu, 27 Apr 2023 11:36:28 +0800
Subject: [PATCH 102/405] Add jacobian and hessian (#53331)

* add jacobian and hessian in paddle.autograd

* disable unitest 'func_multi_input' for bug in high-order gradient of multiply

* add dimension checks

* add support for 0-D tensor

* change return type from Jacobian to Hessian in hessian function

* refine Jacobian _flatten function for single xs

* refine support for 0-D tensor

* 1. add 'func_multi_input' unitest for multiply_grad_kernel bug fixed
already.
2. support non-inplace math operation via magical method overwriting.

* add unitest for math operation and raise error when 0-D tensor is indexed

* add ndim check on ys and xs according to is_batched, and add one unitest

* refine docstring of jacobian and hessian

* move paddle.incubate.autograd.Jacobian/Hessian to paddle.incubate.autograd.functional.Jacobian/Hessian

* remove single_input unitest case because numerical differentiation is wrong

* remove 3 unitest for numerical result(reference result) is wrong

* 1. rename autodiff.py to autograd.py
2. increase TIMEOUT to 100

* cancel modification for functional Jacobian/Hessian

* 1. use tuple as return type instead of list
2. refine docstring

* add more unitest case to improve coverage

* remove 2 unitest of Hessian for numerical result is wrong

* remove 1 unitest of Hessian for numerical result is wrong

* remove 1 unitest of Hessian for numerical result is wrong

* change unit test to shape check

* correct doc and replace incubate API to stable API in _grad
---
 python/paddle/autograd/__init__.py     |   3 +
 python/paddle/autograd/autograd.py     | 712 +++++++++++++++++++++++++
 test/autograd/CMakeLists.txt           |   1 +
 test/autograd/test_autograd_dynamic.py | 668 +++++++++++++++++++++++
 4 files changed, 1384 insertions(+)
 create mode 100644 python/paddle/autograd/autograd.py
 create mode 100644 test/autograd/test_autograd_dynamic.py

diff --git a/python/paddle/autograd/__init__.py b/python/paddle/autograd/__init__.py
index 16dbef6ae44b4..40bcd036f184d 100644
--- a/python/paddle/autograd/__init__.py
+++ b/python/paddle/autograd/__init__.py
@@ -18,12 +18,15 @@
 from ..fluid.dygraph.base import is_grad_enabled  # noqa: F401
 from ..fluid.dygraph.base import set_grad_enabled  # noqa: F401
 from . import backward_mode  # noqa: F401
+from .autograd import jacobian, hessian  # noqa: F401
 from .backward_mode import backward  # noqa: F401
 from .py_layer import PyLayer  # noqa: F401
 from .py_layer import PyLayerContext  # noqa: F401
 from .saved_tensors_hooks import saved_tensors_hooks
 
 __all__ = [  # noqa
+    'jacobian',
+    'hessian',
     'backward',
     'PyLayer',
     'PyLayerContext',
diff --git a/python/paddle/autograd/autograd.py b/python/paddle/autograd/autograd.py
new file mode 100644
index 0000000000000..cfd5442138c3a
--- /dev/null
+++ b/python/paddle/autograd/autograd.py
@@ -0,0 +1,712 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional, Sequence, Tuple, Union
+
+import paddle
+from paddle.fluid import framework
+
+
+def as_tensors(xs):
+    if isinstance(xs, framework.Variable):
+        return xs
+    elif isinstance(xs, Sequence):
+        return tuple(xs)
+    else:
+        return xs
+
+
+class Jacobian:
+    r"""Computes the Jacobian matrix of given xs and ys.
+
+    Once the Jacobian ``J`` is constructed, you can use a multidimensional index
+    to retrieve the submatrix of ``J``, as same as slicing a Tensor. The
+    submatrix is lazily evaluated along row axis, and will be cached once
+    evaluated.
+
+    you can retrieve the submatrix by
+    following methods:
+
+        * J[:], retrieving the full matrix.
+        * J[:, :, j], retrieving the partial derivatives w.r.t. the j'th input
+          variable.
+        * J[:, i, :], retrieving the partial derivatives w.r.t. the i'th output
+          variable.
+        * J[:, i, j], retrieving the partial derivatives w.r.t. the i'th output
+          variable and the j'th input variable.
+
+    Notes:
+
+        Eclipsis index is not supported currently.
+
+    Args:
+
+        ys (Tensor|Tuple[Tensor, ...]): The output derived from xs .
+        xs (Tensor|Tuple[Tensor, ...]): The input tensor(s) .
+        is_batched (bool): If true, the first axis is batch axis. Defaults to
+            False.
+
+    Returns:
+
+        Jacobian (Object): A python object retains the Jacobian matrix.
+
+    """
+
+    def __init__(self, ys, xs, is_batched=False):
+        if not is_batched:
+            if not 0 <= len(xs.shape) <= 1:
+                raise ValueError(
+                    f"xs.ndim should be 0 or 1 when is_batched=False"
+                    f" but got {len(xs.shape)}"
+                )
+            if not 0 <= len(ys.shape) <= 1:
+                raise ValueError(
+                    f"ys.ndim should be 0 or 1 when is_batched=False"
+                    f" but got {len(ys.shape)}"
+                )
+            self._jacobian = _JacobianNoBatch(ys, xs)
+        else:
+            if not 1 <= len(ys.shape) <= 2:
+                raise ValueError(
+                    f"ys.ndim should be 1 or 2 when is_batched=True"
+                    f" but got {len(ys.shape)}"
+                )
+            if not 1 <= len(xs.shape) <= 2:
+                raise ValueError(
+                    f"xs.ndim should be 1 or 2 when is_batched=True"
+                    f" but got {len(xs.shape)}"
+                )
+            self._jacobian = _JacobianBatchFirst(ys, xs)
+
+    @property
+    def shape(self):
+        """The shape of flattened Jacobian matrix."""
+        return self._jacobian.shape
+
+    def __getitem__(self, indexes):
+        return self._jacobian[indexes]
+
+    def __getattr__(self, __name: str):
+        if __name == "shape":
+            return getattr(self._jacobian, __name)
+        if __name == "_evaluate_all":
+            return getattr(self._jacobian, __name)
+        return getattr(self._jacobian._evaluate_all(), __name)
+
+    def __add__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs + rhs
+
+    def __sub__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs - rhs
+
+    def __mul__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs * rhs
+
+    def __div__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs / rhs
+
+    def __truediv__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs / rhs
+
+    def __pow__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs**rhs
+
+    def __mod__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs % rhs
+
+    def __floordiv__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs // rhs
+
+    def __matmul__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs @ rhs
+
+    def __eq__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs == rhs
+
+    def __ne__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs != rhs
+
+    def __lt__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs < rhs
+
+    def __le__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs <= rhs
+
+    def __gt__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs > rhs
+
+    def __ge__(self, other):
+        lhs = self._evaluate_all()
+        rhs = other._evaluate_all() if isinstance(other, Jacobian) else other
+        return lhs >= rhs
+
+
+class Hessian(Jacobian):
+    pass
+
+
+class _Jacobian:
+    """The base class for computing Jacobian matrix.
+
+    ``_Jacobian`` implementes the core logic of multidimensional index and lazy
+    evaluation for Jacobian matrix, subclass only need to overwrite following
+    methods:
+
+        * ``_lazy_axis()``,  return the axis along which will be lazy
+            evaluating.
+        * ``_flatten(xs)``, flattens the inputs ``xs``.
+        * ``_evaluate(index)``, evaluates one slice along ``_lazy_axis`` .
+
+    Notes:
+
+        Because currently PaddlePaddle only support reverse differentiation by
+        ``paddle.grad``, so lazy evaluation is only supported along the row of
+        Jacobian matrix, which means that slicing along row will get better
+        performance.
+
+    """
+
+    def __init__(self, ys, xs):
+        self.original_xs_shape = xs.shape
+        self.original_ys_shape = ys.shape
+        self._xs = xs
+        self._ys = ys
+        if len(self._ys.shape) == 0 and not self.is_batched:
+            self._ys = self._ys.reshape(
+                [
+                    -1,
+                ]
+            )
+        if len(self._ys.shape) == 1 and self.is_batched:
+            self._ys = self._ys.reshape([-1, 1])
+
+        self._flatten_xs = self._flatten(as_tensors(self._xs))
+        self._flatten_ys = self._flatten(as_tensors(self._ys))
+        self._cache = {}
+
+    @property
+    def _lazy_axis(self):
+        """ "The axis of lazily evaluated."""
+        raise NotImplementedError
+
+    def _lazy_indexes(self, indexes):
+        idx = indexes[self._lazy_axis]
+        return (
+            (idx,)
+            if isinstance(idx, int)
+            else tuple(range(idx.start, idx.stop, idx.step))
+        )
+
+    def _flatten(self, xs):
+        raise NotImplementedError
+
+    def _shifted_indexes(self, indexes, lazy_axis_size=0):
+        idx = indexes[self._lazy_axis]
+        shifted_lazy_axis_idx = (
+            0 if isinstance(idx, int) else slice(0, lazy_axis_size, 1)
+        )
+        return (
+            indexes[: self._lazy_axis]
+            + (shifted_lazy_axis_idx,)
+            + indexes[self._lazy_axis + 1 :]
+        )
+
+    def __getitem__(self, indexes):
+        if self.is_batched is False:
+            if len(self.shape) == 0:
+                # xs and ys are both 0-D tensor
+                raise IndexError("0-D tensor can not be indexed.")
+            elif len(self.shape) == 1:
+                # either ys or xs is 0-D tensor
+                indexes = (
+                    (0, indexes)
+                    if len(self.original_ys_shape) == 0
+                    else (indexes, 0)
+                )
+        else:
+            if len(self.shape) == 1:
+                # xs and ys are both 1-D tensor
+                indexes = (indexes, 0, 0)
+            elif len(self.shape) == 2:
+                # either xs or ys is 1-D tensor
+                if isinstance(indexes, slice):
+                    indexes = (indexes, slice(None, None, None))
+                else:
+                    indexes = (
+                        (indexes[0], 0, indexes[1])
+                        if len(self.original_ys_shape) == 1
+                        else (indexes[0], indexes[1], 0)
+                    )
+
+        indexes = _multi_index(indexes, self.inner_shape)
+
+        if isinstance(indexes[self._lazy_axis], int):
+            other_indexes = (
+                indexes[: self._lazy_axis] + indexes[self._lazy_axis + 1 :]
+            )
+            return self._cached_evaluate(indexes[self._lazy_axis])[
+                other_indexes
+            ]
+        lazy_indexes = self._lazy_indexes(indexes)
+        # Using concat and reshape to replace stack operator temporarily, as
+        # it is not a primitive operator.
+        shape = list(self.inner_shape)
+        shape[self._lazy_axis] = len(lazy_indexes)
+        part_jac = paddle.concat(
+            [self._cached_evaluate(i) for i in lazy_indexes],
+            axis=self._lazy_axis,
+        ).reshape(shape)
+        result = part_jac[self._shifted_indexes(indexes, len(lazy_indexes))]
+
+        # squeeze redundant 1 in shape
+        if len(result.shape) > len(self.shape):
+            for _ in range(len(result.shape) - len(self.shape)):
+                result = result.squeeze(-1)
+
+        return result
+
+    def _cached_evaluate(self, k):
+        if k is None:
+            return self._cached_evaluate(0).reshape([])
+        v = self._cache.get(k)
+        if v is None:
+            v = self._evaluate(k)
+            self._cache[k] = v
+        return v
+
+    def _evaluate(self, index):
+        """Evaluate one slice at along lazy axis."""
+        raise NotImplementedError
+
+    def _evaluate_all(self):
+        if len(self.shape) == 0:
+            return self._cached_evaluate(None)
+        else:
+            return self[:]
+
+
+class _JacobianNoBatch(_Jacobian):
+    """Compute Jacobian matrix without batch dimension.
+    Suppose the mapping is :math:`f: R^M \to R^N`, the output shape is
+    ``(N, M)`` .
+    """
+
+    def __init__(self, ys, xs):
+        self.is_batched = False
+        super().__init__(ys, xs)
+        # inner_shape is for convenient, it will regard 0-D tensor as 1-D tensor
+        self.inner_shape = [
+            *(self._flatten_ys.shape[0:1]),
+            *(self._flatten_xs.shape[0:1]),
+        ]
+        self.shape = [
+            *(self.original_ys_shape[0:1]),
+            *(self.original_xs_shape[0:1]),
+        ]
+
+    @property
+    def _lazy_axis(self):
+        return 0
+
+    def _flatten(self, xs):
+        if not isinstance(xs, Sequence):
+            return xs.reshape((-1,))
+        return paddle.concat(tuple(x.reshape((-1,)) for x in xs))
+
+    def _evaluate(self, row_index):
+        return self._flatten(
+            _grad_for_jacobian(
+                self._flatten_ys[row_index],
+                self._xs,
+            )
+        )
+
+
+class _JacobianBatchFirst(_Jacobian):
+    """Compute Jacobian matrix with batch at first axis.
+    Suppose the mapping is :math:`f: R^{B,M} \to R^{B,N}`, the output shape is
+    ``(B, N, M)`` .
+    """
+
+    def __init__(self, ys, xs):
+        self.is_batched = True
+        super().__init__(ys, xs)
+        # inner_shape is for convenient, it will regard 0-D tensor as 1-D tensor
+        self.inner_shape = [
+            *(self._flatten_xs.shape[0:1]),
+            *(self._flatten_ys.shape[1:2]),
+            *(self._flatten_xs.shape[1:2]),
+        ]
+        self.shape = [
+            *(self._flatten_xs.shape[0:1]),
+            *(self.original_ys_shape[1:2]),
+            *(self.original_xs_shape[1:2]),
+        ]
+
+    @property
+    def _lazy_axis(self):
+        return 1
+
+    def _flatten(self, xs):
+        if not isinstance(xs, Sequence):
+            return xs.reshape((xs.shape[0], -1))
+        return paddle.concat(
+            tuple(x.reshape((x.shape[0], -1)) for x in as_tensors(xs)), 1
+        )
+
+    def _evaluate(self, row_index):
+        return self._flatten(
+            _grad_for_jacobian(self._flatten_ys[:, row_index], self._xs)
+        )
+
+
+def _multi_index(indexes, shape):
+    """A tool for parsing N-dimensional index into a standard format.
+
+    Currently supporting following input format:
+        * ([positive|negative|slice], ...), the right-most elements can be
+            omited.
+
+    The standard format after converted is slice tuple which contains N elements:
+        * ([positive|slice], ..., [positive|slice])
+
+    Notes:
+        Ellipsis indexes such as ``(..., i), (i, ...)`` is not supported.
+
+    Args:
+        indexes (tuple): The input indexes.
+        shape (tuple): The input shape.
+
+    Returns:
+        tuple: The standard format index as the above description.
+    """
+    indexes = indexes if isinstance(indexes, Sequence) else (indexes,)
+    if any(isinstance(i, type(Ellipsis)) for i in indexes):
+        raise IndexError('Ellipsis index currently is not supported.')
+    # Fill the right-most elements.
+    indexes = indexes + (slice(0, None, None),) * (len(shape) - len(indexes))
+    # Convert to positive index.
+    positive_indexes = []
+    for i, index in enumerate(indexes):
+        if isinstance(index, slice):
+            index = slice(
+                index.start or 0, index.stop or shape[i], index.step or 1
+            )
+            positive_indexes.append(
+                slice(
+                    index.start + shape[i] if index.start < 0 else index.start,
+                    index.stop + shape[i] if index.stop < 0 else index.stop,
+                    # Negative step means index backward, no need to convert to
+                    # positive interger.
+                    index.step,
+                )
+            )
+        elif isinstance(index, int):
+            positive_indexes.append(index + shape[i] if index < 0 else index)
+        else:
+            raise TypeError(f'Not supported index type {index}.')
+    return tuple(positive_indexes)
+
+
+def jacobian(
+    ys: Union[paddle.Tensor, Tuple[paddle.Tensor, ...]],
+    xs: Union[paddle.Tensor, Tuple[paddle.Tensor, ...]],
+    batch_axis: Optional[int] = None,
+) -> Union[Tuple[Tuple[Jacobian, ...], ...], Tuple[Jacobian, ...], Jacobian]:
+    r"""
+    Computes the Jacobian of the dependent variable ``ys`` versus the independent
+    variable ``xs``.
+
+    Where ``ys`` represents the output of ``xs`` after a certain operation, ``ys`` and
+    ``xs`` can be Tensor or tuple of Tensors, ``batch_axis`` indicates the position of
+    the batch dimension of the parameter data.
+
+    When the input is a tuple Tensors, the returned result is a ``Jacobian`` object with
+    the same number of nesting levels as ``xs``, and each Jacobian has the same shape as
+    The ``xs`` tuples are identical in one-to-one correspondence.
+
+    - When ``batch_axis=None``, only 0-dimensional Tensor or 1-dimensional Tensor is
+          supported, assuming the shape of ``xs`` is ``[N, ]``, the shape of ``ys`` is
+          ``[M, ]``, then the output Jacobian matrix shape is ``[M, N]``.
+
+    - When ``batch_axis=0``, only 1-dimensional Tensor or 2-dimensional Tensor is
+          supported, assuming the shape of ``xs`` is ``[B, N]``, The shape of ``ys`` is
+          ``[B, M]``, then the output Jacobian matrix shape is ``[B, M, N]``.
+
+    After the ``Jacobian`` object is created, the actual calculation process does not
+    occur, but the lazy evaluation method is used for calculation. It can be
+    multi-dimensional indexed to obtain the entire Jacobian matrix or sub-matrix, and
+    the actual calculation will be performed at this time the value is calculated and
+    the result is returned. At the same time, in the actual evaluation process, the
+    calculated sub-matrix will be cached to avoid duplicate calculations in the
+    subsequent indexing process.
+
+    For example, assuming ``Jacobian`` instance ``J`` has shape ``[B, M, N]``, assuming
+    ``M > 4`` , then ``J[:, 1:4:1, :]`` means to get the values from row ``1`` to row
+    ``3`` of ``J``. In actual calculation, only the rows ``1`` to ``3`` are evaluated,
+    and the calculation results of ``1`` to ``3`` will be cached at the granularity of
+    the row, and will be used next time. When obtaining one or more rows of results
+    above, the already calculated parts will not be recalculated.
+
+    Args:
+
+        ys (Union[paddle.Tensor, Tuple[paddle.Tensor, ...]]): Output or tuple of outputs derived from xs.
+        xs (Union[paddle.Tensor, Tuple[paddle.Tensor, ...]]): Input or tuple of inputs.
+        batch_axis (Optional[int], optional): Index of batch axis. Defaults to None.
+
+    Returns:
+
+        Union[Tuple[Tuple[Jacobian, ...], ...], Tuple[Jacobian, ...], Jacobian]: Jacobian(s) of ys deriveted from xs.
+
+    Examples:
+
+        .. code-block:: python
+
+            import paddle
+
+            x1 = paddle.randn([3, ])
+            x2 = paddle.randn([3, ])
+            x1.stop_gradient = False
+            x2.stop_gradient = False
+
+            y = x1 + x2
+
+            J = paddle.autograd.jacobian(y, (x1, x2))
+            J_y_x1 = J[0][:] # evaluate result of dy/dx1
+            J_y_x2 = J[1][:] # evaluate result of dy/dx2
+
+            print(J_y_x1.shape) # [3, 3]
+            print(J_y_x2.shape) # [3, 3]
+    """
+
+    if batch_axis is not None and batch_axis != 0:
+        raise ValueError(
+            f"batch_axis should be None or 0, but got {batch_axis}."
+        )
+
+    # TODO(HydrogenSulfate): support batch_axis > 0
+    is_batched = batch_axis is not None
+    if isinstance(ys, Sequence) and isinstance(xs, Sequence):
+        _jacobian = tuple(
+            tuple(Jacobian(_ys, _xs, is_batched) for _xs in xs) for _ys in ys
+        )
+    elif isinstance(ys, Sequence) and not isinstance(xs, Sequence):
+        _jacobian = tuple(Jacobian(_ys, xs, is_batched) for _ys in ys)
+    elif not isinstance(ys, Sequence) and isinstance(xs, Sequence):
+        _jacobian = tuple(Jacobian(ys, _xs, is_batched) for _xs in xs)
+    else:
+        _jacobian = Jacobian(ys, xs, is_batched)
+
+    return _jacobian
+
+
+def hessian(
+    ys: paddle.Tensor,
+    xs: Union[paddle.Tensor, Tuple[paddle.Tensor, ...]],
+    batch_axis: Optional[int] = None,
+) -> Union[Tuple[Tuple[Hessian, ...], ...], Hessian]:
+    r"""
+    Computes the Jacobian of the dependent variable ``ys`` versus the independent
+    variable ``xs``.
+
+    Among them, ``ys`` means the output of ``xs`` after a certain operation, ``ys`` can
+    only be a single Tensor, ``xs`` can be a Tensor or a Tensor tuple, and
+    ``batch_axis`` means The position of the batch dimension of the parameter data.
+
+    When the input ``xs`` is a Tensor tuple, the returned result is a ``Hessian`` tuple,
+    assuming that the internal shape of the ``xs`` tuple is composed of
+    ``([M1, ], [M2, ]) ``, the shape of the returned result consists of
+    ``(([M1, M1], [M1, M2]), ([M2, M1], [M2, M2]))``
+
+    - When ``batch_axis=None``, only 0-dimensional Tensor or 1-dimensional Tensor is
+        supported, assuming that the shape of ``xs`` is ``[N, ]``, and the shape of ``ys`` is ``[ ]``(0-dimensional Tensor), the final output is a single Hessian matrix whose shape is ``[N, N]``.
+
+    - When ``batch_axis=0``, only 1-dimensional Tensor or 2-dimensional Tensor is
+        supported, assuming that the shape of ``xs`` is ``[B, N]``, and the shape of ``ys`` is `` [B, ]``, the final output Jacobian matrix shape is ``[B, N, N]``.
+
+    After the ``Hessian`` object is created, the complete calculation process does not
+    occur, but a partial lazy evaluation method is used for calculation. It can be
+    multi-dimensionally indexed to obtain the entire Hessian matrix or sub-matrix. At
+    this time, the actual Evaluates the computation and returns the result. At the same
+    time, in the actual evaluation process, the calculated sub-matrix will be cached to
+    avoid repeated calculations in the subsequent indexing process.
+
+    Args:
+
+        ys (paddle.Tensor): Output derived from xs which contain one element.
+        xs (Union[paddle.Tensor, Tuple[paddle.Tensor, ...]]): Input or tuple of inputs.
+        batch_axis (Optional[int], optional): Index of batch axis. Defaults to None.
+
+    Returns:
+
+        Union[Tuple[Tuple[Hessian, ...], ...], Tuple[Hessian, ...], Hessian]: Hessian(s) of ys deriveted from xs.
+
+    Examples:
+
+        .. code-block:: python
+
+            import paddle
+
+            x1 = paddle.randn([3, ])
+            x2 = paddle.randn([4, ])
+            x1.stop_gradient = False
+            x2.stop_gradient = False
+
+            y = x1.sum() + x2.sum()
+
+            H = paddle.autograd.hessian(y, (x1, x2))
+            H_y_x1_x1 = H[0][0][:] # evaluate result of ddy/dx1x1
+            H_y_x1_x2 = H[0][1][:] # evaluate result of ddy/dx1x2
+            H_y_x2_x1 = H[1][0][:] # evaluate result of ddy/dx2x1
+            H_y_x2_x2 = H[1][1][:] # evaluate result of ddy/dx2x2
+
+            print(H_y_x1_x1.shape) # [3, 3]
+            print(H_y_x1_x2.shape) # [3, 4]
+            print(H_y_x2_x1.shape) # [4, 3]
+            print(H_y_x2_x2.shape) # [4, 4]
+    """
+
+    if batch_axis is None:
+        if ys.numel() > 1:
+            raise ValueError(
+                f"Only support ys.numel()({ys.numel()})==1 when batch_axis is None."
+            )
+        ys = ys.reshape(())
+    elif isinstance(batch_axis, int):
+        if ys[0].numel() > 1:
+            raise ValueError(
+                f"Only support ys[0].numel()({ys.numel()})==1 when batch_axis is int"
+            )
+        # TODO(HydrogenSulfate): support batch_axis > 0
+        if batch_axis != 0:
+            raise ValueError("Only support batch_axis=0 yet.")
+        ys = ys.reshape((-1,))
+    else:
+        raise ValueError(
+            f"batch_axis should be None or int, but got {type(batch_axis)}."
+        )
+
+    _jacobian = jacobian(ys, xs, batch_axis)
+    if not isinstance(xs, Sequence):
+        hessian = jacobian(_jacobian, xs, batch_axis)
+
+        # change classname to Hessian instead of Jacobian.
+        hessian.__class__ = Hessian
+    else:
+        hessian = tuple(jacobian(_j, xs, batch_axis) for _j in _jacobian)
+
+        # change classname to Hessian instead of Jacobian.
+        for i in range(len(hessian)):
+            for j in range(len(hessian[0])):
+                hessian[i][j].__class__ = Hessian
+
+    return hessian
+
+
+def _replace_none_with_zero_tensor(xs, refs):
+    if xs is None:
+        xs = paddle.zeros_like(refs)
+        xs.stop_gradient = refs.stop_gradient
+        return xs
+    elif isinstance(xs, Sequence):
+        return tuple(
+            _replace_none_with_zero_tensor(x, refs[i]) for i, x in enumerate(xs)
+        )
+    else:
+        return xs
+
+
+def _grad_for_jacobian(ys, xs, v=None):
+    """A gradient function that can be used in dynamic graph and static graph.
+
+    The ``grad`` combines ``paddle.grad`` used in dynamic graph and
+    ``paddle.static.gradients`` used in static graph, and do following changes:
+
+    * The ``allow_unused`` flag is removed and set defaults to true internally,
+        none in outputs will be replaced by zero tensor.
+    * The ``create_graph`` flag is removed and set defaults to true internally,
+        only makes sense in dynamic graph.
+    * When xs is a single Tensor, ``paddle.grad`` returns a list which only
+        contains one Tensor. It may confuse users, thus in this case we improve
+        to return a single Tensor in _grad_for_jacobian interface.
+
+    Args:
+        ys (Tensor|Sequence[Tensor]): The output tensor or tensor sequence of
+            the graph to compute gradients.
+        xs (Tensor|Sequence[Tensor]): The input tensor or tensor sequence of the graph to
+            compute gradients. The returned values of this API are the
+            gradients of inputs .
+        v (Tensor|Sequence[Tensor]|None,optional): The initial gradient values
+            of outputs . If grad_outputs is None, the initial gradient values of
+            outputs would be Tensors filled with 1; if grad_outputs is not None,
+            it must have the same length as outputs , and in this case, the
+            initial gradient value of the i-th outputs would be: (1) a Tensor
+            filled with 1 when the i-th element of grad_outputs is None;
+            (2) the i-th element of grad_outputs when the i-th element of
+            grad_outputs is a Tensor. Default None.
+
+    Returns:
+        Tensor|tuple[Tensor]: Tensor or a tuple of Tensors, whose length is the
+            same as the Tensor number inside inputs, and the i-th returned
+            Tensor is the sum of gradients of outputs with respect to the i-th
+            inputs.
+    """
+    if paddle.fluid._non_static_mode():
+        # paddle.grad returns a list though the inputs is a signle Tensor. The
+        # follow code snippet fixes the problem by return the first element of
+        # xs_grad when the xs is a signle Tensor.
+        xs_grad = paddle.grad(ys, xs, v, create_graph=True, allow_unused=True)
+        if (
+            isinstance(xs, paddle.fluid.framework.Variable)
+            and isinstance(xs_grad, Sequence)
+            and len(xs_grad) > 0
+        ):
+            xs_grad = xs_grad[0]
+    else:
+        xs_grad = paddle.static.gradients(ys, xs, v)
+        if (
+            isinstance(xs, framework.Variable)
+            and isinstance(xs_grad, Sequence)
+            and len(xs_grad) > 0
+        ):
+            xs_grad = xs_grad[0]
+    return _replace_none_with_zero_tensor(xs_grad, xs)
diff --git a/test/autograd/CMakeLists.txt b/test/autograd/CMakeLists.txt
index f1af779f4f46d..592517cb8e3da 100644
--- a/test/autograd/CMakeLists.txt
+++ b/test/autograd/CMakeLists.txt
@@ -15,6 +15,7 @@ foreach(TEST_OP ${TEST_OPS})
   py_test_modules(${TEST_OP} MODULES ${TEST_OP} ENVS ${GC_ENVS})
 endforeach()
 
+set_tests_properties(test_autograd_dynamic PROPERTIES TIMEOUT 100)
 set_tests_properties(test_autograd_functional_dynamic PROPERTIES TIMEOUT 200)
 set_tests_properties(test_autograd_functional_static PROPERTIES TIMEOUT 160)
 set_tests_properties(test_minimize PROPERTIES TIMEOUT 60)
diff --git a/test/autograd/test_autograd_dynamic.py b/test/autograd/test_autograd_dynamic.py
new file mode 100644
index 0000000000000..e8e5d8b626c5f
--- /dev/null
+++ b/test/autograd/test_autograd_dynamic.py
@@ -0,0 +1,668 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import typing
+import unittest
+
+import config
+import numpy as np
+import utils
+
+import paddle
+import paddle.nn.functional as F
+from paddle.incubate.autograd.utils import as_tensors
+
+
+def make_v(f, inputs):
+    outputs = as_tensors(f(*inputs))
+    return [paddle.ones_like(x) for x in outputs]
+
+
+@utils.place(config.DEVICES)
+@utils.parameterize(
+    (utils.TEST_CASE_NAME, 'func', 'xs'),
+    (
+        ('1d_in_1d_out', utils.square, np.array([2.0, 3.0])),
+        (
+            'single_in_single_out',
+            utils.square,
+            np.random.rand(
+                6,
+            ),
+        ),
+        (
+            'multi_in_single_out',
+            paddle.matmul,
+            (
+                np.random.rand(
+                    4,
+                ),
+                np.random.rand(
+                    4,
+                ),
+            ),
+        ),
+    ),
+)
+class TestJacobianNoBatch(unittest.TestCase):
+    def setUp(self):
+        self._dtype = (
+            self.xs[0].dtype
+            if isinstance(self.xs, typing.Sequence)
+            else self.xs.dtype
+        )
+        self._eps = (
+            config.TOLERANCE.get(str(self._dtype))
+            .get("first_order_grad")
+            .get("eps")
+        )
+        self._rtol = (
+            config.TOLERANCE.get(str(self._dtype))
+            .get("first_order_grad")
+            .get("rtol")
+        )
+        self._atol = (
+            config.TOLERANCE.get(str(self._dtype))
+            .get("first_order_grad")
+            .get("atol")
+        )
+
+    def test_jacobian(self):
+        xs = (
+            [paddle.to_tensor(x, stop_gradient=False) for x in self.xs]
+            if isinstance(self.xs, typing.Sequence)
+            else paddle.to_tensor(self.xs, stop_gradient=False)
+        )
+        ys = (
+            self.func(*xs) if isinstance(xs, typing.Sequence) else self.func(xs)
+        )
+        self._actual = paddle.autograd.jacobian(ys, xs, batch_axis=None)
+        if isinstance(self._actual, (tuple, list)):
+            self._actual = paddle.concat([x[:] for x in self._actual], axis=1)
+        self._expected = self._get_expected()
+
+        Index = collections.namedtuple('Index', ('type', 'value'))
+        indexes = (
+            Index('all', (slice(0, None, None), slice(0, None, None))),
+            Index('row', (0, slice(0, None, None))),
+            Index('col', (slice(0, None, None), 0)),
+            Index('multi-row', (slice(0, 2, 1), slice(0, None, None))),
+        )
+        self.assertEqual(self._actual[:].numpy().dtype, self._expected.dtype)
+        for index in indexes:
+            np.testing.assert_allclose(
+                self._actual.__getitem__(index.value),
+                self._expected.__getitem__(index.value),
+                rtol=self._rtol,
+                atol=self._atol,
+                err_msg=f'Testcase {index.type} index not passed, value is {index.value}',
+            )
+
+    def test_jacobian_attribute_operator(self):
+        xs = (
+            [paddle.to_tensor(x, stop_gradient=False) for x in self.xs]
+            if isinstance(self.xs, typing.Sequence)
+            else paddle.to_tensor(self.xs, stop_gradient=False)
+        )
+        ys = (
+            self.func(*xs) if isinstance(xs, typing.Sequence) else self.func(xs)
+        )
+        self._actual = paddle.autograd.jacobian(ys, xs, batch_axis=None)
+        if isinstance(self._actual, (tuple, list)):
+            self._actual = paddle.concat([x[:] for x in self._actual], axis=1)
+        self._expected = self._get_expected()
+
+        Index = collections.namedtuple('Index', ('type', 'value'))
+        indexes = (
+            Index('all', (slice(0, None, None), slice(0, None, None))),
+            Index('row', (0, slice(0, None, None))),
+            Index('col', (slice(0, None, None), 0)),
+            Index('multi-row', (slice(0, 2, 1), slice(0, None, None))),
+        )
+        self.assertEqual(self._actual.numpy().dtype, self._expected.dtype)
+        for index in indexes:
+            np.testing.assert_allclose(
+                self._actual.__getitem__(index.value),
+                self._expected.__getitem__(index.value),
+                rtol=self._rtol,
+                atol=self._atol,
+                err_msg=f'Testcase {index.type} index not passed, value is {index.value}',
+            )
+
+    def _get_expected(self):
+        xs = (
+            [paddle.to_tensor(x, stop_gradient=False) for x in self.xs]
+            if isinstance(self.xs, typing.Sequence)
+            else paddle.to_tensor(self.xs, stop_gradient=False)
+        )
+        jac = utils._compute_numerical_jacobian(
+            self.func, xs, self._eps, self._dtype
+        )
+        return utils._np_concat_matrix_sequence(jac, utils.MatrixFormat.NM)
+
+
+@utils.place(config.DEVICES)
+@utils.parameterize(
+    (utils.TEST_CASE_NAME, 'func', 'xs'),
+    (
+        (
+            '1d_in_1d_out',
+            utils.square,
+            np.array([[1.0, 2.0, 3.0], [3.0, 4.0, 3.0]]),
+        ),
+        ('multi_in_single_out', utils.square, np.random.rand(2, 3)),
+    ),
+)
+class TestJacobianBatchFirst(unittest.TestCase):
+    def setUp(self):
+        self._dtype = (
+            self.xs[0].dtype
+            if isinstance(self.xs, typing.Sequence)
+            else self.xs.dtype
+        )
+        self._eps = (
+            config.TOLERANCE.get(str(self._dtype))
+            .get("first_order_grad")
+            .get("eps")
+        )
+        self._rtol = (
+            config.TOLERANCE.get(str(self._dtype))
+            .get("first_order_grad")
+            .get("rtol")
+        )
+        self._atol = (
+            config.TOLERANCE.get(str(self._dtype))
+            .get("first_order_grad")
+            .get("atol")
+        )
+
+    def test_jacobian(self):
+        xs = (
+            [paddle.to_tensor(x, stop_gradient=False) for x in self.xs]
+            if isinstance(self.xs, typing.Sequence)
+            else paddle.to_tensor(self.xs, stop_gradient=False)
+        )
+        ys = (
+            self.func(*xs) if isinstance(xs, typing.Sequence) else self.func(xs)
+        )
+        self._actual = paddle.autograd.jacobian(ys, xs, batch_axis=0)
+        self._expected = self._get_expected()
+
+        Index = collections.namedtuple('Index', ('type', 'value'))
+        indexes = (
+            Index(
+                'all',
+                (
+                    slice(0, None, None),
+                    slice(0, None, None),
+                    slice(0, None, None),
+                ),
+            ),
+            Index('row', (slice(0, None, None), 0, slice(0, None, None))),
+            Index('col', (slice(0, None, None), slice(0, None, None), 0)),
+            Index(
+                'batch',
+                (slice(0, 2, None), slice(0, None, None), slice(0, None, None)),
+            ),
+            Index(
+                'multi_row',
+                (slice(0, 1, None), slice(0, 2, 1), slice(0, None, None)),
+            ),
+        )
+        self.assertEqual(self._actual[:].numpy().dtype, self._expected.dtype)
+        for index in indexes:
+            np.testing.assert_allclose(
+                self._actual.__getitem__(index.value),
+                self._expected.__getitem__(index.value),
+                rtol=self._rtol,
+                atol=self._atol,
+                err_msg=f'Testcase {index.type} index not passed, value is {index.value}',
+            )
+
+    def test_jacobian_attribute_operator(self):
+        # test for attribute operator "."
+        xs = (
+            [paddle.to_tensor(x, stop_gradient=False) for x in self.xs]
+            if isinstance(self.xs, typing.Sequence)
+            else paddle.to_tensor(self.xs, stop_gradient=False)
+        )
+        ys = (
+            self.func(*xs) if isinstance(xs, typing.Sequence) else self.func(xs)
+        )
+        self._actual = paddle.autograd.jacobian(ys, xs, batch_axis=0)
+        self._expected = self._get_expected()
+
+        Index = collections.namedtuple('Index', ('type', 'value'))
+        indexes = (
+            Index(
+                'all',
+                (
+                    slice(0, None, None),
+                    slice(0, None, None),
+                    slice(0, None, None),
+                ),
+            ),
+            Index('row', (slice(0, None, None), 0, slice(0, None, None))),
+            Index('col', (slice(0, None, None), slice(0, None, None), 0)),
+            Index(
+                'batch',
+                (slice(0, 2, None), slice(0, None, None), slice(0, None, None)),
+            ),
+            Index(
+                'multi_row',
+                (slice(0, 1, None), slice(0, 2, 1), slice(0, None, None)),
+            ),
+        )
+        self.assertEqual(self._actual.numpy().dtype, self._expected.dtype)
+        for index in indexes:
+            np.testing.assert_allclose(
+                self._actual.__getitem__(index.value),
+                self._expected.__getitem__(index.value),
+                rtol=self._rtol,
+                atol=self._atol,
+                err_msg=f'Testcase {index.type} index not passed, value is {index.value}',
+            )
+
+    def _get_expected(self):
+        xs = (
+            [paddle.to_tensor(x, stop_gradient=False) for x in self.xs]
+            if isinstance(self.xs, typing.Sequence)
+            else paddle.to_tensor(self.xs, stop_gradient=False)
+        )
+        jac = utils._compute_numerical_batch_jacobian(
+            self.func, xs, self._eps, self._dtype, False
+        )
+        jac = utils._np_concat_matrix_sequence(jac, utils.MatrixFormat.NBM)
+        return utils._np_transpose_matrix_format(
+            jac, utils.MatrixFormat.NBM, utils.MatrixFormat.BNM
+        )
+
+
+class TestHessianNoBatch(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.shape = (4,)
+        self.dtype = 'float32'
+        self.np_dtype = np.float32
+        self.numerical_delta = (
+            config.TOLERANCE.get(self.dtype).get("second_order_grad").get("eps")
+        )
+        self.rtol = (
+            config.TOLERANCE.get(self.dtype)
+            .get("second_order_grad")
+            .get("rtol")
+        )
+        self.atol = (
+            config.TOLERANCE.get(self.dtype)
+            .get("second_order_grad")
+            .get("atol")
+        )
+        self.x = paddle.rand(shape=self.shape, dtype=self.dtype)
+        self.y = paddle.rand(shape=self.shape, dtype=self.dtype)
+
+    def func_create_graph_true(self):
+        def func(x):
+            return paddle.sum(F.sigmoid(x))
+
+        numerical_hessian = utils._compute_numerical_hessian(
+            func, self.x, self.numerical_delta, self.np_dtype
+        )
+        numerical_hessian = utils._np_concat_matrix_sequence(numerical_hessian)
+        self.x.stop_gradient = False
+        hessian = paddle.autograd.hessian(func(self.x), self.x, batch_axis=None)
+        assert not hessian[:].stop_gradient
+        np.testing.assert_allclose(
+            hessian[:].numpy(), numerical_hessian, self.rtol, self.atol
+        )
+
+    def func_out_not_single(self):
+        def func(x):
+            return x * x
+
+        with self.assertRaises(ValueError):
+            x = paddle.ones([3])
+            paddle.autograd.hessian(func(x), x, batch_axis=None)
+
+    def func_add(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected + 1.0
+        actual = H + 1.0
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_sub(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected - 1.0
+        actual = H - 1.0
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_mul(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected * 2.0
+        actual = H * 2.0
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_div(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected / 2.0
+        actual = H / 2.0
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_truediv(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected / 2.0
+        actual = H / 2.0
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_pow(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected**3.0
+        actual = H**3.0
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_mod(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected % 1.2
+        actual = H % 1.2
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_matmul(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected @ expected
+        actual = H @ H
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_eq(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected == expected
+        actual = H == H
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_ne(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected != expected
+        actual = H != H
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_lt(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected < expected
+        actual = H < H
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_le(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected <= expected
+        actual = H <= H
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_gt(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected > expected
+        actual = H > H
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_ge(self):
+        def func(x):
+            return (x * x).sum()
+
+        H = paddle.autograd.hessian(func(self.x), self.x)
+        expected = np.diag(np.full((self.x.size,), 2.0))
+
+        expected = expected >= expected
+        actual = H >= H
+        np.testing.assert_allclose(actual, expected, self.rtol, self.atol)
+
+    def func_0Dtensor_index(self):
+        x_0d = self.x[0].reshape([])
+
+        def func(x):
+            return x * x
+
+        with self.assertRaises(IndexError):
+            H = paddle.autograd.hessian(func(x_0d), x_0d)
+            H = H[:]
+
+    def func_2Dtensor(self):
+        x_2d = self.x.reshape([self.x.shape[0] // 2, 2])
+
+        def func(x):
+            return (x * x).sum()
+
+        with self.assertRaises(ValueError):
+            H = paddle.autograd.hessian(func(x_2d), x_2d)
+
+    def test_all_cases(self):
+        self.setUpClass()
+        self.func_create_graph_true()
+        self.func_out_not_single()
+        self.func_add()
+        self.func_sub()
+        self.func_mul()
+        self.func_div()
+        self.func_truediv()
+        self.func_pow()
+        self.func_mod()
+        self.func_matmul()
+        self.func_eq()
+        self.func_ne()
+        self.func_lt()
+        self.func_le()
+        self.func_gt()
+        self.func_ge()
+        self.func_0Dtensor_index()
+        self.func_2Dtensor()
+
+
+class TestHessianBatchFirst(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.x_shape = (5, 2)
+        self.weight_shape = (2, 4)
+        self.y_shape = (5, 2)
+        self.nbatch, self.nrow = 5, 2
+        self.dtype = 'float32'
+        self.np_dtype = np.float32
+        self.numerical_delta = (
+            config.TOLERANCE.get(self.dtype).get('second_order_grad').get('eps')
+        )
+        self.rtol = (
+            config.TOLERANCE.get(self.dtype)
+            .get('second_order_grad')
+            .get('rtol')
+        )
+        self.atol = (
+            config.TOLERANCE.get(self.dtype)
+            .get('second_order_grad')
+            .get('atol')
+        )
+        self.x = paddle.rand(shape=self.x_shape, dtype=self.dtype)
+        self.x.stop_gradient = False
+        self.weight = paddle.rand(shape=self.weight_shape, dtype=self.dtype)
+        self.weight.stop_gradient = False
+        self.y = paddle.rand(shape=self.y_shape, dtype=self.dtype)
+        self.y.stop_gradient = False
+
+    def func_allow_unused(self):
+        def func(x, y):
+            return paddle.matmul(x * x, self.weight)[:, 0:1]
+
+        xs_len = 2
+        expected = utils._compute_numerical_batch_hessian(
+            func, [self.x, self.y], self.numerical_delta, self.np_dtype
+        )
+        expected = np.reshape(
+            np.array(expected),
+            (xs_len, xs_len, self.nrow, self.nbatch, self.nrow),
+        )
+        expected = [list(row) for row in expected]
+        expected = utils._np_concat_matrix_sequence(expected)
+        expected = utils._np_transpose_matrix_format(
+            expected, utils.MatrixFormat.NBM, utils.MatrixFormat.BNM
+        )
+
+        actual = paddle.autograd.hessian(
+            func(self.x, self.y), [self.x, self.y], batch_axis=0
+        )
+        actual = paddle.concat(
+            [
+                paddle.concat([actual[i][j][:] for j in range(2)], axis=2)
+                for i in range(2)
+            ],
+            axis=1,
+        )
+
+        np.testing.assert_allclose(
+            actual.shape, expected.shape, rtol=self.rtol, atol=self.atol
+        )
+
+    def func_stop_gradient(self):
+        def func(x):
+            return paddle.matmul(x * x, self.weight)[:, 0:1]
+
+        expected = utils._compute_numerical_batch_hessian(
+            func, self.x, self.numerical_delta, self.np_dtype
+        )
+
+        x = self.x.clone()
+        x.stop_gradient = True
+        H = paddle.autograd.hessian(func(self.x), self.x, batch_axis=0)[:]
+        actual = utils._np_transpose_matrix_format(
+            H[:].numpy(), utils.MatrixFormat.BNM, utils.MatrixFormat.NBM
+        )
+        actual = actual.reshape((H.shape[1], -1))
+
+        np.testing.assert_allclose(
+            actual.shape, np.asarray(expected).shape, self.rtol, self.atol
+        )
+
+    def func_out_not_single(self):
+        def func(x):
+            return x * x
+
+        with self.assertRaises(ValueError):
+            x = paddle.ones((3, 3))
+            paddle.autograd.hessian(func(x), x, batch_axis=0)
+
+    def func_batch_axis_except_0(self):
+        def func(x):
+            return x * x
+
+        with self.assertRaises(ValueError):
+            x = paddle.ones([3])
+            paddle.autograd.hessian(func(x), x, batch_axis=2)
+
+    def func_ndim_bigger_than_2(self):
+        def func(x):
+            return (x * x).sum()
+
+        with self.assertRaises(ValueError):
+            x = paddle.ones([3, 3, 3, 3])
+            paddle.autograd.hessian(func(x), x, batch_axis=0)
+
+    def func_batch_axis_str(self):
+        def func(x):
+            return (x * x).sum()
+
+        with self.assertRaises(ValueError):
+            x = paddle.ones([3, 3, 3, 3])
+            paddle.autograd.hessian(func(x), x, batch_axis="0")
+
+    def func_ellipsis_index(self):
+        def func(x):
+            return (x * x).sum()
+
+        with self.assertRaises(IndexError):
+            x = paddle.ones([2, 3])
+            H = paddle.autograd.hessian(func(x), x, batch_axis=0)[..., 1]
+
+    def test_all_cases(self):
+        self.setUpClass()
+        self.func_allow_unused()
+        self.func_stop_gradient()
+        self.func_out_not_single()
+        self.func_batch_axis_except_0()
+        self.func_ndim_bigger_than_2()
+        self.func_batch_axis_str()
+        self.func_ellipsis_index()
+
+
+if __name__ == "__main__":
+    np.random.seed(2022)
+    unittest.main()

From 89d1dd2ec0273d1d4d13c8346bff97c3b0e4a7d3 Mon Sep 17 00:00:00 2001
From: HongyuJia <jiahongyu@baidu.com>
Date: Thu, 27 Apr 2023 13:03:41 +0800
Subject: [PATCH 103/405] [CustomOP Unittest] XPU unittest only keep forward
 test (#53021)

---
 test/custom_op/custom_relu_op_xpu.cc          | 127 --------------
 .../test_custom_relu_op_xpu_setup.py          | 156 +-----------------
 2 files changed, 3 insertions(+), 280 deletions(-)

diff --git a/test/custom_op/custom_relu_op_xpu.cc b/test/custom_op/custom_relu_op_xpu.cc
index ee717785ad848..8d9e2e2af4980 100644
--- a/test/custom_op/custom_relu_op_xpu.cc
+++ b/test/custom_op/custom_relu_op_xpu.cc
@@ -31,28 +31,6 @@ void relu_cpu_forward_kernel(const data_t* x_data,
   }
 }
 
-template <typename data_t>
-void relu_cpu_backward_kernel(const data_t* grad_out_data,
-                              const data_t* out_data,
-                              data_t* grad_x_data,
-                              int64_t out_numel) {
-  for (int64_t i = 0; i < out_numel; ++i) {
-    grad_x_data[i] =
-        grad_out_data[i] * (out_data[i] > static_cast<data_t>(0) ? 1. : 0.);
-  }
-}
-
-template <typename data_t>
-void relu_cpu_double_backward_kernel(const data_t* out_data,
-                                     const data_t* ddx_data,
-                                     data_t* ddout_data,
-                                     int64_t ddout_numel) {
-  for (int64_t i = 0; i < ddout_numel; ++i) {
-    ddout_data[i] =
-        ddx_data[i] * (out_data[i] > static_cast<data_t>(0) ? 1. : 0.);
-  }
-}
-
 std::vector<paddle::Tensor> relu_cpu_forward(const paddle::Tensor& x) {
   CHECK_CPU_INPUT(x);
   auto out = paddle::empty_like(x);
@@ -66,77 +44,12 @@ std::vector<paddle::Tensor> relu_cpu_forward(const paddle::Tensor& x) {
   return {out};
 }
 
-std::vector<paddle::Tensor> relu_cpu_backward(const paddle::Tensor& x,
-                                              const paddle::Tensor& out,
-                                              const paddle::Tensor& grad_out) {
-  auto grad_x = paddle::empty_like(x);
-
-  PD_DISPATCH_FLOATING_TYPES(out.type(), "relu_cpu_backward", ([&] {
-                               relu_cpu_backward_kernel<data_t>(
-                                   grad_out.data<data_t>(),
-                                   out.data<data_t>(),
-                                   grad_x.data<data_t>(),
-                                   out.size());
-                             }));
-
-  return {grad_x};
-}
-
-std::vector<paddle::Tensor> relu_cpu_double_backward(
-    const paddle::Tensor& out, const paddle::Tensor& ddx) {
-  CHECK_CPU_INPUT(out);
-  CHECK_CPU_INPUT(ddx);
-  auto ddout = paddle::empty(out.shape(), out.dtype(), out.place());
-
-  PD_DISPATCH_FLOATING_TYPES(out.type(), "relu_cpu_double_backward", ([&] {
-                               relu_cpu_double_backward_kernel<data_t>(
-                                   out.data<data_t>(),
-                                   ddx.data<data_t>(),
-                                   ddout.mutable_data<data_t>(out.place()),
-                                   ddout.size());
-                             }));
-
-  return {ddout};
-}
-
 std::vector<paddle::Tensor> relu_xpu_forward(const paddle::Tensor& x) {
   CHECK_XPU_INPUT(x);
   auto out = paddle::relu(x);
   return {out};
 }
 
-std::vector<paddle::Tensor> relu_xpu_backward(const paddle::Tensor& x,
-                                              const paddle::Tensor& out,
-                                              const paddle::Tensor& grad_out) {
-  CHECK_XPU_INPUT(x);
-  CHECK_XPU_INPUT(out);
-  CHECK_XPU_INPUT(grad_out);
-  auto grad_x = paddle::empty_like(x, x.dtype(), x.place());
-  auto ones = paddle::experimental::full_like(x, 1.0, x.dtype(), x.place());
-  auto zeros = paddle::experimental::full_like(x, 0.0, x.dtype(), x.place());
-  auto condition = paddle::experimental::greater_than(x, zeros);
-
-  grad_x = grad_out * paddle::where(condition, ones, zeros);
-
-  return {grad_x};
-}
-
-std::vector<paddle::Tensor> relu_xpu_double_backward(
-    const paddle::Tensor& out, const paddle::Tensor& ddx) {
-  CHECK_XPU_INPUT(out);
-  CHECK_XPU_INPUT(ddx);
-  auto ddout = paddle::empty(out.shape(), out.dtype(), out.place());
-  auto ones =
-      paddle::experimental::full_like(out, 1.0, out.dtype(), out.place());
-  auto zeros =
-      paddle::experimental::full_like(out, 0.0, out.dtype(), out.place());
-  auto condition = paddle::experimental::greater_than(out, zeros);
-
-  ddout = paddle::multiply(ddx, paddle::where(condition, ones, zeros));
-
-  return {ddout};
-}
-
 std::vector<paddle::Tensor> ReluForward(const paddle::Tensor& x) {
   if (x.is_cpu()) {
     return relu_cpu_forward(x);
@@ -147,47 +60,7 @@ std::vector<paddle::Tensor> ReluForward(const paddle::Tensor& x) {
   }
 }
 
-std::vector<paddle::Tensor> ReluBackward(const paddle::Tensor& x,
-                                         const paddle::Tensor& out,
-                                         const paddle::Tensor& grad_out) {
-  if (x.is_cpu()) {
-    return relu_cpu_backward(x, out, grad_out);
-  } else if (x.is_xpu()) {
-    return relu_xpu_backward(x, out, grad_out);
-  } else {
-    PD_THROW("Not implemented.");
-  }
-}
-
-std::vector<paddle::Tensor> ReluDoubleBackward(const paddle::Tensor& out,
-                                               const paddle::Tensor& ddx) {
-  if (out.is_cpu()) {
-    return relu_cpu_double_backward(out, ddx);
-  } else if (out.place().GetType() == phi::AllocationType::XPU) {
-    return relu_xpu_double_backward(out, ddx);
-  } else {
-    PD_THROW("Not implemented.");
-  }
-}
-
-std::vector<std::vector<int64_t>> ReluDoubleBackwardInferShape(
-    const std::vector<int64_t>& out_shape,
-    const std::vector<int64_t>& ddx_shape) {
-  return {out_shape};
-}
-
 PD_BUILD_OP(custom_relu)
     .Inputs({"X"})
     .Outputs({"Out"})
     .SetKernelFn(PD_KERNEL(ReluForward));
-
-PD_BUILD_GRAD_OP(custom_relu)
-    .Inputs({"X", "Out", paddle::Grad("Out")})
-    .Outputs({paddle::Grad("X")})
-    .SetKernelFn(PD_KERNEL(ReluBackward));
-
-PD_BUILD_DOUBLE_GRAD_OP(custom_relu)
-    .Inputs({"Out", paddle::Grad(paddle::Grad("X"))})
-    .Outputs({paddle::Grad(paddle::Grad("Out"))})
-    .SetKernelFn(PD_KERNEL(ReluDoubleBackward))
-    .SetInferShapeFn(PD_INFER_SHAPE(ReluDoubleBackwardInferShape));
diff --git a/test/custom_op/test_custom_relu_op_xpu_setup.py b/test/custom_op/test_custom_relu_op_xpu_setup.py
index e054eadafd03a..db8d56c742911 100644
--- a/test/custom_op/test_custom_relu_op_xpu_setup.py
+++ b/test/custom_op/test_custom_relu_op_xpu_setup.py
@@ -34,15 +34,7 @@ def custom_relu_dynamic(func, device, dtype, np_x, use_func=True):
     t.retain_grads()
 
     out = func(t) if use_func else paddle.nn.functional.relu(t)
-    out.retain_grads()
-    out.stop_gradient = False
-
-    out.backward()
-
-    if t.grad is None:
-        return out.numpy(), t.grad
-    else:
-        return out.numpy(), t.grad.numpy()
+    return out.numpy()
 
 
 def custom_relu_static(
@@ -71,80 +63,6 @@ def custom_relu_static(
     return out_v
 
 
-def custom_relu_static_inference(func, device, np_data, np_label, path_prefix):
-    paddle.set_device(device)
-
-    with static.scope_guard(static.Scope()):
-        with static.program_guard(static.Program()):
-            # simple module
-            data = static.data(
-                name='data', shape=[None, 1, 28, 28], dtype='float32'
-            )
-            label = static.data(name='label', shape=[None, 1], dtype='int64')
-
-            hidden = static.nn.fc(data, size=128)
-            hidden = func(hidden)
-            hidden = static.nn.fc(hidden, size=128)
-            predict = static.nn.fc(hidden, size=10, activation='softmax')
-            loss = paddle.nn.functional.cross_entropy(input=hidden, label=label)
-            avg_loss = paddle.mean(loss)
-
-            opt = paddle.optimizer.SGD(learning_rate=0.1)
-            opt.minimize(avg_loss)
-
-            # run start up model
-            exe = static.Executor()
-            exe.run(static.default_startup_program())
-
-            # train
-            for _ in range(4):
-                exe.run(
-                    static.default_main_program(),
-                    feed={'data': np_data, 'label': np_label},
-                    fetch_list=[avg_loss],
-                )
-
-            # save inference model
-            static.save_inference_model(path_prefix, [data], [predict], exe)
-
-            # get train predict value
-            predict_v = exe.run(
-                static.default_main_program(),
-                feed={'data': np_data, 'label': np_label},
-                fetch_list=[predict],
-            )
-
-    return predict_v
-
-
-def custom_relu_double_grad_dynamic(func, device, dtype, np_x, use_func=True):
-
-    paddle.set_device(device)
-
-    t = paddle.to_tensor(np_x, dtype=dtype, stop_gradient=False)
-    t.retain_grads()
-
-    out = func(t) if use_func else paddle.nn.functional.relu(t)
-    out.retain_grads()
-    dx = paddle.grad(
-        outputs=out,
-        inputs=t,
-        grad_outputs=paddle.ones_like(t),
-        create_graph=True,
-        retain_graph=True,
-    )
-
-    ddout = paddle.grad(
-        outputs=dx[0],
-        inputs=out.grad,
-        grad_outputs=paddle.ones_like(t),
-        create_graph=False,
-    )
-
-    assert ddout[0].numpy() is not None
-    return dx[0].numpy(), ddout[0].numpy()
-
-
 class TestNewCustomOpXpuSetUpInstall(unittest.TestCase):
     def setUp(self):
         cur_dir = os.path.dirname(os.path.abspath(__file__))
@@ -189,79 +107,11 @@ def test_static(self):
     def test_dynamic(self):
         for dtype in self.dtypes:
             x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
-            out, x_grad = custom_relu_dynamic(
-                self.custom_op, self.device, dtype, x
-            )
-            pd_out, pd_x_grad = custom_relu_dynamic(
-                self.custom_op, self.device, dtype, x, False
-            )
-            check_output(out, pd_out, "out")
-            check_output(x_grad, pd_x_grad, "x_grad")
-
-    def test_static_save_and_load_inference_model(self):
-        paddle.enable_static()
-        np_data = np.random.random((1, 1, 28, 28)).astype("float32")
-        np_label = np.random.random((1, 1)).astype("int64")
-        path_prefix = "self.custom_op_inference/custom_relu"
-
-        predict = custom_relu_static_inference(
-            self.custom_op, self.device, np_data, np_label, path_prefix
-        )
-        # load inference model
-        with static.scope_guard(static.Scope()):
-            exe = static.Executor()
-            [
-                inference_program,
-                feed_target_names,
-                fetch_targets,
-            ] = static.load_inference_model(path_prefix, exe)
-            predict_infer = exe.run(
-                inference_program,
-                feed={feed_target_names[0]: np_data},
-                fetch_list=fetch_targets,
-            )
-            check_output(predict, predict_infer, "predict")
-        paddle.disable_static()
-
-    def test_static_save_and_run_inference_predictor(self):
-        paddle.enable_static()
-        np_data = np.random.random((1, 1, 28, 28)).astype("float32")
-        np_label = np.random.random((1, 1)).astype("int64")
-        path_prefix = "self.custom_op_inference/custom_relu"
-        from paddle.inference import Config, create_predictor
-
-        predict = custom_relu_static_inference(
-            self.custom_op, self.device, np_data, np_label, path_prefix
-        )
-        # load inference model
-        config = Config(path_prefix + ".pdmodel", path_prefix + ".pdiparams")
-        predictor = create_predictor(config)
-        input_tensor = predictor.get_input_handle(
-            predictor.get_input_names()[0]
-        )
-        input_tensor.reshape(np_data.shape)
-        input_tensor.copy_from_cpu(np_data.copy())
-        predictor.run()
-        output_tensor = predictor.get_output_handle(
-            predictor.get_output_names()[0]
-        )
-        predict_infer = output_tensor.copy_to_cpu()
-        predict = np.array(predict).flatten()
-        predict_infer = np.array(predict_infer).flatten()
-        check_output_allclose(predict, predict_infer, "predict")
-        paddle.disable_static()
-
-    def test_func_double_grad_dynamic(self):
-        for dtype in self.dtypes:
-            x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
-            out, dx_grad = custom_relu_double_grad_dynamic(
-                self.custom_op, self.device, dtype, x
-            )
-            pd_out, pd_dx_grad = custom_relu_double_grad_dynamic(
+            out = custom_relu_dynamic(self.custom_op, self.device, dtype, x)
+            pd_out = custom_relu_dynamic(
                 self.custom_op, self.device, dtype, x, False
             )
             check_output(out, pd_out, "out")
-            check_output(dx_grad, pd_dx_grad, "dx_grad")
 
     def test_with_dataloader(self):
         paddle.disable_static()

From 3278dec7c887e0742ac032989e0fd6280b927b4a Mon Sep 17 00:00:00 2001
From: HongyuJia <jiahongyu@baidu.com>
Date: Thu, 27 Apr 2023 13:03:50 +0800
Subject: [PATCH 104/405] [Fix CppExtension Unittest] Change CUDAExtension to
 CppExtension if necessary (#53352)

* [Fix CppExtension Unittest] Change CUDAExtension to CppExtension if necessary

* Temporarily test cpp_extension under GPU

* Split mixed_extension unittest
---
 .../fluid/tests/cpp_extension/CMakeLists.txt  |  15 +-
 .../cpp_extension/cpp_extension_setup.py      |  17 +-
 .../cpp_extension/test_cpp_extension_jit.py   |   5 +-
 .../cpp_extension/test_cpp_extension_setup.py | 184 --------------
 .../test_mixed_extension_setup.py             | 234 ++++++++++++++++++
 5 files changed, 259 insertions(+), 196 deletions(-)
 create mode 100644 python/paddle/fluid/tests/cpp_extension/test_mixed_extension_setup.py

diff --git a/python/paddle/fluid/tests/cpp_extension/CMakeLists.txt b/python/paddle/fluid/tests/cpp_extension/CMakeLists.txt
index e9292b51751a1..61241edb8fc46 100644
--- a/python/paddle/fluid/tests/cpp_extension/CMakeLists.txt
+++ b/python/paddle/fluid/tests/cpp_extension/CMakeLists.txt
@@ -1,5 +1,12 @@
-py_test(test_cpp_extension_setup SRCS test_cpp_extension_setup.py)
-py_test(test_cpp_extension_jit SRCS test_cpp_extension_jit.py)
+if(WITH_TESTING)
+  if(WITH_GPU)
+    py_test(test_cpp_extension_setup SRCS test_cpp_extension_setup.py)
+    py_test(test_cpp_extension_jit SRCS test_cpp_extension_jit.py)
 
-set_tests_properties(test_cpp_extension_setup PROPERTIES TIMEOUT 120)
-set_tests_properties(test_cpp_extension_jit PROPERTIES TIMEOUT 120)
+    set_tests_properties(test_cpp_extension_setup PROPERTIES TIMEOUT 120)
+    set_tests_properties(test_cpp_extension_jit PROPERTIES TIMEOUT 120)
+  endif()
+endif()
+
+py_test(test_mixed_extension_setup SRCS test_mixed_extension_setup.py)
+set_tests_properties(test_mixed_extension_setup PROPERTIES TIMEOUT 120)
diff --git a/python/paddle/fluid/tests/cpp_extension/cpp_extension_setup.py b/python/paddle/fluid/tests/cpp_extension/cpp_extension_setup.py
index 5a4ff2afd6c63..ebede6aa5a6ab 100644
--- a/python/paddle/fluid/tests/cpp_extension/cpp_extension_setup.py
+++ b/python/paddle/fluid/tests/cpp_extension/cpp_extension_setup.py
@@ -17,7 +17,8 @@
 
 from utils import extra_compile_args
 
-from paddle.utils.cpp_extension import CUDAExtension, setup
+import paddle
+from paddle.utils.cpp_extension import CppExtension, CUDAExtension, setup
 
 paddle_includes = []
 for site_packages_path in getsitepackages():
@@ -30,14 +31,16 @@
 # Add current dir, search custom_power.h
 paddle_includes.append(os.path.dirname(os.path.abspath(__file__)))
 
+sources = ["custom_extension.cc", "custom_sub.cc"]
+Extension = CppExtension
+if paddle.is_compiled_with_cuda():
+    sources.append("custom_relu_forward.cu")
+    Extension = CUDAExtension
+
 setup(
     name='custom_cpp_extension',
-    ext_modules=CUDAExtension(
-        sources=[
-            "custom_extension.cc",
-            "custom_sub.cc",
-            "custom_relu_forward.cu",
-        ],
+    ext_modules=Extension(
+        sources=sources,
         include_dirs=paddle_includes,
         extra_compile_args=extra_compile_args,
         verbose=True,
diff --git a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py b/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py
index 5723df1b585b0..eac97c8457f46 100644
--- a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py
+++ b/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py
@@ -28,7 +28,10 @@
     sys.exit()
 
 # Compile and load cpp extension Just-In-Time.
-sources = ["custom_extension.cc", "custom_sub.cc", "custom_relu_forward.cu"]
+sources = ["custom_extension.cc", "custom_sub.cc"]
+if paddle.is_compiled_with_cuda():
+    sources.append("custom_relu_forward.cu")
+
 paddle_includes = []
 for site_packages_path in getsitepackages():
     paddle_includes.append(
diff --git a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py b/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py
index 15d0cb77d03d5..2de2dd80deac9 100644
--- a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py
+++ b/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py
@@ -21,80 +21,9 @@
 from utils import check_output
 
 import paddle
-from paddle import static
 from paddle.utils.cpp_extension.extension_utils import run_cmd
 
 
-def custom_relu_static(
-    func, device, dtype, np_x, use_func=True, test_infer=False
-):
-    paddle.enable_static()
-    paddle.set_device(device)
-
-    with static.scope_guard(static.Scope()):
-        with static.program_guard(static.Program()):
-            x = static.data(name='X', shape=[None, 8], dtype=dtype)
-            x.stop_gradient = False
-            out = func(x) if use_func else paddle.nn.functional.relu(x)
-            static.append_backward(out)
-
-            exe = static.Executor()
-            exe.run(static.default_startup_program())
-            # in static graph mode, x data has been covered by out
-            out_v = exe.run(
-                static.default_main_program(),
-                feed={'X': np_x},
-                fetch_list=[out.name],
-            )
-
-    paddle.disable_static()
-    return out_v
-
-
-def custom_relu_dynamic(func, device, dtype, np_x, use_func=True):
-    paddle.set_device(device)
-
-    t = paddle.to_tensor(np_x, dtype=dtype)
-    t.stop_gradient = False
-
-    out = func(t) if use_func else paddle.nn.functional.relu(t)
-    out.stop_gradient = False
-
-    out.backward()
-
-    if t.grad is None:
-        return out.numpy(), t.grad
-    else:
-        return out.numpy(), t.grad.numpy()
-
-
-def custom_relu_double_grad_dynamic(func, device, dtype, np_x, use_func=True):
-    paddle.set_device(device)
-
-    t = paddle.to_tensor(np_x, dtype=dtype, stop_gradient=False)
-    t.retain_grads()
-
-    out = func(t) if use_func else paddle.nn.functional.relu(t)
-    out.retain_grads()
-    dx = paddle.grad(
-        outputs=out,
-        inputs=t,
-        grad_outputs=paddle.ones_like(t),
-        create_graph=True,
-        retain_graph=True,
-    )
-
-    ddout = paddle.grad(
-        outputs=dx[0],
-        inputs=out.grad,
-        grad_outputs=paddle.ones_like(t),
-        create_graph=False,
-    )
-
-    assert ddout[0].numpy() is not None
-    return dx[0].numpy(), ddout[0].numpy()
-
-
 class TestCppExtensionSetupInstall(unittest.TestCase):
     """
     Tests setup install cpp extensions.
@@ -117,21 +46,6 @@ def setUp(self):
             custom_egg_path
         )
         sys.path.append(os.path.join(site_dir, custom_egg_path[0]))
-
-        # install mixed custom_op and extension
-        cmd = 'cd {} && {} mix_relu_and_extension_setup.py install'.format(
-            cur_dir, sys.executable
-        )
-        run_cmd(cmd)
-
-        site_dir = site.getsitepackages()[0]
-        custom_egg_path = [
-            x for x in os.listdir(site_dir) if 'mix_relu_extension' in x
-        ]
-        assert len(custom_egg_path) == 1, "Matched egg number is %d." % len(
-            custom_egg_path
-        )
-        sys.path.append(os.path.join(site_dir, custom_egg_path[0]))
         #################################
 
         # config seed
@@ -145,19 +59,11 @@ def tearDown(self):
         pass
 
     def test_cpp_extension(self):
-        # Extension
         self._test_extension_function_plain()
-        self._test_extension_function_mixed()
         self._test_vector_tensor()
         self._test_extension_class()
         self._test_nullable_tensor()
         self._test_optional_tensor()
-        # Custom op
-        self._test_static()
-        self._test_dynamic()
-        self._test_double_grad_dynamic()
-        if paddle.is_compiled_with_cuda():
-            self._test_cuda_relu()
 
     def _test_extension_function_plain(self):
         import custom_cpp_extension
@@ -177,25 +83,6 @@ def _test_extension_function_plain(self):
             target_out = np.exp(np_x) - np.exp(np_y)
             np.testing.assert_allclose(out.numpy(), target_out, atol=1e-5)
 
-    def _test_extension_function_mixed(self):
-        import mix_relu_extension
-
-        for dtype in self.dtypes:
-            np_x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
-            x = paddle.to_tensor(np_x, dtype=dtype)
-            np_y = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
-            y = paddle.to_tensor(np_y, dtype=dtype)
-
-            # Test mix_relu_extension
-            out = mix_relu_extension.custom_add2(x, y)
-            target_out = np.exp(np_x) + np.exp(np_y)
-            np.testing.assert_allclose(out.numpy(), target_out, atol=1e-5)
-
-            # Test we can call a method not defined in the main C++ file.
-            out = mix_relu_extension.custom_sub2(x, y)
-            target_out = np.exp(np_x) - np.exp(np_y)
-            np.testing.assert_allclose(out.numpy(), target_out, atol=1e-5)
-
     def _test_extension_class(self):
         import custom_cpp_extension
 
@@ -263,77 +150,6 @@ def _test_optional_tensor(self):
             err_msg=f'extension out: {x},\n numpy out: {x_np}',
         )
 
-    def _test_static(self):
-        import mix_relu_extension
-
-        for dtype in self.dtypes:
-            x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
-            out = custom_relu_static(
-                mix_relu_extension.custom_relu, "CPU", dtype, x
-            )
-            pd_out = custom_relu_static(
-                mix_relu_extension.custom_relu, "CPU", dtype, x, False
-            )
-            np.testing.assert_array_equal(
-                out,
-                pd_out,
-                err_msg='custom op out: {},\n paddle api out: {}'.format(
-                    out, pd_out
-                ),
-            )
-
-    def _test_dynamic(self):
-        import mix_relu_extension
-
-        for dtype in self.dtypes:
-            x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
-            out, x_grad = custom_relu_dynamic(
-                mix_relu_extension.custom_relu, "CPU", dtype, x
-            )
-            pd_out, pd_x_grad = custom_relu_dynamic(
-                mix_relu_extension.custom_relu, "CPU", dtype, x, False
-            )
-            np.testing.assert_array_equal(
-                out,
-                pd_out,
-                err_msg='custom op out: {},\n paddle api out: {}'.format(
-                    out, pd_out
-                ),
-            )
-            np.testing.assert_array_equal(
-                x_grad,
-                pd_x_grad,
-                err_msg='custom op x grad: {},\n paddle api x grad: {}'.format(
-                    x_grad, pd_x_grad
-                ),
-            )
-
-    def _test_double_grad_dynamic(self):
-        import mix_relu_extension
-
-        for dtype in self.dtypes:
-            x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
-            out, dx_grad = custom_relu_double_grad_dynamic(
-                mix_relu_extension.custom_relu, "CPU", dtype, x
-            )
-            pd_out, pd_dx_grad = custom_relu_double_grad_dynamic(
-                mix_relu_extension.custom_relu, "CPU", dtype, x, False
-            )
-            np.testing.assert_array_equal(
-                out,
-                pd_out,
-                err_msg='custom op out: {},\n paddle api out: {}'.format(
-                    out, pd_out
-                ),
-            )
-            np.testing.assert_array_equal(
-                dx_grad,
-                pd_dx_grad,
-                err_msg='custom op dx grad: {},\n paddle api dx grad: {}'.format(
-                    dx_grad, pd_dx_grad
-                ),
-            )
-
     def _test_cuda_relu(self):
         import custom_cpp_extension
 
diff --git a/python/paddle/fluid/tests/cpp_extension/test_mixed_extension_setup.py b/python/paddle/fluid/tests/cpp_extension/test_mixed_extension_setup.py
new file mode 100644
index 0000000000000..92aceff5067af
--- /dev/null
+++ b/python/paddle/fluid/tests/cpp_extension/test_mixed_extension_setup.py
@@ -0,0 +1,234 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import site
+import sys
+import unittest
+
+import numpy as np
+
+import paddle
+from paddle import static
+from paddle.utils.cpp_extension.extension_utils import run_cmd
+
+
+def custom_relu_static(
+    func, device, dtype, np_x, use_func=True, test_infer=False
+):
+    paddle.enable_static()
+    paddle.set_device(device)
+
+    with static.scope_guard(static.Scope()):
+        with static.program_guard(static.Program()):
+            x = static.data(name='X', shape=[None, 8], dtype=dtype)
+            x.stop_gradient = False
+            out = func(x) if use_func else paddle.nn.functional.relu(x)
+            static.append_backward(out)
+
+            exe = static.Executor()
+            exe.run(static.default_startup_program())
+            # in static graph mode, x data has been covered by out
+            out_v = exe.run(
+                static.default_main_program(),
+                feed={'X': np_x},
+                fetch_list=[out.name],
+            )
+
+    paddle.disable_static()
+    return out_v
+
+
+def custom_relu_dynamic(func, device, dtype, np_x, use_func=True):
+    paddle.set_device(device)
+
+    t = paddle.to_tensor(np_x, dtype=dtype)
+    t.stop_gradient = False
+
+    out = func(t) if use_func else paddle.nn.functional.relu(t)
+    out.stop_gradient = False
+
+    out.backward()
+
+    if t.grad is None:
+        return out.numpy(), t.grad
+    else:
+        return out.numpy(), t.grad.numpy()
+
+
+def custom_relu_double_grad_dynamic(func, device, dtype, np_x, use_func=True):
+    paddle.set_device(device)
+
+    t = paddle.to_tensor(np_x, dtype=dtype, stop_gradient=False)
+    t.retain_grads()
+
+    out = func(t) if use_func else paddle.nn.functional.relu(t)
+    out.retain_grads()
+    dx = paddle.grad(
+        outputs=out,
+        inputs=t,
+        grad_outputs=paddle.ones_like(t),
+        create_graph=True,
+        retain_graph=True,
+    )
+
+    ddout = paddle.grad(
+        outputs=dx[0],
+        inputs=out.grad,
+        grad_outputs=paddle.ones_like(t),
+        create_graph=False,
+    )
+
+    assert ddout[0].numpy() is not None
+    return dx[0].numpy(), ddout[0].numpy()
+
+
+class TestCppExtensionSetupInstall(unittest.TestCase):
+    """
+    Tests setup install cpp extensions.
+    """
+
+    def setUp(self):
+        cur_dir = os.path.dirname(os.path.abspath(__file__))
+        # install mixed custom_op and extension
+        # compile, install the custom op egg into site-packages under background
+        cmd = 'cd {} && {} mix_relu_and_extension_setup.py install'.format(
+            cur_dir, sys.executable
+        )
+        run_cmd(cmd)
+
+        site_dir = site.getsitepackages()[0]
+        custom_egg_path = [
+            x for x in os.listdir(site_dir) if 'mix_relu_extension' in x
+        ]
+        assert len(custom_egg_path) == 1, "Matched egg number is %d." % len(
+            custom_egg_path
+        )
+        sys.path.append(os.path.join(site_dir, custom_egg_path[0]))
+        #################################
+
+        # config seed
+        SEED = 2021
+        paddle.seed(SEED)
+        paddle.framework.random._manual_program_seed(SEED)
+
+        self.dtypes = ['float32', 'float64']
+
+    def tearDown(self):
+        pass
+
+    def test_cpp_extension(self):
+        # Extension
+        self._test_extension_function_mixed()
+        # Custom op
+        self._test_static()
+        self._test_dynamic()
+        self._test_double_grad_dynamic()
+
+    def _test_extension_function_mixed(self):
+        import mix_relu_extension
+
+        for dtype in self.dtypes:
+            np_x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            x = paddle.to_tensor(np_x, dtype=dtype)
+            np_y = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            y = paddle.to_tensor(np_y, dtype=dtype)
+
+            # Test mix_relu_extension
+            out = mix_relu_extension.custom_add2(x, y)
+            target_out = np.exp(np_x) + np.exp(np_y)
+            np.testing.assert_allclose(out.numpy(), target_out, atol=1e-5)
+
+            # Test we can call a method not defined in the main C++ file.
+            out = mix_relu_extension.custom_sub2(x, y)
+            target_out = np.exp(np_x) - np.exp(np_y)
+            np.testing.assert_allclose(out.numpy(), target_out, atol=1e-5)
+
+    def _test_static(self):
+        import mix_relu_extension
+
+        for dtype in self.dtypes:
+            x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            out = custom_relu_static(
+                mix_relu_extension.custom_relu, "CPU", dtype, x
+            )
+            pd_out = custom_relu_static(
+                mix_relu_extension.custom_relu, "CPU", dtype, x, False
+            )
+            np.testing.assert_array_equal(
+                out,
+                pd_out,
+                err_msg='custom op out: {},\n paddle api out: {}'.format(
+                    out, pd_out
+                ),
+            )
+
+    def _test_dynamic(self):
+        import mix_relu_extension
+
+        for dtype in self.dtypes:
+            x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            out, x_grad = custom_relu_dynamic(
+                mix_relu_extension.custom_relu, "CPU", dtype, x
+            )
+            pd_out, pd_x_grad = custom_relu_dynamic(
+                mix_relu_extension.custom_relu, "CPU", dtype, x, False
+            )
+            np.testing.assert_array_equal(
+                out,
+                pd_out,
+                err_msg='custom op out: {},\n paddle api out: {}'.format(
+                    out, pd_out
+                ),
+            )
+            np.testing.assert_array_equal(
+                x_grad,
+                pd_x_grad,
+                err_msg='custom op x grad: {},\n paddle api x grad: {}'.format(
+                    x_grad, pd_x_grad
+                ),
+            )
+
+    def _test_double_grad_dynamic(self):
+        import mix_relu_extension
+
+        for dtype in self.dtypes:
+            x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            out, dx_grad = custom_relu_double_grad_dynamic(
+                mix_relu_extension.custom_relu, "CPU", dtype, x
+            )
+            pd_out, pd_dx_grad = custom_relu_double_grad_dynamic(
+                mix_relu_extension.custom_relu, "CPU", dtype, x, False
+            )
+            np.testing.assert_array_equal(
+                out,
+                pd_out,
+                err_msg='custom op out: {},\n paddle api out: {}'.format(
+                    out, pd_out
+                ),
+            )
+            np.testing.assert_array_equal(
+                dx_grad,
+                pd_dx_grad,
+                err_msg='custom op dx grad: {},\n paddle api dx grad: {}'.format(
+                    dx_grad, pd_dx_grad
+                ),
+            )
+
+
+if __name__ == '__main__':
+    if os.name == 'nt' or sys.platform.startswith('darwin'):
+        # only support Linux now
+        sys.exit()
+    unittest.main()

From 18e9dcdcdb79467fdc799e332f0f504bfabfb6a1 Mon Sep 17 00:00:00 2001
From: Zhang Ting <zhangting_2017@163.com>
Date: Thu, 27 Apr 2023 14:11:35 +0800
Subject: [PATCH 105/405] [AMP] support OD level and skip dynamic loss scaling
 for bf16 (#53289)

* support OD level and skip dynamic loss scaling for bf16
---
 paddle/fluid/eager/amp_utils.h                |  6 +-
 paddle/fluid/imperative/amp_auto_cast.h       |  1 +
 paddle/fluid/pybind/imperative.cc             |  1 +
 python/paddle/amp/amp_lists.py                | 16 ++++-
 python/paddle/amp/auto_cast.py                | 36 +++++-----
 python/paddle/amp/grad_scaler.py              | 14 ++++
 ...perative_auto_mixed_precision_for_eager.py | 13 ++--
 test/amp/test_amp_api.py                      | 66 +++++++++++++++++++
 8 files changed, 129 insertions(+), 24 deletions(-)
 create mode 100644 test/amp/test_amp_api.py

diff --git a/paddle/fluid/eager/amp_utils.h b/paddle/fluid/eager/amp_utils.h
index ac9edc569df9f..2e06eaab8acae 100644
--- a/paddle/fluid/eager/amp_utils.h
+++ b/paddle/fluid/eager/amp_utils.h
@@ -129,7 +129,11 @@ inline phi::DataType GetAmpDestDtype(
                  ->count(op_name)) {
     dst_type = phi::DataType::FLOAT32;
   } else {
-    dst_type = GetPromoteType(op_name, amp_tensors_vector, amp_setting_dtype);
+    if (amp_level == paddle::imperative::AmpLevel::OD) {
+      dst_type = phi::DataType::FLOAT32;
+    } else {
+      dst_type = GetPromoteType(op_name, amp_tensors_vector, amp_setting_dtype);
+    }
   }
 
   if (dst_type == amp_setting_dtype &&
diff --git a/paddle/fluid/imperative/amp_auto_cast.h b/paddle/fluid/imperative/amp_auto_cast.h
index ced07b953d0c7..31dfc9dec57ab 100644
--- a/paddle/fluid/imperative/amp_auto_cast.h
+++ b/paddle/fluid/imperative/amp_auto_cast.h
@@ -31,6 +31,7 @@ enum class AmpLevel {
   O1,      // amp, mixed fp32-fp16
   O2,      // almost fp16
   O3,      // fp16
+  OD,      // only conv and matmul use low precison.
 };
 
 std::tuple<std::unordered_set<std::string>,
diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc
index 372fae12ec3bc..8d5bd524a1c14 100644
--- a/paddle/fluid/pybind/imperative.cc
+++ b/paddle/fluid/pybind/imperative.cc
@@ -2152,6 +2152,7 @@ void BindImperative(py::module *m_ptr) {
 
   py::enum_<paddle::imperative::AmpLevel>(m, "AmpLevel", py::arithmetic())
       .value("O0", paddle::imperative::AmpLevel::O0)
+      .value("OD", paddle::imperative::AmpLevel::OD)
       .value("O1", paddle::imperative::AmpLevel::O1)
       .value("O2", paddle::imperative::AmpLevel::O2)
       .value("O3", paddle::imperative::AmpLevel::O3)
diff --git a/python/paddle/amp/amp_lists.py b/python/paddle/amp/amp_lists.py
index f70c8f5ed7f91..51c557b9481c6 100644
--- a/python/paddle/amp/amp_lists.py
+++ b/python/paddle/amp/amp_lists.py
@@ -91,10 +91,19 @@
 BF16_BLACK_LIST = set()
 
 
+# At OD level, ops in WHITE_LIST will use FP16/BF16 and the others will use FP32.
 def white_list():
     white_list = {
-        "float16": {"O1": FP16_WHITE_LIST, "O2": FP16_WHITE_LIST},
-        "bfloat16": {"O1": BF16_WHITE_LIST, "O2": BF16_WHITE_LIST},
+        "float16": {
+            "OD": FP16_WHITE_LIST,
+            "O1": FP16_WHITE_LIST,
+            "O2": FP16_WHITE_LIST,
+        },
+        "bfloat16": {
+            "OD": BF16_WHITE_LIST,
+            "O1": BF16_WHITE_LIST,
+            "O2": BF16_WHITE_LIST,
+        },
     }
     return white_list
 
@@ -102,9 +111,10 @@ def white_list():
 def black_list():
     black_list = {
         "float16": {
+            "OD": set(),
             "O1": FP16_BLACK_LIST | FP16_EXTRA_BLACK_LIST,
             "O2": FP16_EXTRA_BLACK_LIST,
         },
-        "bfloat16": {"O1": BF16_BLACK_LIST, "O2": set()},
+        "bfloat16": {"OD": set(), "O1": BF16_BLACK_LIST, "O2": set()},
     }
     return black_list
diff --git a/python/paddle/amp/auto_cast.py b/python/paddle/amp/auto_cast.py
index ae9c957df68fa..d1eaf0dbd13ae 100644
--- a/python/paddle/amp/auto_cast.py
+++ b/python/paddle/amp/auto_cast.py
@@ -48,6 +48,7 @@ def __init__(self):
         self.model_parameters = []
         self.use_master_grad = False
         self.already_register_final_backward_hook = False
+        self.amp_dtype = 'float32'
 
     def __setattr__(self, name, val):
         self.__dict__[name] = val
@@ -320,10 +321,8 @@ def amp_guard(
 
     # check amp_level: O0-O2
     level = level.upper()
-    if not (level in ['O0', 'O1', 'O2']):
-        raise ValueError(
-            "level should be O0, O1 or O2. O0 represents fp32 train mode, O1 represents AMP train mode, O2 represents pure fp16/bf16 train mode."
-        )
+    if not (level in ['O0', 'OD', 'O1', 'O2']):
+        raise ValueError("level should be O0, OD, O1 or O2.")
 
     # check amp_dtype: float16 or bfloat16
     dtype = dtype.lower()
@@ -384,8 +383,11 @@ def amp_guard(
             )
 
     amp_dtype = dtype
+    amp_global_state().amp_dtype = amp_dtype
 
-    if level == 'O1':
+    if level == 'OD':
+        amp_level = AMP_LEVEL.OD
+    elif level == 'O1':
         amp_level = AMP_LEVEL.O1
     elif level == 'O2':
         amp_level = AMP_LEVEL.O2
@@ -642,22 +644,24 @@ def auto_cast(
 ):
     """
     Create a context which enables auto-mixed-precision(AMP) of operators executed in dynamic graph mode.
-    If enabled, the input data type (float32 or float16) of each operator is decided
+    If enabled, the input data type (float32, float16 or bfloat16) of each operator is decided
     by autocast algorithm for better performance.
 
-    Commonly, it is used together with `GradScaler` to achieve Auto-Mixed-Precision in
-    imperative mode. It is used together with `decorator` to achieve Pure fp16 in imperative mode.
+    Commonly, it is used together with `GradScaler` and `decorator` to achieve Auto-Mixed-Precision in
+    imperative mode.
 
     Args:
         enable(bool, optional): Enable auto-mixed-precision or not. Default is True.
-        custom_white_list(set|list|tuple, optional): The custom white_list. It's the set of ops that support
-             fp16 calculation and are considered numerically-safe and performance-critical. These ops
-             will be converted to fp16.
-        custom_black_list(set|list|tuple, optional): The custom black_list. The set of ops that support fp16
-             calculation and are considered numerically-dangerous and whose effects may also be
-             observed in downstream ops. These ops will not be converted to fp16.
-        level(str, optional): Auto mixed precision level. Accepted values are "O1" and "O2": O1 represent mixed precision, the input data type of each operator will be casted by white_list and black_list;
-             O2 represent Pure fp16, all operators parameters and input data will be casted to fp16, except operators in black_list, don't support fp16 kernel and batchnorm. Default is O1(amp)
+        custom_white_list(set|list|tuple, optional): A default white list is already set. Usually there is no need to set custom white list.
+             The set of ops should be considered numerically-safe and performance-critical. These ops will be converted to float16/bfloat16.
+        custom_black_list(set|list|tuple, optional): A default black list is already set. You can set a custom black list according to the model.
+             The set of ops are considered numerically-dangerous and whose effects may also be observed in downstream ops. These ops will not be
+             converted to float16/bfloat16.
+        level(str, optional): Auto mixed precision level. Accepted values are "O1", "O2" and "OD": At the O1 level, operators in the white list
+             will use float16/bfloat16 inputs for calculations, and operators in the black list will use float32 inputs for calculations. At the O2
+             level, model's parameters will be casted to float16/bfloat16 by using `decorator`, and operators that have all float16/bfloat16 inputs
+             will be converted to float16/bfloat16, and that have any float32 input will be converted to float32. For the OD level, operators in
+             default white list will compute in float16/bfloat16, and the others will compute in float32. Default is O1.
         dtype(str, optional): Whether to use 'float16' or 'bfloat16'. Default is 'float16'.
 
     Examples:
diff --git a/python/paddle/amp/grad_scaler.py b/python/paddle/amp/grad_scaler.py
index 0f6d9f21a32c6..2cade3482e9c3 100644
--- a/python/paddle/amp/grad_scaler.py
+++ b/python/paddle/amp/grad_scaler.py
@@ -24,6 +24,8 @@
 from paddle.fluid.dygraph import to_variable
 from paddle.fluid.framework import _dygraph_tracer, dygraph_only
 
+from .auto_cast import amp_global_state
+
 
 class OptimizerState(Enum):
     INIT = 0
@@ -179,6 +181,18 @@ def scale(self, var):
         """
         check_type(var, "var", core.eager.Tensor, 'AmpScaler.scale()')
 
+        if (
+            self._enable
+            and amp_global_state().amp_dtype != 'float16'
+            and self._use_dynamic_loss_scaling
+        ):
+            self._enable = False
+            self._use_dynamic_loss_scaling = False
+            warnings.warn(
+                'It is not recommended to use dynamic loss scaling for %s, so GradScaler is disable by default.'
+                % (amp_global_state().amp_dtype)
+            )
+
         if not self._enable:
             return var
 
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py
index 8d24febaff213..5de19dfb4113b 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py
@@ -194,8 +194,11 @@ def test_mode_exception(self):
 
 class TestAmpScaler(unittest.TestCase):
     def scale(self):
+        if not paddle.amp.is_float16_supported():
+            return
         with fluid.dygraph.guard():
-            data = paddle.rand([10, 1024])
+            with paddle.amp.auto_cast(dtype='float16'):
+                data = paddle.rand([10, 1024])
             scaler = paddle.amp.AmpScaler(init_loss_scaling=1024)
             scaled_data = scaler.scale(data)
             self.assertEqual(
@@ -333,9 +336,9 @@ def nan_inf(self):
             )
             scaler = paddle.amp.AmpScaler(init_loss_scaling=1024)
             data = fluid.dygraph.to_variable(inp_np)
-
-            out = model(data)
-            loss = paddle.mean(out)
+            with paddle.amp.auto_cast(dtype='float16'):
+                out = model(data)
+                loss = paddle.mean(out)
             scaled_loss = scaler.scale(loss)
             scaled_loss.backward()
             optimize_ops, params_grads = scaler.minimize(optimizer, scaled_loss)
@@ -348,6 +351,8 @@ def nan_inf(self):
                 )
 
     def test_nan_inf(self):
+        if not paddle.amp.is_float16_supported():
+            return
         self.nan_inf()
 
     def step_update_exception(self):
diff --git a/test/amp/test_amp_api.py b/test/amp/test_amp_api.py
new file mode 100644
index 0000000000000..7d397c7043206
--- /dev/null
+++ b/test/amp/test_amp_api.py
@@ -0,0 +1,66 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from amp_base_models import AmpTestBase
+
+import paddle
+
+
+class TestAutoCast(AmpTestBase):
+    def test_amp_OD_level(self):
+        conv = paddle.nn.Conv2D(
+            in_channels=1, out_channels=6, kernel_size=3, bias_attr=False
+        )
+        linear = paddle.nn.Linear(in_features=4, out_features=4)
+        with paddle.amp.auto_cast(level='OD'):
+            out1 = conv(paddle.rand(shape=[1, 1, 6, 6], dtype='float32'))
+            out2 = out1 + paddle.rand(shape=out1.shape, dtype='float16')
+            out3 = linear(out2)
+
+        self.assertEqual(out1.dtype, paddle.float16)
+        self.assertEqual(out2.dtype, paddle.float32)
+        self.assertEqual(out3.dtype, paddle.float32)
+
+
+class TestGradScaler(AmpTestBase):
+    def test_amp_grad_scaler(self):
+        model = paddle.nn.Conv2D(3, 2, 3)
+        optimizer = paddle.optimizer.SGD(
+            learning_rate=0.01, parameters=model.parameters()
+        )
+        scaler = paddle.amp.GradScaler()
+        data = paddle.rand([1, 3, 8, 8], dtype='float32')
+        paddle.amp.debugging.enable_operator_stats_collection()
+        with paddle.amp.auto_cast(
+            custom_black_list=['conv2d'], dtype='bfloat16'
+        ):
+            out = model(data)
+            loss = out.mean()
+        scaled = scaler.scale(loss)
+        scaled.backward()
+        scaler.minimize(optimizer, scaled)
+        optimizer.clear_grad()
+        paddle.amp.debugging.disable_operator_stats_collection()
+        op_list = paddle.fluid.core.get_low_precision_op_list()
+
+        self.assertEqual(scaler._enable, False)
+        self.assertEqual(scaler._use_dynamic_loss_scaling, False)
+        self.assertTrue('scale' not in op_list)
+        self.assertTrue('check_finite_and_unscale' not in op_list)
+
+
+if __name__ == '__main__':
+    unittest.main()

From 25b4ba7fef890aa74813ba47cecf175cc5bbe3a6 Mon Sep 17 00:00:00 2001
From: Sonder <55493212+AndSonder@users.noreply.github.com>
Date: Thu, 27 Apr 2023 14:15:57 +0800
Subject: [PATCH 106/405] Move fused feedforward (#53166)

* trans fused_feedward Compute function to phi

* add register info

* remove maxfunctor

* move fused feedward to phi

* remove sig file

* remove fliud include

* add include

* add include

* add sig file

* add output register info

* fix sig file

* Update fused_feedforward_sig.cc

* fix grad kernel

* update output register info

* fix

* open fused_feedforward static build

* add optional and fix code style

* fix output info for fused attention

* add optional param

* merge
---
 .../operators/fused/fused_attention_op.cu     |    2 +
 .../operators/fused/fused_feedforward_op.cu   | 1254 +++++++++--------
 paddle/phi/kernels/funcs/functors.h           |    5 -
 .../kernels/fused_feedforward_grad_kernel.h   |    4 +-
 .../kernels/fusion/gpu/fused_dropout_helper.h |   33 +-
 .../xpu/fused_feedforward_grad_kernel.cc      |    6 +-
 .../fluid/tests/unittests/CMakeLists.txt      |    4 +
 7 files changed, 673 insertions(+), 635 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_attention_op.cu b/paddle/fluid/operators/fused/fused_attention_op.cu
index de62fe38653b5..b1212f410fe9f 100644
--- a/paddle/fluid/operators/fused/fused_attention_op.cu
+++ b/paddle/fluid/operators/fused/fused_attention_op.cu
@@ -813,6 +813,8 @@ PD_REGISTER_KERNEL(fused_attention,
                    phi::dtype::float16,
                    double,
                    float) {
+  kernel->OutputAt(9).SetDataType(phi::DataType::UINT8);
+  kernel->OutputAt(14).SetDataType(phi::DataType::UINT8);
   if (kernel_key.dtype() == phi::DataType::FLOAT16) {
     kernel->OutputAt(0).SetDataType(phi::DataType::FLOAT32);
     kernel->OutputAt(1).SetDataType(phi::DataType::FLOAT32);
diff --git a/paddle/fluid/operators/fused/fused_feedforward_op.cu b/paddle/fluid/operators/fused/fused_feedforward_op.cu
index 2c3456d31025b..ee40633e4252b 100644
--- a/paddle/fluid/operators/fused/fused_feedforward_op.cu
+++ b/paddle/fluid/operators/fused/fused_feedforward_op.cu
@@ -13,634 +13,668 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/framework/op_version_registry.h"
-#include "paddle/fluid/operators/fused/fused_dropout_helper.h"
-#include "paddle/fluid/operators/matmul_v2_op.h"
+#include "paddle/fluid/operators/fused/fused_attention_utils.h"
 #include "paddle/phi/api/include/tensor.h"
+#include "paddle/phi/core/errors.h"
 #include "paddle/phi/kernels/funcs/blas/blas.h"
 #include "paddle/phi/kernels/funcs/broadcast_function.h"
 #include "paddle/phi/kernels/funcs/elementwise_functor.h"
 #include "paddle/phi/kernels/funcs/layer_norm_impl.cu.h"
+#include "paddle/phi/kernels/fusion/gpu/fused_dropout_helper.h"
+#include "paddle/phi/kernels/impl/matmul_grad_kernel_impl.h"
+
+namespace phi {
+namespace fusion {
+
+template <typename T, typename Context>
+void MatMul(const phi::GPUContext& dev_ctx,
+            const phi::DenseTensor& a,
+            const phi::DenseTensor& b,
+            phi::DenseTensor* c) {
+  auto blas = phi::funcs::GetBlas<Context, T>(dev_ctx);
+  auto a_2d = phi::FoldInitDims(a);
+  auto b_2d = phi::FoldInitDims(b);
+  auto mat_dim_a = phi::funcs::CreateMatrixDescriptor(a_2d.dims(), 0, false);
+  auto mat_dim_b = phi::funcs::CreateMatrixDescriptor(b_2d.dims(), 0, false);
+  T alpha = static_cast<T>(1.0);
+  blas.MatMul(a, mat_dim_a, b, mat_dim_b, alpha, c, T(0));
+}
 
-#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
-#include "paddle/fluid/distributed/collective/process_group_nccl.h"
-#include "paddle/fluid/platform/collective_helper.h"
-#include "paddle/fluid/platform/device/gpu/nccl_helper.h"
-#endif
-
-namespace paddle {
-namespace operators {
-
-template <typename T>
-static void AllReduce(phi::DenseTensor& tensor,  // NOLINT
-                      const int ring_id,
-                      const phi::GPUContext& ctx) {
-  if (ring_id == -1) return;
-#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
-  auto map = paddle::distributed::ProcessGroupMapFromGid::getInstance();
-
-  if (map->has(ring_id)) {
-    paddle::distributed::ProcessGroup* pg = map->get(ring_id);
-    auto pg_nccl = static_cast<distributed::ProcessGroupNCCL*>(pg);
-    paddle::distributed::AllreduceOptions opts;
-    opts.reduce_op = distributed::ReduceOp::SUM;
-    auto task = pg_nccl->AllReduce(&tensor, tensor, opts, true, true);
-    task->Wait();
+template <typename T, typename Context>
+void FFN(const phi::GPUContext& dev_ctx,
+         const phi::DenseTensor& x,
+         const phi::DenseTensor& linear1_weight,
+         const phi::DenseTensor* linear1_bias,
+         const phi::DenseTensor& linear2_weight,
+         const phi::DenseTensor* linear2_bias,
+         const phi::DenseTensor* ln1_scale,
+         const phi::DenseTensor* ln1_bias,
+         const phi::DenseTensor* ln2_scale,
+         const phi::DenseTensor* ln2_bias,
+         phi::DenseTensor* out,
+         phi::DenseTensor* dropout1_mask,
+         phi::DenseTensor* dropout2_mask,
+         phi::DenseTensor* ln1_mean,
+         phi::DenseTensor* ln1_variance,
+         phi::DenseTensor* ln2_mean,
+         phi::DenseTensor* ln2_variance,
+         phi::DenseTensor* linear1_out,
+         phi::DenseTensor* ln1_out,
+         phi::DenseTensor* dropout1_out,
+         phi::DenseTensor* dropout2_out,
+         const int bsz_seq,
+         const int d_model,
+         const int dim_feedforward,
+         const std::string& act_method,
+         const bool pre_layer_norm,
+         const float epsilon1,
+         const float epsilon2,
+         const bool add_residual,
+         const int ring_id,
+         const phi::fusion::DropoutParam& dropout_param1,
+         const phi::fusion::DropoutParam& dropout_param2) {
+  phi::fusion::FusedDropoutLayerNormHelper<T, uint8_t> pre_layernorm_helper(
+      bsz_seq, d_model, epsilon1);
+  phi::fusion::FusedDropoutHelper<T, uint8_t> fused_act_dropout_helper(
+      dev_ctx, bsz_seq, dim_feedforward, dropout_param1);
+  phi::fusion::FusedDropoutLayerNormHelper<T, uint8_t>
+      fused_dropout_layernorm_helper(
+          dev_ctx, bsz_seq, d_model, dropout_param2, epsilon2);
+
+  using U = phi::funcs::LayerNormParamType<T>;
+  const phi::DenseTensor* in = &x;
+
+  const U* ln1_scale_ptr =
+      ln1_scale == nullptr ? nullptr : ln1_scale->data<U>();
+  const U* ln1_bias_ptr = ln1_bias == nullptr ? nullptr : ln1_bias->data<U>();
+  const U* ln2_scale_ptr =
+      ln2_scale == nullptr ? nullptr : ln2_scale->data<U>();
+  const U* ln2_bias_ptr = ln2_bias == nullptr ? nullptr : ln2_bias->data<U>();
+  const T* linear1_bias_ptr =
+      linear1_bias == nullptr ? nullptr : linear1_bias->data<T>();
+  const T* linear2_bias_ptr =
+      linear2_bias == nullptr ? nullptr : linear2_bias->data<T>();
+
+  if (pre_layer_norm) {
+    pre_layernorm_helper.LayerNorm(dev_ctx,
+                                   x.data<T>(),
+                                   ln1_scale_ptr,
+                                   ln1_bias_ptr,
+                                   ln1_out->data<T>(),
+                                   ln1_mean->data<U>(),
+                                   ln1_variance->data<U>());
+    in = ln1_out;
+  }
+  MatMul<T, Context>(dev_ctx, *in, linear1_weight, linear1_out);
+  fused_act_dropout_helper.DropoutActBias(dev_ctx,
+                                          linear1_out->data<T>(),
+                                          linear1_bias_ptr,
+                                          act_method,
+                                          dropout1_out->data<T>(),
+                                          dropout1_mask->data<uint8_t>());
+  phi::DenseTensor linear2_out;
+  linear2_out.Resize({bsz_seq, d_model});
+  dev_ctx.template Alloc<T>(&linear2_out, linear2_out.numel() * sizeof(T));
+  MatMul<T, Context>(dev_ctx, *dropout1_out, linear2_weight, &linear2_out);
+
+  // tensor model parallel
+  phi::fusion::AllReduce<T>(linear2_out, ring_id, dev_ctx);
+
+  const T* residual_ptr = add_residual ? x.data<T>() : nullptr;
+  if (!pre_layer_norm) {
+    // TODO(Xreki): support post layer_norm case when add_residual is false.
+    PADDLE_ENFORCE_EQ(add_residual,
+                      true,
+                      phi::errors::InvalidArgument(
+                          "Attribute add_residual is expected to be true "
+                          "when pre_layer_norm is false."));
+
+    fused_dropout_layernorm_helper.LayernormResidualDropoutBias(
+        dev_ctx,
+        linear2_out.data<T>(),
+        residual_ptr,
+        linear2_bias_ptr,
+        ln2_scale_ptr,
+        ln2_bias_ptr,
+        dropout2_out->data<T>(),
+        dropout2_mask->data<uint8_t>(),
+        out->data<T>(),
+        ln2_mean->data<U>(),
+        ln2_variance->data<U>());
   } else {
-    auto dtype = platform::ToNCCLDataType(
-        framework::TransToProtoVarType(tensor.dtype()));
-    int64_t numel = tensor.numel();
-    const void* sendbuff = tensor.data<T>();
-    auto place = ctx.GetPlace();
-    void* recvbuff = ctx.Alloc<T>(&tensor, tensor.numel() * sizeof(T));
-    auto comm = platform::NCCLCommContext::Instance().Get(ring_id, place);
-    auto stream = ctx.stream();
-    PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::ncclAllReduce(
-        sendbuff, recvbuff, numel, dtype, ncclSum, comm->comm(), stream));
+    fused_dropout_layernorm_helper.ResidualDropoutBias(
+        dev_ctx,
+        linear2_out.data<T>(),
+        residual_ptr,
+        linear2_bias_ptr,
+        out->data<T>(),
+        dropout2_mask->data<uint8_t>());
   }
-#else
-  PADDLE_THROW(platform::errors::Unimplemented(
-      "PaddlePaddle should compile with NCCL or RCCL when used tensor model "
-      "parallel op."));
-#endif
 }
 
-template <typename T, typename DeviceContext>
-class FusedFeedForwardKernel : public framework::OpKernel<T> {
- public:
-  void MatMul(const phi::GPUContext& ctx,
-              const phi::DenseTensor& a,
-              const phi::DenseTensor& b,
-              phi::DenseTensor* c) const {
-    auto blas = phi::funcs::GetBlas<DeviceContext, T>(ctx);
-    auto a_2d = FoldInitDims(a);
-    auto b_2d = FoldInitDims(b);
-    auto mat_dim_a = phi::funcs::CreateMatrixDescriptor(a_2d.dims(), 0, false);
-    auto mat_dim_b = phi::funcs::CreateMatrixDescriptor(b_2d.dims(), 0, false);
-    T alpha = static_cast<T>(1.0);
-    blas.MatMul(a, mat_dim_a, b, mat_dim_b, alpha, c, T(0));
+template <typename T, typename Context>
+void FusedFeedForwardKernel(const Context& dev_ctx,
+                            const DenseTensor& x,
+                            const paddle::optional<DenseTensor>& dropout1_seed,
+                            const paddle::optional<DenseTensor>& dropout2_seed,
+                            const DenseTensor& linear1_weight,
+                            const paddle::optional<DenseTensor>& linear1_bias,
+                            const DenseTensor& linear2_weight,
+                            const paddle::optional<DenseTensor>& linear2_bias,
+                            const paddle::optional<DenseTensor>& ln1_scale,
+                            const paddle::optional<DenseTensor>& ln1_bias,
+                            const paddle::optional<DenseTensor>& ln2_scale,
+                            const paddle::optional<DenseTensor>& ln2_bias,
+                            bool pre_layer_norm,
+                            float ln1_epsilon,
+                            float ln2_epsilon,
+                            const std::string& act_method,
+                            float dropout1_prob,
+                            float dropout2_prob,
+                            const std::string& dropout1_implementation,
+                            const std::string& dropout2_implementation,
+                            bool is_test,
+                            bool dropout1_fix_seed,
+                            bool dropout2_fix_seed,
+                            int dropout1_seed_val,
+                            int dropout2_seed_val,
+                            bool add_residual,
+                            int ring_id,
+                            DenseTensor* out,
+                            DenseTensor* dropout1_mask,
+                            DenseTensor* dropout2_mask,
+                            DenseTensor* ln1_mean,
+                            DenseTensor* ln1_variance,
+                            DenseTensor* ln2_mean,
+                            DenseTensor* ln2_variance,
+                            DenseTensor* linear1_out,
+                            DenseTensor* ln1_out,
+                            DenseTensor* dropout1_out,
+                            DenseTensor* dropout2_out) {
+  auto* x_ptr = &x;
+  auto* linear1_weight_ptr = &linear1_weight;
+  auto* linear1_bias_ptr = linear1_bias.get_ptr();
+  auto* linear2_weight_ptr = &linear2_weight;
+  auto* linear2_bias_ptr = linear2_bias.get_ptr();
+
+  auto* ln1_scale_ptr = pre_layer_norm ? ln1_scale.get_ptr() : nullptr;
+  auto* ln1_bias_ptr = pre_layer_norm ? ln1_bias.get_ptr() : nullptr;
+  auto* ln2_scale_ptr = !pre_layer_norm ? ln2_scale.get_ptr() : nullptr;
+  auto* ln2_bias_ptr = !pre_layer_norm ? ln2_bias.get_ptr() : nullptr;
+
+  if (!pre_layer_norm) {
+    ln1_mean = nullptr;
+    ln1_variance = nullptr;
+    ln1_out = nullptr;
+  } else {
+    ln2_mean = nullptr;
+    ln2_variance = nullptr;
   }
 
-  void FFN(const phi::GPUContext& ctx,
-           const phi::DenseTensor& x,
-           const phi::DenseTensor& linear1_weight,
-           const phi::DenseTensor* linear1_bias,
-           const phi::DenseTensor& linear2_weight,
-           const phi::DenseTensor* linear2_bias,
-           const phi::DenseTensor* ln1_scale,
-           const phi::DenseTensor* ln1_bias,
-           const phi::DenseTensor* ln2_scale,
-           const phi::DenseTensor* ln2_bias,
-           phi::DenseTensor* out,
-           phi::DenseTensor* dropout1_mask,
-           phi::DenseTensor* dropout2_mask,
-           phi::DenseTensor* ln1_mean,
-           phi::DenseTensor* ln1_variance,
-           phi::DenseTensor* ln2_mean,
-           phi::DenseTensor* ln2_variance,
-           phi::DenseTensor* linear1_out,
-           phi::DenseTensor* ln1_out,
-           phi::DenseTensor* dropout1_out,
-           phi::DenseTensor* dropout2_out,
-           const int bsz_seq,
-           const int d_model,
-           const int dim_feedforward,
-           const std::string& act_method,
-           const bool pre_layer_norm,
-           const float epsilon1,
-           const float epsilon2,
-           const bool add_residual,
-           const int ring_id,
-           const DropoutParam& dropout_param1,
-           const DropoutParam& dropout_param2) const {
-    FusedDropoutLayerNormHelper<T, uint8_t> pre_layernorm_helper(
-        bsz_seq, d_model, epsilon1);
-    FusedDropoutHelper<T, uint8_t> fused_act_dropout_helper(
-        ctx, bsz_seq, dim_feedforward, dropout_param1);
-    FusedDropoutLayerNormHelper<T, uint8_t> fused_dropout_layernorm_helper(
-        ctx, bsz_seq, d_model, dropout_param2, epsilon2);
-
-    using U = phi::funcs::LayerNormParamType<T>;
-    const phi::DenseTensor* in = &x;
-
-    const U* ln1_scale_ptr =
-        ln1_scale == nullptr ? nullptr : ln1_scale->data<U>();
-    const U* ln1_bias_ptr = ln1_bias == nullptr ? nullptr : ln1_bias->data<U>();
-    const U* ln2_scale_ptr =
-        ln2_scale == nullptr ? nullptr : ln2_scale->data<U>();
-    const U* ln2_bias_ptr = ln2_bias == nullptr ? nullptr : ln2_bias->data<U>();
-    const T* linear1_bias_ptr =
-        linear1_bias == nullptr ? nullptr : linear1_bias->data<T>();
-    const T* linear2_bias_ptr =
-        linear2_bias == nullptr ? nullptr : linear2_bias->data<T>();
-
-    if (pre_layer_norm) {
-      pre_layernorm_helper.LayerNorm(ctx,
-                                     x.data<T>(),
-                                     ln1_scale_ptr,
-                                     ln1_bias_ptr,
-                                     ln1_out->data<T>(),
-                                     ln1_mean->data<U>(),
-                                     ln1_variance->data<U>());
-      in = ln1_out;
-    }
-    MatMul(ctx, *in, linear1_weight, linear1_out);
-    fused_act_dropout_helper.DropoutActBias(ctx,
-                                            linear1_out->data<T>(),
-                                            linear1_bias_ptr,
-                                            act_method,
-                                            dropout1_out->data<T>(),
-                                            dropout1_mask->data<uint8_t>());
-    phi::DenseTensor linear2_out;
-    linear2_out.Resize({bsz_seq, d_model});
-    ctx.Alloc<T>(&linear2_out, linear2_out.numel() * sizeof(T));
-    MatMul(ctx, *dropout1_out, linear2_weight, &linear2_out);
+  bool is_upscale_in_train1 = dropout1_implementation == "upscale_in_train";
+  bool is_upscale_in_train2 = dropout2_implementation == "upscale_in_train";
+  auto* dropout1_seed_ptr = dropout1_seed.get_ptr();
+  auto* dropout2_seed_ptr = dropout2_seed.get_ptr();
+
+  phi::fusion::DropoutParam dropout_param1(dropout1_fix_seed,
+                                           0,
+                                           is_test,
+                                           is_upscale_in_train1,
+                                           dropout1_prob,
+                                           dropout1_seed_ptr,
+                                           dropout1_seed_val);
+  phi::fusion::DropoutParam dropout_param2(dropout2_fix_seed,
+                                           0,
+                                           is_test,
+                                           is_upscale_in_train2,
+                                           dropout2_prob,
+                                           dropout2_seed_ptr,
+                                           dropout2_seed_val);
+
+  using U = phi::funcs::LayerNormParamType<T>;
+  dev_ctx.template Alloc<T>(out, out->numel() * sizeof(T));
+  dev_ctx.template Alloc<uint8_t>(dropout1_mask,
+                                  dropout1_mask->numel() * sizeof(uint8_t));
+  dev_ctx.template Alloc<uint8_t>(dropout2_mask,
+                                  dropout2_mask->numel() * sizeof(uint8_t));
+  if (pre_layer_norm) {
+    dev_ctx.template Alloc<U>(ln1_mean, ln1_mean->numel() * sizeof(U));
+    dev_ctx.template Alloc<U>(ln1_variance, ln1_variance->numel() * sizeof(U));
+    dev_ctx.template Alloc<T>(ln1_out, ln1_out->numel() * sizeof(T));
+  } else {
+    dev_ctx.template Alloc<U>(ln2_mean, ln2_mean->numel() * sizeof(U));
+    dev_ctx.template Alloc<U>(ln2_variance, ln2_variance->numel() * sizeof(U));
+  }
 
-    // tensor model parallel
-    AllReduce<T>(linear2_out, ring_id, ctx);
-
-    const T* residual_ptr = add_residual ? x.data<T>() : nullptr;
-    if (!pre_layer_norm) {
-      // TODO(Xreki): support post layer_norm case when add_residual is false.
-      PADDLE_ENFORCE_EQ(add_residual,
-                        true,
-                        platform::errors::InvalidArgument(
-                            "Attribute add_residual is expected to be true "
-                            "when pre_layer_norm is false."));
-
-      fused_dropout_layernorm_helper.LayernormResidualDropoutBias(
-          ctx,
-          linear2_out.data<T>(),
-          residual_ptr,
-          linear2_bias_ptr,
-          ln2_scale_ptr,
-          ln2_bias_ptr,
-          dropout2_out->data<T>(),
-          dropout2_mask->data<uint8_t>(),
-          out->data<T>(),
-          ln2_mean->data<U>(),
-          ln2_variance->data<U>());
-    } else {
-      fused_dropout_layernorm_helper.ResidualDropoutBias(
-          ctx,
-          linear2_out.data<T>(),
-          residual_ptr,
-          linear2_bias_ptr,
-          out->data<T>(),
-          dropout2_mask->data<uint8_t>());
-    }
+  dev_ctx.template Alloc<T>(linear1_out, linear1_out->numel() * sizeof(T));
+  dev_ctx.template Alloc<T>(dropout1_out, dropout1_out->numel() * sizeof(T));
+  dev_ctx.template Alloc<T>(dropout2_out, dropout2_out->numel() * sizeof(T));
+
+  auto x_dim = x_ptr->dims();
+  auto mat_dim_x = phi::funcs::CreateMatrixDescriptor(
+      phi::RowMatrixFromVector(x_dim), 0, false);
+
+  auto dim = linear1_weight_ptr->dims();
+  int d_model = dim[0];
+  int dim_feedforward = dim[dim.size() - 1];
+  int bsz_seq = mat_dim_x.batch_size_ * mat_dim_x.height_;
+
+  phi::fusion::FFN<T, Context>(dev_ctx,
+                               x,
+                               linear1_weight,
+                               linear1_bias_ptr,
+                               linear2_weight,
+                               linear2_bias_ptr,
+                               ln1_scale_ptr,
+                               ln1_bias_ptr,
+                               ln2_scale_ptr,
+                               ln2_bias_ptr,
+                               out,
+                               dropout1_mask,
+                               dropout2_mask,
+                               ln1_mean,
+                               ln1_variance,
+                               ln2_mean,
+                               ln2_variance,
+                               linear1_out,
+                               ln1_out,
+                               dropout1_out,
+                               dropout2_out,
+                               bsz_seq,
+                               d_model,
+                               dim_feedforward,
+                               act_method,
+                               pre_layer_norm,
+                               ln1_epsilon,
+                               ln2_epsilon,
+                               add_residual,
+                               ring_id,
+                               dropout_param1,
+                               dropout_param2);
+}
+
+template <typename T, typename Context>
+void MatMulGrad(const phi::GPUContext& dev_ctx,
+                const phi::DenseTensor& d_out,
+                const phi::DenseTensor& a,
+                const phi::DenseTensor& b,
+                phi::DenseTensor* d_a,
+                phi::DenseTensor* d_b) {
+  auto blas = phi::funcs::GetBlas<Context, T>(dev_ctx);
+  auto a_2d = phi::FoldInitDims(a);
+  auto b_2d = phi::FoldInitDims(b);
+  auto mat_dim_a = phi::funcs::CreateMatrixDescriptor(a_2d.dims(), 0, true);
+  auto mat_dim_b = phi::funcs::CreateMatrixDescriptor(b_2d.dims(), 0, true);
+  auto mat_dim_dout =
+      phi::funcs::CreateMatrixDescriptor(d_out.dims(), 0, false);
+  T alpha = static_cast<T>(1.0);
+  blas.MatMul(d_out, mat_dim_dout, b, mat_dim_b, alpha, d_a, T(0));
+  blas.MatMul(a, mat_dim_a, d_out, mat_dim_dout, alpha, d_b, T(0));
+}
+
+template <typename T, typename Context>
+void FFNGrad(const phi::GPUContext& dev_ctx,
+             const phi::DenseTensor& d_out,
+             const phi::DenseTensor& x,
+             const phi::DenseTensor& dropout1_mask,
+             const phi::DenseTensor& dropout2_mask,
+             const phi::DenseTensor& linear1_out,
+             const phi::DenseTensor* ln1_out,
+             const phi::DenseTensor& dropout1_out,
+             const phi::DenseTensor* dropout2_out,
+             const phi::DenseTensor& linear1_weight,
+             const phi::DenseTensor* linear1_bias,
+             const phi::DenseTensor& linear2_weight,
+             const phi::DenseTensor* ln1_gamma,
+             const phi::DenseTensor* ln1_beta,
+             const phi::DenseTensor* ln1_mean,
+             const phi::DenseTensor* ln1_variance,
+             const phi::DenseTensor* ln2_gamma,
+             const phi::DenseTensor* ln2_beta,
+             const phi::DenseTensor* ln2_mean,
+             const phi::DenseTensor* ln2_variance,
+             phi::DenseTensor* d_x,
+             phi::DenseTensor* d_linear1_weight,
+             phi::DenseTensor* d_linear1_bias,
+             phi::DenseTensor* d_linear2_weight,
+             phi::DenseTensor* d_linear2_bias,
+             phi::DenseTensor* d_ln1_gamma,
+             phi::DenseTensor* d_ln1_beta,
+             phi::DenseTensor* d_ln2_gamma,
+             phi::DenseTensor* d_ln2_beta,
+             const int bsz_seq,
+             const int d_model,
+             const int dim_feedforward,
+             const phi::fusion::DropoutParam& dropout_param1,
+             const phi::fusion::DropoutParam& dropout_param2,
+             const std::string& act_method,
+             const bool pre_layer_norm,
+             const float epsilon1,
+             const float epsilon2,
+             const bool add_residual,
+             const int ring_id) {
+  phi::fusion::FusedDropoutLayerNormHelper<T, uint8_t> pre_layernorm_helper(
+      bsz_seq, d_model, epsilon1);
+  phi::fusion::FusedDropoutHelper<T, uint8_t> fused_act_dropout_helper(
+      dev_ctx, bsz_seq, dim_feedforward, dropout_param1);
+  phi::fusion::FusedDropoutLayerNormHelper<T, uint8_t>
+      fused_dropout_layernorm_helper(
+          dev_ctx, bsz_seq, d_model, dropout_param2, epsilon2);
+
+  using U = phi::funcs::LayerNormParamType<T>;
+  const U* ln1_gamma_ptr =
+      ln1_gamma == nullptr ? nullptr : ln1_gamma->data<U>();
+  const U* ln1_beta_ptr = ln1_beta == nullptr ? nullptr : ln1_beta->data<U>();
+  const U* ln2_gamma_ptr =
+      ln2_gamma == nullptr ? nullptr : ln2_gamma->data<U>();
+  const U* ln2_beta_ptr = ln2_beta == nullptr ? nullptr : ln2_beta->data<U>();
+  const T* linear1_bias_ptr =
+      linear1_bias == nullptr ? nullptr : linear1_bias->data<T>();
+  T* d_linear1_bias_ptr =
+      d_linear1_bias == nullptr ? nullptr : d_linear1_bias->data<T>();
+  T* d_linear2_bias_ptr =
+      d_linear2_bias == nullptr ? nullptr : d_linear2_bias->data<T>();
+  U* d_ln1_gamma_ptr =
+      d_ln1_gamma == nullptr ? nullptr : d_ln1_gamma->data<U>();
+  U* d_ln1_beta_ptr = d_ln1_beta == nullptr ? nullptr : d_ln1_beta->data<U>();
+  U* d_ln2_gamma_ptr =
+      d_ln2_gamma == nullptr ? nullptr : d_ln2_gamma->data<U>();
+  U* d_ln2_beta_ptr = d_ln2_beta == nullptr ? nullptr : d_ln2_beta->data<U>();
+
+  phi::DenseTensor d_linear2_out, d_dropout2_out, d_residual;
+  d_linear2_out.Resize({bsz_seq, d_model});
+  dev_ctx.template Alloc<T>(&d_linear2_out, d_linear2_out.numel() * sizeof(T));
+  d_dropout2_out.Resize({bsz_seq, d_model});
+  dev_ctx.template Alloc<T>(&d_dropout2_out,
+                            d_dropout2_out.numel() * sizeof(T));
+
+  T* d_residual_ptr = nullptr;
+  if (add_residual) {
+    d_residual.Resize(d_x->dims());
+    d_residual_ptr =
+        dev_ctx.template Alloc<T>(&d_residual, d_residual.numel() * sizeof(T));
+  }
+  if (pre_layer_norm) {
+    fused_dropout_layernorm_helper.ResidualDropoutBiasGrad(
+        dev_ctx,
+        d_out.data<T>(),
+        dropout2_mask.data<uint8_t>(),
+        d_linear2_out.data<T>(),
+        d_residual_ptr,
+        d_linear2_bias_ptr);
+  } else {
+    fused_dropout_layernorm_helper.LayernormResidualDropoutBiasGrad(
+        dev_ctx,
+        d_out.data<T>(),
+        dropout2_out->data<T>(),
+        dropout2_mask.data<uint8_t>(),
+        ln2_gamma_ptr,
+        ln2_mean->data<U>(),
+        ln2_variance->data<U>(),
+        d_dropout2_out.data<T>(),
+        d_ln2_gamma_ptr,
+        d_ln2_beta_ptr,
+        d_linear2_out.data<T>(),
+        d_linear2_bias_ptr,
+        d_residual_ptr);
   }
 
-  void Compute(const framework::ExecutionContext& context) const override {
-    auto* x = context.Input<phi::DenseTensor>("X");
-    auto* linear1_weight = context.Input<phi::DenseTensor>("Linear1Weight");
-    auto* linear1_bias = context.Input<phi::DenseTensor>("Linear1Bias");
-    auto* linear2_weight = context.Input<phi::DenseTensor>("Linear2Weight");
-    auto* linear2_bias = context.Input<phi::DenseTensor>("Linear2Bias");
-    const bool pre_layer_norm = context.Attr<bool>("pre_layer_norm");
-    auto& dev_ctx = context.template device_context<phi::GPUContext>();
-
-    auto* ln1_scale =
-        pre_layer_norm ? context.Input<phi::DenseTensor>("Ln1Scale") : nullptr;
-    auto* ln1_bias =
-        pre_layer_norm ? context.Input<phi::DenseTensor>("Ln1Bias") : nullptr;
-    auto* ln2_scale =
-        !pre_layer_norm ? context.Input<phi::DenseTensor>("Ln2Scale") : nullptr;
-    auto* ln2_bias =
-        !pre_layer_norm ? context.Input<phi::DenseTensor>("Ln2Bias") : nullptr;
-
-    auto* ln1_mean =
-        pre_layer_norm ? context.Output<phi::DenseTensor>("Ln1Mean") : nullptr;
-    auto* ln1_variance = pre_layer_norm
-                             ? context.Output<phi::DenseTensor>("Ln1Variance")
-                             : nullptr;
-    auto* ln2_mean =
-        !pre_layer_norm ? context.Output<phi::DenseTensor>("Ln2Mean") : nullptr;
-    auto* ln2_variance = !pre_layer_norm
-                             ? context.Output<phi::DenseTensor>("Ln2Variance")
-                             : nullptr;
-    auto* out = context.Output<phi::DenseTensor>("Out");
-    auto* dropout1_mask = context.Output<phi::DenseTensor>("Dropout1Mask");
-    auto* dropout2_mask = context.Output<phi::DenseTensor>("Dropout2Mask");
-    auto* linear1_out = context.Output<phi::DenseTensor>("Linear1Out");
-    auto* ln1_out =
-        pre_layer_norm ? context.Output<phi::DenseTensor>("Ln1Out") : nullptr;
-    auto* dropout1_out = context.Output<phi::DenseTensor>("Dropout1Out");
-    auto* dropout2_out = context.Output<phi::DenseTensor>("Dropout2Out");
-
-    const std::string act_method = context.Attr<std::string>("act_method");
-
-    const float epsilon1 = context.Attr<float>("ln1_epsilon");
-    const float epsilon2 = context.Attr<float>("ln2_epsilon");
-    const int ring_id = context.Attr<int>("ring_id");
-    const bool add_residual = context.Attr<bool>("add_residual");
-
-    DropoutParam dropout_param1(context, 1);
-    DropoutParam dropout_param2(context, 2);
-
-    using U = phi::funcs::LayerNormParamType<T>;
-    dev_ctx.Alloc<T>(out, out->numel() * sizeof(T));
-    dev_ctx.Alloc<uint8_t>(dropout1_mask,
-                           dropout1_mask->numel() * sizeof(uint8_t));
-    dev_ctx.Alloc<uint8_t>(dropout2_mask,
-                           dropout2_mask->numel() * sizeof(uint8_t));
-    if (pre_layer_norm) {
-      dev_ctx.Alloc<U>(ln1_mean, ln1_mean->numel() * sizeof(U));
-      dev_ctx.Alloc<U>(ln1_variance, ln1_variance->numel() * sizeof(U));
-      dev_ctx.Alloc<T>(ln1_out, ln1_out->numel() * sizeof(T));
-    } else {
-      dev_ctx.Alloc<U>(ln2_mean, ln2_mean->numel() * sizeof(U));
-      dev_ctx.Alloc<U>(ln2_variance, ln2_variance->numel() * sizeof(U));
-    }
-
-    dev_ctx.Alloc<T>(linear1_out, linear1_out->numel() * sizeof(T));
-    dev_ctx.Alloc<T>(dropout1_out, dropout1_out->numel() * sizeof(T));
-    dev_ctx.Alloc<T>(dropout2_out, dropout2_out->numel() * sizeof(T));
-
-    auto x_dim = x->dims();
-    auto mat_dim_x = phi::funcs::CreateMatrixDescriptor(
-        RowMatrixFromVector(x_dim), 0, false);
-
-    auto dim = linear1_weight->dims();
-    int d_model = dim[0];
-    int dim_feedforward = dim[dim.size() - 1];
-    int bsz_seq = mat_dim_x.batch_size_ * mat_dim_x.height_;
-
-    FFN(context.cuda_device_context(),
-        *x,
-        *linear1_weight,
-        linear1_bias,
-        *linear2_weight,
-        linear2_bias,
-        ln1_scale,
-        ln1_bias,
-        ln2_scale,
-        ln2_bias,
-        out,
-        dropout1_mask,
-        dropout2_mask,
-        ln1_mean,
-        ln1_variance,
-        ln2_mean,
-        ln2_variance,
-        linear1_out,
-        ln1_out,
-        dropout1_out,
-        dropout2_out,
-        bsz_seq,
-        d_model,
-        dim_feedforward,
-        act_method,
-        pre_layer_norm,
-        epsilon1,
-        epsilon2,
-        add_residual,
-        ring_id,
-        dropout_param1,
-        dropout_param2);
+  phi::DenseTensor d_dropout1_out;
+  d_dropout1_out.Resize({bsz_seq, dim_feedforward});
+  dev_ctx.template Alloc<T>(&d_dropout1_out,
+                            d_dropout1_out.numel() * sizeof(T));
+  MatMulGrad<T, Context>(dev_ctx,
+                         d_linear2_out,
+                         dropout1_out,
+                         linear2_weight,
+                         &d_dropout1_out,
+                         d_linear2_weight);
+
+  phi::DenseTensor d_linear1_out;
+  d_linear1_out.Resize({bsz_seq, dim_feedforward});
+  dev_ctx.template Alloc<T>(&d_linear1_out, d_linear1_out.numel() * sizeof(T));
+  fused_act_dropout_helper.DropoutActBiasGrad(dev_ctx,
+                                              d_dropout1_out.data<T>(),
+                                              linear1_out.data<T>(),
+                                              linear1_bias_ptr,
+                                              dropout1_mask.data<uint8_t>(),
+                                              d_linear1_out.data<T>(),
+                                              d_linear1_bias_ptr,
+                                              act_method);
+
+  if (pre_layer_norm) {
+    phi::DenseTensor d_ln1_out;
+    d_ln1_out.Resize({bsz_seq, d_model});
+    dev_ctx.template Alloc<T>(&d_ln1_out, d_ln1_out.numel() * sizeof(T));
+    MatMulGrad<T, Context>(dev_ctx,
+                           d_linear1_out,
+                           *ln1_out,
+                           linear1_weight,
+                           &d_ln1_out,
+                           d_linear1_weight);
+    // tensor model parallel
+    phi::fusion::AllReduce<T>(d_ln1_out, ring_id, dev_ctx);
+    pre_layernorm_helper.LayerNormGrad(dev_ctx,
+                                       d_ln1_out.data<T>(),
+                                       x.data<T>(),
+                                       ln1_gamma_ptr,
+                                       ln1_mean->data<U>(),
+                                       ln1_variance->data<U>(),
+                                       d_x->data<T>(),
+                                       d_ln1_gamma_ptr,
+                                       d_ln1_beta_ptr);
+  } else {
+    MatMulGrad<T, Context>(
+        dev_ctx, d_linear1_out, x, linear1_weight, d_x, d_linear1_weight);
+    // tensor model parallel
+    phi::fusion::AllReduce<T>(*d_x, ring_id, dev_ctx);
   }
-};
-
-template <typename T, typename DeviceContext>
-class FusedFeedForwardGradKernel : public framework::OpKernel<T> {
- public:
-  void MatMulGrad(const phi::GPUContext& ctx,
-                  const phi::DenseTensor& d_out,
-                  const phi::DenseTensor& a,
-                  const phi::DenseTensor& b,
-                  phi::DenseTensor* d_a,
-                  phi::DenseTensor* d_b) const {
-    auto blas = phi::funcs::GetBlas<DeviceContext, T>(ctx);
-    auto a_2d = FoldInitDims(a);
-    auto b_2d = FoldInitDims(b);
-    auto mat_dim_a = phi::funcs::CreateMatrixDescriptor(a_2d.dims(), 0, true);
-    auto mat_dim_b = phi::funcs::CreateMatrixDescriptor(b_2d.dims(), 0, true);
-    auto mat_dim_dout =
-        phi::funcs::CreateMatrixDescriptor(d_out.dims(), 0, false);
-    T alpha = static_cast<T>(1.0);
-    blas.MatMul(d_out, mat_dim_dout, b, mat_dim_b, alpha, d_a, T(0));
-    blas.MatMul(a, mat_dim_a, d_out, mat_dim_dout, alpha, d_b, T(0));
+
+  if (add_residual) {
+    // gradient accumulation
+    std::vector<const phi::DenseTensor*> ins = {&d_residual, d_x};
+    std::vector<phi::DenseTensor*> outs = {d_x};
+    phi::funcs::ElementwiseKernel<T>(
+        dev_ctx, ins, &outs, phi::funcs::AddFunctor<T>());
   }
+}
 
-  void FFNGrad(const phi::GPUContext& ctx,
-               const phi::DenseTensor& d_out,
-               const phi::DenseTensor& x,
-               const phi::DenseTensor& dropout1_mask,
-               const phi::DenseTensor& dropout2_mask,
-               const phi::DenseTensor& linear1_out,
-               const phi::DenseTensor* ln1_out,
-               const phi::DenseTensor& dropout1_out,
-               const phi::DenseTensor* dropout2_out,
-               const phi::DenseTensor& linear1_weight,
-               const phi::DenseTensor* linear1_bias,
-               const phi::DenseTensor& linear2_weight,
-               const phi::DenseTensor* ln1_gamma,
-               const phi::DenseTensor* ln1_beta,
-               const phi::DenseTensor* ln1_mean,
-               const phi::DenseTensor* ln1_variance,
-               const phi::DenseTensor* ln2_gamma,
-               const phi::DenseTensor* ln2_beta,
-               const phi::DenseTensor* ln2_mean,
-               const phi::DenseTensor* ln2_variance,
-               phi::DenseTensor* d_x,
-               phi::DenseTensor* d_linear1_weight,
-               phi::DenseTensor* d_linear1_bias,
-               phi::DenseTensor* d_linear2_weight,
-               phi::DenseTensor* d_linear2_bias,
-               phi::DenseTensor* d_ln1_gamma,
-               phi::DenseTensor* d_ln1_beta,
-               phi::DenseTensor* d_ln2_gamma,
-               phi::DenseTensor* d_ln2_beta,
-               const int bsz_seq,
-               const int d_model,
-               const int dim_feedforward,
-               const DropoutParam& dropout_param1,
-               const DropoutParam& dropout_param2,
-               const std::string& act_method,
-               const bool pre_layer_norm,
-               const float epsilon1,
-               const float epsilon2,
-               const bool add_residual,
-               const int ring_id) const {
-    FusedDropoutLayerNormHelper<T, uint8_t> pre_layernorm_helper(
-        bsz_seq, d_model, epsilon1);
-    FusedDropoutHelper<T, uint8_t> fused_act_dropout_helper(
-        ctx, bsz_seq, dim_feedforward, dropout_param1);
-    FusedDropoutLayerNormHelper<T, uint8_t> fused_dropout_layernorm_helper(
-        ctx, bsz_seq, d_model, dropout_param2, epsilon2);
-
-    using U = phi::funcs::LayerNormParamType<T>;
-    const U* ln1_gamma_ptr =
-        ln1_gamma == nullptr ? nullptr : ln1_gamma->data<U>();
-    const U* ln1_beta_ptr = ln1_beta == nullptr ? nullptr : ln1_beta->data<U>();
-    const U* ln2_gamma_ptr =
-        ln2_gamma == nullptr ? nullptr : ln2_gamma->data<U>();
-    const U* ln2_beta_ptr = ln2_beta == nullptr ? nullptr : ln2_beta->data<U>();
-    const T* linear1_bias_ptr =
-        linear1_bias == nullptr ? nullptr : linear1_bias->data<T>();
-    T* d_linear1_bias_ptr =
-        d_linear1_bias == nullptr ? nullptr : d_linear1_bias->data<T>();
-    T* d_linear2_bias_ptr =
-        d_linear2_bias == nullptr ? nullptr : d_linear2_bias->data<T>();
-    U* d_ln1_gamma_ptr =
-        d_ln1_gamma == nullptr ? nullptr : d_ln1_gamma->data<U>();
-    U* d_ln1_beta_ptr = d_ln1_beta == nullptr ? nullptr : d_ln1_beta->data<U>();
-    U* d_ln2_gamma_ptr =
-        d_ln2_gamma == nullptr ? nullptr : d_ln2_gamma->data<U>();
-    U* d_ln2_beta_ptr = d_ln2_beta == nullptr ? nullptr : d_ln2_beta->data<U>();
-
-    phi::DenseTensor d_linear2_out, d_dropout2_out, d_residual;
-    d_linear2_out.Resize({bsz_seq, d_model});
-    ctx.Alloc<T>(&d_linear2_out, d_linear2_out.numel() * sizeof(T));
-    d_dropout2_out.Resize({bsz_seq, d_model});
-    ctx.Alloc<T>(&d_dropout2_out, d_dropout2_out.numel() * sizeof(T));
-
-    T* d_residual_ptr = nullptr;
-    if (add_residual) {
-      d_residual.Resize(d_x->dims());
-      d_residual_ptr =
-          ctx.Alloc<T>(&d_residual, d_residual.numel() * sizeof(T));
-    }
-    if (pre_layer_norm) {
-      fused_dropout_layernorm_helper.ResidualDropoutBiasGrad(
-          ctx,
-          d_out.data<T>(),
-          dropout2_mask.data<uint8_t>(),
-          d_linear2_out.data<T>(),
-          d_residual_ptr,
-          d_linear2_bias_ptr);
-    } else {
-      fused_dropout_layernorm_helper.LayernormResidualDropoutBiasGrad(
-          ctx,
-          d_out.data<T>(),
-          dropout2_out->data<T>(),
-          dropout2_mask.data<uint8_t>(),
-          ln2_gamma_ptr,
-          ln2_mean->data<U>(),
-          ln2_variance->data<U>(),
-          d_dropout2_out.data<T>(),
-          d_ln2_gamma_ptr,
-          d_ln2_beta_ptr,
-          d_linear2_out.data<T>(),
-          d_linear2_bias_ptr,
-          d_residual_ptr);
-    }
-
-    phi::DenseTensor d_dropout1_out;
-    d_dropout1_out.Resize({bsz_seq, dim_feedforward});
-    ctx.Alloc<T>(&d_dropout1_out, d_dropout1_out.numel() * sizeof(T));
-    MatMulGrad(ctx,
-               d_linear2_out,
-               dropout1_out,
-               linear2_weight,
-               &d_dropout1_out,
-               d_linear2_weight);
-
-    phi::DenseTensor d_linear1_out;
-    d_linear1_out.Resize({bsz_seq, dim_feedforward});
-    ctx.Alloc<T>(&d_linear1_out, d_linear1_out.numel() * sizeof(T));
-    fused_act_dropout_helper.DropoutActBiasGrad(ctx,
-                                                d_dropout1_out.data<T>(),
-                                                linear1_out.data<T>(),
-                                                linear1_bias_ptr,
-                                                dropout1_mask.data<uint8_t>(),
-                                                d_linear1_out.data<T>(),
-                                                d_linear1_bias_ptr,
-                                                act_method);
-
-    if (pre_layer_norm) {
-      phi::DenseTensor d_ln1_out;
-      d_ln1_out.Resize({bsz_seq, d_model});
-      ctx.Alloc<T>(&d_ln1_out, d_ln1_out.numel() * sizeof(T));
-      MatMulGrad(ctx,
-                 d_linear1_out,
-                 *ln1_out,
-                 linear1_weight,
-                 &d_ln1_out,
-                 d_linear1_weight);
-      // tensor model parallel
-      AllReduce<T>(d_ln1_out, ring_id, ctx);
-      pre_layernorm_helper.LayerNormGrad(ctx,
-                                         d_ln1_out.data<T>(),
-                                         x.data<T>(),
-                                         ln1_gamma_ptr,
-                                         ln1_mean->data<U>(),
-                                         ln1_variance->data<U>(),
-                                         d_x->data<T>(),
-                                         d_ln1_gamma_ptr,
-                                         d_ln1_beta_ptr);
-    } else {
-      MatMulGrad(ctx, d_linear1_out, x, linear1_weight, d_x, d_linear1_weight);
-      // tensor model parallel
-      AllReduce<T>(*d_x, ring_id, ctx);
-    }
-
-    if (add_residual) {
-      // gradient accumulation
-      std::vector<const phi::DenseTensor*> ins = {&d_residual, d_x};
-      std::vector<phi::DenseTensor*> outs = {d_x};
-      phi::funcs::ElementwiseKernel<T>(
-          ctx, ins, &outs, phi::funcs::AddFunctor<T>());
-    }
+template <typename T, typename Context>
+void FusedFeedForwardGradKernel(
+    const Context& dev_ctx,
+    const DenseTensor& out_grad,
+    const DenseTensor& x,
+    const DenseTensor& linear1_weight,
+    const paddle::optional<DenseTensor>& linear1_bias,
+    const DenseTensor& linear2_weight,
+    const DenseTensor& dropout1_mask,
+    const DenseTensor& dropout2_mask,
+    const DenseTensor& linear1_out,
+    const DenseTensor& dropout1_out,
+    const paddle::optional<DenseTensor>& dropout2_out,
+    const paddle::optional<DenseTensor>& ln1_scale,
+    const paddle::optional<DenseTensor>& ln1_bias,
+    const paddle::optional<DenseTensor>& ln1_out,
+    const paddle::optional<DenseTensor>& ln1_mean,
+    const paddle::optional<DenseTensor>& ln1_variance,
+    const paddle::optional<DenseTensor>& ln2_scale,
+    const paddle::optional<DenseTensor>& ln2_bias,
+    const paddle::optional<DenseTensor>& ln2_mean,
+    const paddle::optional<DenseTensor>& ln2_variance,
+    const paddle::optional<DenseTensor>& linear2_bias,
+    bool pre_layer_norm,
+    float ln1_epsilon,
+    float ln2_epsilon,
+    const std::string& act_method,
+    float dropout1_prob,
+    float dropout2_prob,
+    const std::string& dropout1_implementation,
+    const std::string& dropout2_implementation,
+    bool is_test,
+    bool dropout1_fix_seed,
+    bool dropout2_fix_seed,
+    int dropout1_seed_val,
+    int dropout2_seed_val,
+    bool add_residual,
+    int ring_id,
+    DenseTensor* x_grad,
+    DenseTensor* ln1_scale_grad,
+    DenseTensor* ln1_bias_grad,
+    DenseTensor* ln2_scale_grad,
+    DenseTensor* ln2_bias_grad,
+    DenseTensor* linear1_weight_grad,
+    DenseTensor* linear1_bias_grad,
+    DenseTensor* linear2_weight_grad,
+    DenseTensor* linear2_bias_grad) {
+  using U = phi::funcs::LayerNormParamType<T>;
+
+  auto* ln1_out_ptr = pre_layer_norm ? ln1_out.get_ptr() : nullptr;
+  auto* dropout2_out_ptr = dropout2_out.get_ptr();
+  auto* linear1_bias_ptr = linear1_bias.get_ptr();
+  auto* ln1_mean_ptr = pre_layer_norm ? ln1_mean.get_ptr() : nullptr;
+  auto* ln1_variance_ptr = pre_layer_norm ? ln1_variance.get_ptr() : nullptr;
+  auto* ln1_scale_ptr = pre_layer_norm ? ln1_scale.get_ptr() : nullptr;
+  auto* ln1_bias_ptr = pre_layer_norm ? ln1_bias.get_ptr() : nullptr;
+  auto* ln2_mean_ptr = !pre_layer_norm ? ln2_mean.get_ptr() : nullptr;
+  auto* ln2_variance_ptr = !pre_layer_norm ? ln2_variance.get_ptr() : nullptr;
+  auto* ln2_scale_ptr = !pre_layer_norm ? ln2_scale.get_ptr() : nullptr;
+  auto* ln2_bias_ptr = !pre_layer_norm ? ln2_bias.get_ptr() : nullptr;
+
+  auto* d_x = x_grad;
+  auto* d_ln1_scale = pre_layer_norm ? ln1_scale_grad : nullptr;
+  auto* d_ln1_bias = pre_layer_norm ? ln1_bias_grad : nullptr;
+  auto* d_ln2_scale = pre_layer_norm ? nullptr : ln2_scale_grad;
+  auto* d_ln2_bias = pre_layer_norm ? nullptr : ln2_bias_grad;
+  auto* d_linear1_weight = linear1_weight_grad;
+  auto* d_linear1_bias = linear1_bias_grad;
+  auto* d_linear2_weight = linear2_weight_grad;
+  auto* d_linear2_bias = linear2_bias_grad;
+
+  bool is_upscale_in_train1 = dropout1_implementation == "upscale_in_train";
+  bool is_upscale_in_train2 = dropout2_implementation == "upscale_in_train";
+
+  phi::fusion::DropoutParam dropout_param1(dropout1_fix_seed,
+                                           0,
+                                           is_test,
+                                           is_upscale_in_train1,
+                                           dropout1_prob,
+                                           nullptr,
+                                           dropout1_seed_val);
+  phi::fusion::DropoutParam dropout_param2(dropout2_fix_seed,
+                                           0,
+                                           is_test,
+                                           is_upscale_in_train2,
+                                           dropout2_prob,
+                                           nullptr,
+                                           dropout2_seed_val);
+
+  dev_ctx.template Alloc<T>(d_x, d_x->numel() * sizeof(T));
+  if (d_ln1_scale) {
+    dev_ctx.template Alloc<U>(d_ln1_scale, d_ln1_scale->numel() * sizeof(U));
+  }
+  if (d_ln1_bias) {
+    dev_ctx.template Alloc<U>(d_ln1_bias, d_ln1_bias->numel() * sizeof(U));
+  }
+  if (d_ln2_scale) {
+    dev_ctx.template Alloc<U>(d_ln2_scale, d_ln2_scale->numel() * sizeof(U));
+  }
+  if (d_ln2_bias) {
+    dev_ctx.template Alloc<U>(d_ln2_bias, d_ln2_bias->numel() * sizeof(U));
   }
+  if (d_linear1_bias) {
+    dev_ctx.template Alloc<T>(d_linear1_bias,
+                              d_linear1_bias->numel() * sizeof(T));
+  }
+  if (d_linear2_bias) {
+    dev_ctx.template Alloc<T>(d_linear2_bias,
+                              d_linear2_bias->numel() * sizeof(T));
+  }
+  dev_ctx.template Alloc<T>(d_linear1_weight,
+                            d_linear1_weight->numel() * sizeof(T));
+  dev_ctx.template Alloc<T>(d_linear2_weight,
+                            d_linear2_weight->numel() * sizeof(T));
+
+  auto x_dim = x.dims();
+  auto mat_dim_x = phi::funcs::CreateMatrixDescriptor(
+      phi::RowMatrixFromVector(x_dim), 0, false);
+
+  auto linear1_weight_dim = linear1_weight.dims();
+  int d_model = linear1_weight_dim[0];
+  int dim_feedforward = linear1_weight_dim[linear1_weight_dim.size() - 1];
+  int bsz_seq = mat_dim_x.batch_size_ * mat_dim_x.height_;
+
+  FFNGrad<T, Context>(dev_ctx,
+                      out_grad,
+                      x,
+                      dropout1_mask,
+                      dropout2_mask,
+                      linear1_out,
+                      ln1_out_ptr,
+                      dropout1_out,
+                      dropout2_out_ptr,
+                      linear1_weight,
+                      linear1_bias_ptr,
+                      linear2_weight,
+                      ln1_scale_ptr,
+                      ln1_bias_ptr,
+                      ln1_mean_ptr,
+                      ln1_variance_ptr,
+                      ln2_scale_ptr,
+                      ln2_bias_ptr,
+                      ln2_mean_ptr,
+                      ln2_variance_ptr,
+                      d_x,
+                      d_linear1_weight,
+                      d_linear1_bias,
+                      d_linear2_weight,
+                      d_linear2_bias,
+                      d_ln1_scale,
+                      d_ln1_bias,
+                      d_ln2_scale,
+                      d_ln2_bias,
+                      bsz_seq,
+                      d_model,
+                      dim_feedforward,
+                      dropout_param1,
+                      dropout_param2,
+                      act_method,
+                      pre_layer_norm,
+                      ln1_epsilon,
+                      ln2_epsilon,
+                      add_residual,
+                      ring_id);
+}
+}  // namespace fusion
+}  // namespace phi
+
+PD_REGISTER_KERNEL(fused_feedforward,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::fusion::FusedFeedForwardKernel,
+                   float,
+                   double,
+                   phi::dtype::float16) {
+  kernel->OutputAt(1).SetDataType(phi::DataType::UINT8);
+  kernel->OutputAt(2).SetDataType(phi::DataType::UINT8);
+  if (kernel_key.dtype() == phi::DataType::FLOAT16) {
+    kernel->OutputAt(3).SetDataType(phi::DataType::FLOAT32);
+    kernel->OutputAt(4).SetDataType(phi::DataType::FLOAT32);
+    kernel->OutputAt(5).SetDataType(phi::DataType::FLOAT32);
+    kernel->OutputAt(6).SetDataType(phi::DataType::FLOAT32);
+  }
+}
 
-  void Compute(const framework::ExecutionContext& context) const override {
-    using U = phi::funcs::LayerNormParamType<T>;
-    auto& dev_ctx = context.template device_context<phi::GPUContext>();
-    auto d_out =
-        *context.Input<phi::DenseTensor>(framework::GradVarName("Out"));
-    auto x = *context.Input<phi::DenseTensor>("X");
-    const bool pre_layer_norm = context.Attr<bool>("pre_layer_norm");
-    auto dropout1_mask = *context.Input<phi::DenseTensor>("Dropout1Mask");
-    auto dropout2_mask = *context.Input<phi::DenseTensor>("Dropout2Mask");
-    auto linear1_out = *context.Input<phi::DenseTensor>("Linear1Out");
-    auto* ln1_out =
-        pre_layer_norm ? context.Input<phi::DenseTensor>("Ln1Out") : nullptr;
-    auto dropout1_out = *context.Input<phi::DenseTensor>("Dropout1Out");
-    auto* dropout2_out = context.Input<phi::DenseTensor>("Dropout2Out");
-    auto linear1_weight = *context.Input<phi::DenseTensor>("Linear1Weight");
-    auto* linear1_bias = context.Input<phi::DenseTensor>("Linear1Bias");
-    auto linear2_weight = *context.Input<phi::DenseTensor>("Linear2Weight");
-    auto* ln1_mean =
-        pre_layer_norm ? context.Input<phi::DenseTensor>("Ln1Mean") : nullptr;
-    auto* ln1_variance = pre_layer_norm
-                             ? context.Input<phi::DenseTensor>("Ln1Variance")
-                             : nullptr;
-    auto* ln1_scale =
-        pre_layer_norm ? context.Input<phi::DenseTensor>("Ln1Scale") : nullptr;
-    auto* ln1_bias =
-        pre_layer_norm ? context.Input<phi::DenseTensor>("Ln1Bias") : nullptr;
-    auto* ln2_mean =
-        !pre_layer_norm ? context.Input<phi::DenseTensor>("Ln2Mean") : nullptr;
-    auto* ln2_variance = !pre_layer_norm
-                             ? context.Input<phi::DenseTensor>("Ln2Variance")
-                             : nullptr;
-    auto* ln2_scale =
-        !pre_layer_norm ? context.Input<phi::DenseTensor>("Ln2Scale") : nullptr;
-    auto* ln2_bias =
-        !pre_layer_norm ? context.Input<phi::DenseTensor>("Ln2Bias") : nullptr;
-
-    auto* d_x = context.Output<phi::DenseTensor>(framework::GradVarName("X"));
-    auto* d_ln1_scale = pre_layer_norm ? context.Output<phi::DenseTensor>(
-                                             framework::GradVarName("Ln1Scale"))
-                                       : nullptr;
-    auto* d_ln1_bias = pre_layer_norm ? context.Output<phi::DenseTensor>(
-                                            framework::GradVarName("Ln1Bias"))
-                                      : nullptr;
-    auto* d_ln2_scale = pre_layer_norm
-                            ? nullptr
-                            : context.Output<phi::DenseTensor>(
-                                  framework::GradVarName("Ln2Scale"));
-    auto* d_ln2_bias = pre_layer_norm ? nullptr
-                                      : context.Output<phi::DenseTensor>(
-                                            framework::GradVarName("Ln2Bias"));
-    auto* d_linear1_weight = context.Output<phi::DenseTensor>(
-        framework::GradVarName("Linear1Weight"));
-    auto* d_linear1_bias =
-        context.Output<phi::DenseTensor>(framework::GradVarName("Linear1Bias"));
-    auto* d_linear2_weight = context.Output<phi::DenseTensor>(
-        framework::GradVarName("Linear2Weight"));
-    auto* d_linear2_bias =
-        context.Output<phi::DenseTensor>(framework::GradVarName("Linear2Bias"));
-
-    const float epsilon1 = context.Attr<float>("ln1_epsilon");
-    const float epsilon2 = context.Attr<float>("ln2_epsilon");
-    const bool add_residual = context.Attr<bool>("add_residual");
-    const int ring_id = context.Attr<int>("ring_id");
-    const std::string act_method = context.Attr<std::string>("act_method");
-    DropoutParam dropout_param1(context, 1);
-    DropoutParam dropout_param2(context, 2);
-
-    dev_ctx.Alloc<T>(d_x, d_x->numel() * sizeof(T));
-    if (d_ln1_scale) {
-      dev_ctx.Alloc<U>(d_ln1_scale, d_ln1_scale->numel() * sizeof(U));
-    }
-    if (d_ln1_bias) {
-      dev_ctx.Alloc<U>(d_ln1_bias, d_ln1_bias->numel() * sizeof(U));
-    }
-    if (d_ln2_scale) {
-      dev_ctx.Alloc<U>(d_ln2_scale, d_ln2_scale->numel() * sizeof(U));
-    }
-    if (d_ln2_bias) {
-      dev_ctx.Alloc<U>(d_ln2_bias, d_ln2_bias->numel() * sizeof(U));
-    }
-    if (d_linear1_bias) {
-      dev_ctx.Alloc<T>(d_linear1_bias, d_linear1_bias->numel() * sizeof(T));
-    }
-    if (d_linear2_bias) {
-      dev_ctx.Alloc<T>(d_linear2_bias, d_linear2_bias->numel() * sizeof(T));
-    }
-    dev_ctx.Alloc<T>(d_linear1_weight, d_linear1_weight->numel() * sizeof(T));
-    dev_ctx.Alloc<T>(d_linear2_weight, d_linear2_weight->numel() * sizeof(T));
-
-    auto x_dim = x.dims();
-    auto mat_dim_x = phi::funcs::CreateMatrixDescriptor(
-        RowMatrixFromVector(x_dim), 0, false);
-
-    auto linear1_weight_dim = linear1_weight.dims();
-    int d_model = linear1_weight_dim[0];
-    int dim_feedforward = linear1_weight_dim[linear1_weight_dim.size() - 1];
-    int bsz_seq = mat_dim_x.batch_size_ * mat_dim_x.height_;
-
-    FFNGrad(context.cuda_device_context(),
-            d_out,
-            x,
-            dropout1_mask,
-            dropout2_mask,
-            linear1_out,
-            ln1_out,
-            dropout1_out,
-            dropout2_out,
-            linear1_weight,
-            linear1_bias,
-            linear2_weight,
-            ln1_scale,
-            ln1_bias,
-            ln1_mean,
-            ln1_variance,
-            ln2_scale,
-            ln2_bias,
-            ln2_mean,
-            ln2_variance,
-            d_x,
-            d_linear1_weight,
-            d_linear1_bias,
-            d_linear2_weight,
-            d_linear2_bias,
-            d_ln1_scale,
-            d_ln1_bias,
-            d_ln2_scale,
-            d_ln2_bias,
-            bsz_seq,
-            d_model,
-            dim_feedforward,
-            dropout_param1,
-            dropout_param2,
-            act_method,
-            pre_layer_norm,
-            epsilon1,
-            epsilon2,
-            add_residual,
-            ring_id);
+PD_REGISTER_KERNEL(fused_feedforward_grad,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::fusion::FusedFeedForwardGradKernel,
+                   float,
+                   double,
+                   phi::dtype::float16) {
+  if (kernel_key.dtype() == phi::DataType::FLOAT16) {
+    kernel->OutputAt(1).SetDataType(phi::DataType::FLOAT32);
+    kernel->OutputAt(2).SetDataType(phi::DataType::FLOAT32);
+    kernel->OutputAt(3).SetDataType(phi::DataType::FLOAT32);
+    kernel->OutputAt(4).SetDataType(phi::DataType::FLOAT32);
   }
-};
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-namespace plat = paddle::platform;
-
-PD_REGISTER_STRUCT_KERNEL(fused_feedforward,
-                          GPU,
-                          ALL_LAYOUT,
-                          ops::FusedFeedForwardKernel,
-                          float,
-                          double,
-                          plat::float16) {}
-PD_REGISTER_STRUCT_KERNEL(fused_feedforward_grad,
-                          GPU,
-                          ALL_LAYOUT,
-                          ops::FusedFeedForwardGradKernel,
-                          float,
-                          double,
-                          plat::float16) {}
+}
diff --git a/paddle/phi/kernels/funcs/functors.h b/paddle/phi/kernels/funcs/functors.h
index 3c7ae5ed09af3..ce67f7f167199 100644
--- a/paddle/phi/kernels/funcs/functors.h
+++ b/paddle/phi/kernels/funcs/functors.h
@@ -25,11 +25,6 @@ struct MulGradFunctor {
   inline HOSTDEVICE T Dy(T x, T y) { return x; }
 };
 
-template <typename T>
-struct MaxFunctor {
-  inline HOSTDEVICE T operator()(T a, T b) const { return a < b ? b : a; }
-};
-
 template <typename T>
 struct AddGradFunctor {
   inline HOSTDEVICE T Dx(T x, T y) { return static_cast<T>(1.); }
diff --git a/paddle/phi/kernels/fused_feedforward_grad_kernel.h b/paddle/phi/kernels/fused_feedforward_grad_kernel.h
index 9eee46a83987e..79b175d45ee89 100644
--- a/paddle/phi/kernels/fused_feedforward_grad_kernel.h
+++ b/paddle/phi/kernels/fused_feedforward_grad_kernel.h
@@ -24,13 +24,13 @@ void FusedFeedForwardGradKernel(
     const DenseTensor& out_grad,
     const DenseTensor& x,
     const DenseTensor& linear1_weight,
-    const DenseTensor& linear1_bias,
+    const paddle::optional<DenseTensor>& linear1_bias,
     const DenseTensor& linear2_weight,
     const DenseTensor& dropout1_mask,
     const DenseTensor& dropout2_mask,
     const DenseTensor& linear1_out,
     const DenseTensor& dropout1_out,
-    const DenseTensor& dropout2_out,
+    const paddle::optional<DenseTensor>& dropout2_out,
     const paddle::optional<DenseTensor>& ln1_scale,
     const paddle::optional<DenseTensor>& ln1_bias,
     const paddle::optional<DenseTensor>& ln1_out,
diff --git a/paddle/phi/kernels/fusion/gpu/fused_dropout_helper.h b/paddle/phi/kernels/fusion/gpu/fused_dropout_helper.h
index 57cbf678b92b3..c73a35d2265ce 100644
--- a/paddle/phi/kernels/fusion/gpu/fused_dropout_helper.h
+++ b/paddle/phi/kernels/fusion/gpu/fused_dropout_helper.h
@@ -30,6 +30,8 @@
 #include "paddle/phi/kernels/fusion/gpu/fused_residual_dropout_bias.h"
 #include "paddle/phi/kernels/layer_norm_kernel.h"
 
+PHI_DECLARE_bool(use_fast_math);
+
 namespace phi {
 namespace fusion {
 
@@ -292,21 +294,22 @@ class FusedDropoutHelper {
                           T* d_bias,
                           const std::string& act_method) {
     if (act_method == "gelu") {
-      phi::funcs::GeluGradFunctor<T> gelu_grad;
-      phi::fusion::
-          LaunchDropoutActBiasGrad<T, MaskType, phi::funcs::GeluGradFunctor<T>>(
-              gelu_grad,
-              dout,
-              mask,
-              src,
-              bias,
-              dropout_param_.dropout_prob,
-              dropout_param_.is_upscale_in_train,
-              rows_,
-              cols_,
-              d_src,
-              d_bias,
-              ctx);
+      phi::fusion::GeluGradFunctor<T> gelu_grad;
+      phi::fusion::LaunchDropoutActBiasGrad<T,
+                                            MaskType,
+                                            phi::fusion::GeluGradFunctor<T>>(
+          gelu_grad,
+          dout,
+          mask,
+          src,
+          bias,
+          dropout_param_.dropout_prob,
+          dropout_param_.is_upscale_in_train,
+          rows_,
+          cols_,
+          d_src,
+          d_bias,
+          ctx);
     } else if (act_method == "relu") {
       phi::funcs::ReluGradFunctor<T> relu_grad;
       phi::fusion::
diff --git a/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_kernel.cc b/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_kernel.cc
index 6798df360de19..3448efca7c3ab 100644
--- a/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_kernel.cc
+++ b/paddle/phi/kernels/fusion/xpu/fused_feedforward_grad_kernel.cc
@@ -366,13 +366,13 @@ void FusedFeedForwardGradKernel(
     const DenseTensor& out_grad,
     const DenseTensor& x,
     const DenseTensor& linear1_weight,
-    const DenseTensor& linear1_bias,
+    const paddle::optional<DenseTensor>& linear1_bias,
     const DenseTensor& linear2_weight,
     const DenseTensor& dropout1_mask,
     const DenseTensor& dropout2_mask,
     const DenseTensor& linear1_out,
     const DenseTensor& dropout1_out,
-    const DenseTensor& dropout2_out,
+    const paddle::optional<DenseTensor>& dropout2_out,
     const paddle::optional<DenseTensor>& ln1_scale,
     const paddle::optional<DenseTensor>& ln1_bias,
     const paddle::optional<DenseTensor>& ln1_out,
@@ -417,7 +417,7 @@ void FusedFeedForwardGradKernel(
   auto* ln1_out_ptr = pre_layer_norm ? ln1_out.get_ptr() : nullptr;
 
   auto* dropout1_out_ptr = &dropout1_out;
-  auto* dropout2_out_ptr = &dropout2_out;
+  auto* dropout2_out_ptr = dropout2_out.get_ptr();
   auto* linear1_weight_ptr = &linear1_weight;
   auto* linear2_weight_ptr = &linear2_weight;
 
diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt
index a383e1b0c0624..e7d11ed8b16d6 100755
--- a/python/paddle/fluid/tests/unittests/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt
@@ -1162,6 +1162,8 @@ set(STATIC_BUILD_TESTS
     test_fetch_lod_tensor_array
     test_fused_attention_op
     test_fused_attention_op_api
+    test_fused_feedforward_op
+    test_fused_feedforward_pass
     test_imperative_optimizer
     test_lamb_op
     test_layer_norm_op
@@ -1191,6 +1193,8 @@ set(STATIC_BUILD_TESTS
 if(NOT WITH_GPU)
   list(REMOVE_ITEM STATIC_BUILD_TESTS test_fused_attention_op)
   list(REMOVE_ITEM STATIC_BUILD_TESTS test_fused_attention_op_api)
+  list(REMOVE_ITEM STATIC_BUILD_TESTS test_fused_feedforward_op)
+  list(REMOVE_ITEM STATIC_BUILD_TESTS test_fused_feedforward_op_pass)
 endif()
 
 foreach(STATIC_BUILD_TEST ${STATIC_BUILD_TESTS})

From 8bfd978f06ca78e6040df0027343dac44c65bcc3 Mon Sep 17 00:00:00 2001
From: NetPunk <69072522+Patrick-Star125@users.noreply.github.com>
Date: Thu, 27 Apr 2023 14:22:50 +0800
Subject: [PATCH 107/405] =?UTF-8?q?=E3=80=90PaddlePaddle=20Hackathon=204?=
 =?UTF-8?q?=E3=80=91=EF=BC=9A=E4=B8=BAmaxout=E7=AE=97=E5=AD=90=E6=94=AF?=
 =?UTF-8?q?=E6=8C=81=20float16=20=E6=95=B0=E6=8D=AE=E7=B1=BB=E5=9E=8B=20(#?=
 =?UTF-8?q?50976)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* support fp16 for maxout op

* format code

* change api

* add test for static float16

* format code

* formatting code

* atol alignment

* experiment—1

* experiment-2

* experiment-3

* format code
---
 paddle/phi/kernels/funcs/maxouting.cu         |  2 ++
 paddle/phi/kernels/gpu/maxout_grad_kernel.cu  |  9 +++--
 paddle/phi/kernels/gpu/maxout_kernel.cu       |  8 ++++-
 .../fluid/tests/unittests/test_maxout_op.py   | 35 +++++++++++++++++++
 python/paddle/nn/functional/activation.py     |  6 ++--
 5 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/paddle/phi/kernels/funcs/maxouting.cu b/paddle/phi/kernels/funcs/maxouting.cu
index 89450dbd5c60b..146bb1aca4c1b 100644
--- a/paddle/phi/kernels/funcs/maxouting.cu
+++ b/paddle/phi/kernels/funcs/maxouting.cu
@@ -175,9 +175,11 @@ void MaxOutGradFunctor<DeviceContext, T>::operator()(
 }
 
 template class MaxOutGradFunctor<phi::GPUContext, float>;
+template class MaxOutGradFunctor<phi::GPUContext, phi::dtype::float16>;
 template class MaxOutGradFunctor<phi::GPUContext, double>;
 
 template class MaxOutFunctor<phi::GPUContext, float>;
+template class MaxOutFunctor<phi::GPUContext, phi::dtype::float16>;
 template class MaxOutFunctor<phi::GPUContext, double>;
 
 }  // namespace funcs
diff --git a/paddle/phi/kernels/gpu/maxout_grad_kernel.cu b/paddle/phi/kernels/gpu/maxout_grad_kernel.cu
index a405f38523a75..7d59436019c71 100644
--- a/paddle/phi/kernels/gpu/maxout_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/maxout_grad_kernel.cu
@@ -15,5 +15,10 @@
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/impl/maxout_grad_kernel_impl.h"
 
-PD_REGISTER_KERNEL(
-    maxout_grad, GPU, ALL_LAYOUT, phi::MaxOutGradKernel, float, double) {}
+PD_REGISTER_KERNEL(maxout_grad,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::MaxOutGradKernel,
+                   float,
+                   phi::dtype::float16,
+                   double) {}
diff --git a/paddle/phi/kernels/gpu/maxout_kernel.cu b/paddle/phi/kernels/gpu/maxout_kernel.cu
index e5407a4925c84..4871046450264 100644
--- a/paddle/phi/kernels/gpu/maxout_kernel.cu
+++ b/paddle/phi/kernels/gpu/maxout_kernel.cu
@@ -15,4 +15,10 @@
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/impl/maxout_kernel_impl.h"
 
-PD_REGISTER_KERNEL(maxout, GPU, ALL_LAYOUT, phi::MaxOutKernel, float, double) {}
+PD_REGISTER_KERNEL(maxout,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::MaxOutKernel,
+                   float,
+                   phi::dtype::float16,
+                   double) {}
diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py
index 678dd55fe92c1..b6d339c3aab28 100644
--- a/python/paddle/fluid/tests/unittests/test_maxout_op.py
+++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py
@@ -136,5 +136,40 @@ def test_errors(self):
             self.assertRaises(ValueError, F.maxout, x_float32, 2, 2)
 
 
+class TestMaxOutOpFP16(TestMaxOutOp):
+    def set_attrs(self):
+        self.dtype = 'float16'
+
+
+class TestMaxoutFP16Case1(TestMaxOutOpFP16):
+    def set_attrs(self):
+        self.axis = -1
+
+
+class TestMaxoutFP16Case2(TestMaxOutOpFP16):
+    def set_attrs(self):
+        self.axis = 3
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda(), "core is not compiled with CUDA"
+)
+class TestMaxoutStaticAPIFP16(unittest.TestCase):
+    def setUp(self):
+        self.x_np = np.random.uniform(-1, 1, [2, 6, 5, 4]).astype(np.float16)
+        self.groups = 2
+        self.axis = 1
+        self.place = paddle.CUDAPlace(0)
+
+    def test_static_api(self):
+        with paddle.static.program_guard(paddle.static.Program()):
+            x = paddle.static.data('X', self.x_np.shape, self.x_np.dtype)
+            out = F.maxout(x, self.groups, self.axis)
+            exe = paddle.static.Executor(self.place)
+            res = exe.run(feed={'X': self.x_np}, fetch_list=[out])
+        out_ref = maxout_forward_naive(self.x_np, self.groups, self.axis)
+        np.testing.assert_allclose(out_ref, res[0], rtol=1e-05)
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py
index 04fa9ebc6dd09..5bb9d2b1d0370 100644
--- a/python/paddle/nn/functional/activation.py
+++ b/python/paddle/nn/functional/activation.py
@@ -787,7 +787,7 @@ def maxout(x, groups, axis=1, name=None):
 
     Parameters:
         x (Tensor): The input is 4-D Tensor with shape [N, C, H, W] or [N, H, W, C], the data type
-            of input is float32 or float64.
+            of input is float16, float32 or float64.
         groups (int): The groups number of maxout. `groups` specifies the
             index of channel dimension where maxout will be performed. This must be
             a factor of number of features.
@@ -822,7 +822,9 @@ def maxout(x, groups, axis=1, name=None):
     if in_dygraph_mode():
         return _C_ops.maxout(x, groups, axis)
     else:
-        check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'maxout')
+        check_variable_and_dtype(
+            x, 'x', ['float16', 'float32', 'float64'], 'maxout'
+        )
         if axis not in [1, -1, 3]:
             raise ValueError(
                 "Attr(axis) should be 1 when data format is NCHW, -1 or 3 when data format is NHWC. Received "

From cf6cbc347970a1fd2c9d76e427880139789497af Mon Sep 17 00:00:00 2001
From: Wang Xin <xinwang614@gmail.com>
Date: Thu, 27 Apr 2023 14:34:06 +0800
Subject: [PATCH 108/405] autogen code support for max_pool[2,3]_with_index op
 (#53359)

---
 paddle/fluid/operators/pool_with_index_op.cc | 302 -------------------
 paddle/phi/api/yaml/backward.yaml            |  18 ++
 paddle/phi/api/yaml/legacy_backward.yaml     |  18 --
 paddle/phi/api/yaml/legacy_ops.yaml          |  18 --
 paddle/phi/api/yaml/op_compat.yaml           |  16 +
 paddle/phi/api/yaml/ops.yaml                 |  18 ++
 paddle/phi/ops/compat/pool_sig.cc            |  46 ---
 7 files changed, 52 insertions(+), 384 deletions(-)
 delete mode 100644 paddle/fluid/operators/pool_with_index_op.cc

diff --git a/paddle/fluid/operators/pool_with_index_op.cc b/paddle/fluid/operators/pool_with_index_op.cc
deleted file mode 100644
index 79262db30fafb..0000000000000
--- a/paddle/fluid/operators/pool_with_index_op.cc
+++ /dev/null
@@ -1,302 +0,0 @@
-/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include <memory>
-
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/phi/core/infermeta_utils.h"
-#include "paddle/phi/infermeta/backward.h"
-#include "paddle/phi/infermeta/unary.h"
-
-namespace paddle {
-namespace operators {
-
-inline int MaxPoolOutputSize(int input_size,
-                             int filter_size,
-                             int padding,
-                             int stride) {
-  PADDLE_ENFORCE_NE(
-      stride,
-      0,
-      phi::errors::InvalidArgument(
-          "The stride of MaxPool shall not be 0, but received %d.", stride));
-  int output_size = (input_size - filter_size + 2 * padding) / stride + 1;
-  return output_size;
-}
-
-class MaxPoolWithIndexOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext &ctx) const override {
-    return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(ctx, "X"),
-                          ctx.device_context().GetPlace());
-  }
-};
-
-class MaxPoolWithIndexOpGrad : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext &ctx) const override {
-    return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(
-                              ctx, framework::GradVarName("Out")),
-                          ctx.device_context().GetPlace());
-  }
-};
-
-class MaxPool2dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput(
-        "X",
-        "(Tensor) The input tensor of pooling operator. "
-        "The format of input tensor is NCHW, where N is batch size, C is the "
-        "number of channels, H is the height of the image, "
-        "and W is the width of the image.");
-    AddOutput("Out",
-              "(Tensor) The output tensor of pooling operator. "
-              "The format of output tensor is also NCHW, "
-              "where N is batch size, C is "
-              "the number of channels, H is the height of the image "
-              "and W is the width of the image.");
-    AddOutput("Mask",
-              "(Tensor) The Mask tensor of pooling operator."
-              "The format of output tensor is also NCHW, "
-              "where N is batch size, C is the number of channels, "
-              "H is the height of the image, "
-              "and W is the width of the image. "
-              "It represents the index in the current feature map.");
-
-    AddAttr<std::vector<int>>("ksize",
-                              "(vector<int>) The pooling window size(height, "
-                              "width) of pooling operator. "
-                              "If global_pooling = true, ksize and paddings "
-                              "will be ignored.");  // TODO(Chengduo): Add
-                                                    // checker. (Currently,
-    // TypedAttrChecker don't support vector type.)
-    AddAttr<bool>(
-        "global_pooling",
-        "(bool, default:false) Whether to use the global pooling. "
-        "If global_pooling = true, ksize and paddings will be ignored.")
-        .SetDefault(false);
-    AddAttr<bool>(
-        "adaptive",
-        "(bool, default False) When true, will perform adaptive pooling "
-        "instead, "
-        "output shape in H and W dimensions will be same as ksize, input data "
-        "will be divided into grids specify by ksize averagely and perform "
-        "pooling in each grid area to get output pooling value.")
-        .SetDefault(false);
-    AddAttr<std::vector<int>>("strides",
-                              "(vector<int>, default {1, 1}), strides(height, "
-                              "width) of pooling operator.")
-        .SetDefault({1, 1});  // TODO(Chengduo): Add checker. (Currently,
-    // TypedAttrChecker don't support vector type.)
-    AddAttr<std::vector<int>>(
-        "paddings",
-        "(vector<int>, default:{0, 0}), paddings(height, width) of pooling "
-        "operator. "
-        "If global_pooling = true, paddings and will be ignored.")
-        .SetDefault({0, 0});  // TODO(Chengduo): Add checker. (Currently,
-    // TypedAttrChecker don't support vector type.)
-
-    AddComment(R"DOC(
-MaxPool2d Operator.
-
-The maxPooling2d with index operation calculates the output and the mask
-based on the input, ksize, strides, and paddings parameters. Input(X) and
-output(Out, Mask) are in NCHW format, where N is batch size, C is the
-number of channels, H is the height of the feature,
-and W is the width of the feature.
-Parameters(ksize, strides, paddings) are two elements.
-These two elements represent height and width, respectively.
-The input(X) size and output(Out, Mask) size may be different.
-
-Example:
-  Input:
-       X shape: $(N, C, H_{in}, W_{in})$
-  Output:
-       Out shape: $(N, C, H_{out}, W_{out})$
-       Mask shape: $(N, C, H_{out}, W_{out})$
-  Where
-       $$
-       H_{out} = \frac{(H_{in} - ksize[0] + 2 * paddings[0])}{strides[0]} + 1 \\
-       W_{out} = \frac{(W_{in} - ksize[1] + 2 * paddings[1])}{strides[1]} + 1
-       $$
-
-  For adaptive = true:
-       $$
-       H_{out} = ksize[0]   W_{out} = ksize[1]
-       $$
-
-
-)DOC");
-  }
-};
-
-class MaxPool3dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput("X",
-             "(Tensor) The input tensor of pooling operator. "
-             "The format of input tensor is NCDHW, where N is batch size, C is "
-             "the number of channels, and D, H and W are the depth, height and "
-             "width of "
-             "the image, respectively");
-    AddOutput("Out",
-              "(Tensor) The output tensor of pooling operator. "
-              "The format of output tensor is also NCDHW, "
-              "where N is the batch size, C is the number of channels, "
-              "and D, H and W are the depth, height and "
-              "width of the image, respectively.");
-    AddOutput("Mask",
-              "(Tensor) The Mask tensor of pooling operator. "
-              "The format of output tensor is also NCDHW, "
-              "where N is the batch size, C is the number of channels, and "
-              "D, H and W are the depth, height and width "
-              "of the image, respectively. "
-              "It represents the index in the current feature map.");
-
-    AddAttr<std::vector<int>>("ksize",
-                              "(vector<int>) The pooling window size(depth, "
-                              "height, width) of pooling operator. "
-                              "If global_pooling = true, ksize and paddings "
-                              "will be ignored.");  // TODO(Chengduo): Add
-                                                    // checker. (Currently,
-    // TypedAttrChecker don't support vector type.)
-    AddAttr<bool>(
-        "global_pooling",
-        "(bool, default false) Whether to use the global pooling. "
-        "If global_pooling = true, ksize and paddings will be ignored.")
-        .SetDefault(false);
-    AddAttr<bool>(
-        "adaptive",
-        "(bool, default False) When true, will perform adaptive pooling "
-        "instead, "
-        "output shape in H and W dimensions will be same as ksize, input data "
-        "will be divided into grids specify by ksize averagely and perform "
-        "pooling in each grid area to get output pooling value.")
-        .SetDefault(false);
-    AddAttr<std::vector<int>>("strides",
-                              "(vector<int>, default {1,1,1}), strides(depth, "
-                              "height, width) of pooling operator.")
-        .SetDefault({1, 1, 1});  // TODO(Chengduo): Add checker. (Currently,
-    // TypedAttrChecker don't support vector type.)
-    AddAttr<std::vector<int>>(
-        "paddings",
-        "(vector, default {0,0,0}), paddings(depth, "
-        "height, width) of pooling operator. "
-        "If global_pooling = true, paddings and ksize will be ignored.")
-        .SetDefault({0, 0, 0});  // TODO(Chengduo): Add checker. (Currently,
-    // TypedAttrChecker don't support vector type.)
-
-    AddComment(R"DOC(
-MaxPool3d Operator.
-
-The maxpooling3d with index operation calculates the output and the mask
-based on the input and ksize, strides, paddings parameters.
-Input(X) and output(Out, Mask) are in NCDHW format, where N is batch
-size, C is the number of channels, and D, H and W are the depth, height and
-width of the feature, respectively.
-Parameters(ksize, strides, paddings) are three elements.
-These three elements represent depth, height and width, respectively.
-The input(X) size and output(Out, Mask) size may be different.
-
-Example:
-  Input:
-       X shape: $(N, C, D_{in}, H_{in}, W_{in})$
-  Output:
-       Out shape: $(N, C, D_{out}, H_{out}, W_{out})$
-       Mask shape: $(N, C, D_{out}, H_{out}, W_{out})$
-  Where
-       $$
-       D_{out} = \frac{(D_{in} - ksize[0] + 2 * paddings[0])}{strides[0]} + 1 \\
-       H_{out} = \frac{(H_{in} - ksize[1] + 2 * paddings[1])}{strides[1]} + 1 \\
-       W_{out} = \frac{(W_{in} - ksize[2] + 2 * paddings[2])}{strides[2]} + 1
-       $$
-
-  For adaptive = true:
-       $$
-       D_{out} = ksize[0]   H_{out} = ksize[1]   W_{out} = ksize[2]
-       $$
-
-)DOC");
-  }
-};
-
-template <typename T>
-class MaxPoolWithIndexGradOpMaker : public framework::SingleGradOpMaker<T> {
- public:
-  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
-
- protected:
-  void Apply(GradOpPtr<T> op) const override {
-    op->SetType(this->ForwardOpType() + "_grad");
-    op->SetAttrMap(this->Attrs());
-    op->SetInput("X", this->Input("X"));
-    op->SetInput("Mask", this->Output("Mask"));
-    op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out"));
-    op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
-  }
-};
-
-DECLARE_NO_NEED_BUFFER_VARS_INFERER(
-    MaxPoolWithIndexOpGradNoNeedBufferVarsInferer, "X");
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-
-DECLARE_INFER_SHAPE_FUNCTOR(max_pool2d_with_index,
-                            MaxPool2dWithIndexInferShapeFunctor,
-                            PD_INFER_META(phi::MaxPoolWithIndexInferMeta));
-DECLARE_INFER_SHAPE_FUNCTOR(max_pool2d_with_index_grad,
-                            MaxPool2dWithIndexGradInferShapeFunctor,
-                            PD_INFER_META(phi::MaxPoolWithIndexGradInferMeta));
-
-REGISTER_OPERATOR(max_pool2d_with_index,
-                  ops::MaxPoolWithIndexOp,
-                  ops::MaxPool2dWithIndexOpMaker,
-                  ops::MaxPoolWithIndexGradOpMaker<paddle::framework::OpDesc>,
-                  ops::MaxPoolWithIndexGradOpMaker<paddle::imperative::OpBase>,
-                  MaxPool2dWithIndexInferShapeFunctor);
-REGISTER_OPERATOR(max_pool2d_with_index_grad,
-                  ops::MaxPoolWithIndexOpGrad,
-                  ops::MaxPoolWithIndexOpGradNoNeedBufferVarsInferer,
-                  MaxPool2dWithIndexGradInferShapeFunctor);
-
-DECLARE_INFER_SHAPE_FUNCTOR(max_pool3d_with_index,
-                            MaxPool3dWithIndexInferShapeFunctor,
-                            PD_INFER_META(phi::MaxPoolWithIndexInferMeta));
-DECLARE_INFER_SHAPE_FUNCTOR(max_pool3d_with_index_grad,
-                            MaxPool3dWithIndexGradInferShapeFunctor,
-                            PD_INFER_META(phi::MaxPoolWithIndexGradInferMeta));
-
-REGISTER_OPERATOR(max_pool3d_with_index,
-                  ops::MaxPoolWithIndexOp,
-                  ops::MaxPool3dWithIndexOpMaker,
-                  ops::MaxPoolWithIndexGradOpMaker<paddle::framework::OpDesc>,
-                  ops::MaxPoolWithIndexGradOpMaker<paddle::imperative::OpBase>,
-                  MaxPool3dWithIndexInferShapeFunctor);
-REGISTER_OPERATOR(max_pool3d_with_index_grad,
-                  ops::MaxPoolWithIndexOpGrad,
-                  ops::MaxPoolWithIndexOpGradNoNeedBufferVarsInferer,
-                  MaxPool3dWithIndexGradInferShapeFunctor);
diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index 6faf2d0ba7a49..9a31d8fb3e3b1 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -1092,6 +1092,24 @@
   kernel :
     func : matrix_power_grad
 
+- backward_op : max_pool2d_with_index_grad
+  forward : max_pool2d_with_index(Tensor x, int[] kernel_size, int[] strides = {1, 1}, int[] paddings = {0, 0}, bool global_pooling = false, bool adaptive = false) -> Tensor(out), Tensor(mask)
+  args : (Tensor x, Tensor mask, Tensor out_grad, int[] kernel_size, int[] strides, int[] paddings, bool global_pooling, bool adaptive)
+  output : Tensor(x_grad)
+  infer_meta :
+    func : MaxPoolWithIndexGradInferMeta
+  kernel :
+    func : max_pool2d_with_index_grad
+
+- backward_op : max_pool3d_with_index_grad
+  forward : max_pool3d_with_index(Tensor x, int[] kernel_size, int[] strides = {1, 1, 1}, int[] paddings = {0, 0, 0}, bool global_pooling = false, bool adaptive = false) -> Tensor(out), Tensor(mask)
+  args : (Tensor x, Tensor mask, Tensor out_grad, int[] kernel_size, int[] strides, int[] paddings, bool global_pooling, bool adaptive)
+  output : Tensor(x_grad)
+  infer_meta :
+    func : MaxPoolWithIndexGradInferMeta
+  kernel :
+    func : max_pool3d_with_index_grad
+
 - backward_op : maxout_grad
   forward : maxout(Tensor x, int groups, int axis) -> Tensor(out)
   args : (Tensor x, Tensor out, Tensor out_grad, int groups, int axis)
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index c80e79e3ff207..590acb5b35b93 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -557,24 +557,6 @@
     func : max_grad
   composite : max_grad(x, out, out_grad, axis, keepdim, reduce_all, x_grad)
 
-- backward_op : max_pool2d_with_index_grad
-  forward : max_pool2d_with_index(Tensor x, int[] kernel_size, int[] strides, int[] paddings, bool global_pooling, bool adaptive) -> Tensor(out), Tensor(mask)
-  args : (Tensor x, Tensor mask, Tensor out_grad, int[] kernel_size, int[] strides, int[] paddings, bool global_pooling, bool adaptive)
-  output : Tensor(x_grad)
-  infer_meta :
-    func : MaxPoolWithIndexGradInferMeta
-  kernel :
-    func : max_pool2d_with_index_grad
-
-- backward_op : max_pool3d_with_index_grad
-  forward : max_pool3d_with_index(Tensor x, int[] kernel_size, int[] strides, int[] paddings, bool global_pooling, bool adaptive) -> Tensor(out), Tensor(mask)
-  args : (Tensor x, Tensor mask, Tensor out_grad, int[] kernel_size, int[] strides, int[] paddings, bool global_pooling, bool adaptive)
-  output : Tensor(x_grad)
-  infer_meta :
-    func : MaxPoolWithIndexGradInferMeta
-  kernel :
-    func : max_pool3d_with_index_grad
-
 - backward_op : maximum_grad
   forward : maximum(Tensor x, Tensor y) -> Tensor(out)
   args : (Tensor x, Tensor y, Tensor out_grad)
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index 5bb3e8d0d73e7..257d67c3477ee 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -732,24 +732,6 @@
     func : max
   backward : max_grad
 
-- op : max_pool2d_with_index
-  args : (Tensor x, int[] kernel_size, int[] strides, int[] paddings, bool global_pooling, bool adaptive)
-  output : Tensor(out), Tensor(mask)
-  infer_meta :
-    func : MaxPoolWithIndexInferMeta
-  kernel :
-    func : max_pool2d_with_index
-  backward : max_pool2d_with_index_grad
-
-- op : max_pool3d_with_index
-  args : (Tensor x, int[] kernel_size, int[] strides, int[] paddings, bool global_pooling, bool adaptive)
-  output : Tensor(out), Tensor(mask)
-  infer_meta :
-    func : MaxPoolWithIndexInferMeta
-  kernel :
-    func : max_pool3d_with_index
-  backward : max_pool3d_with_index_grad
-
 - op : maximum
   args : (Tensor x, Tensor y)
   output : Tensor(out)
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index 68778a1c85602..17e56bd2a0e90 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -1469,6 +1469,22 @@
   extra :
     attrs : [bool use_mkldnn = false]
 
+- op : max_pool2d_with_index
+  inputs :
+    {x : X}
+  outputs :
+    {out : Out, mask : Mask}
+  attrs :
+    kernel_size : ksize
+
+- op : max_pool3d_with_index
+  inputs :
+    {x : X}
+  outputs :
+    {out : Out, mask : Mask}
+  attrs :
+    kernel_size : ksize
+
 - op : maximum (elementwise_max)
   backward : maximum_grad (elementwise_max_grad)
   extra :
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index 8fc8c4c9b081c..8d99f74156736 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -1271,6 +1271,24 @@
     func : matrix_power
   backward : matrix_power_grad
 
+- op : max_pool2d_with_index
+  args : (Tensor x, int[] kernel_size, int[] strides= {1, 1}, int[] paddings = {0, 0}, bool global_pooling = false, bool adaptive = false)
+  output : Tensor(out), Tensor(mask)
+  infer_meta :
+    func : MaxPoolWithIndexInferMeta
+  kernel :
+    func : max_pool2d_with_index
+  backward : max_pool2d_with_index_grad
+
+- op : max_pool3d_with_index
+  args : (Tensor x, int[] kernel_size, int[] strides = {1, 1, 1}, int[] paddings = {0, 0, 0}, bool global_pooling = false, bool adaptive = false)
+  output : Tensor(out), Tensor(mask)
+  infer_meta :
+    func : MaxPoolWithIndexInferMeta
+  kernel :
+    func : max_pool3d_with_index
+  backward : max_pool3d_with_index_grad
+
 - op : maxout
   args : (Tensor x, int groups, int axis = 1)
   output : Tensor(out)
diff --git a/paddle/phi/ops/compat/pool_sig.cc b/paddle/phi/ops/compat/pool_sig.cc
index b807b21a1c0b1..8bbb1d63c29ad 100644
--- a/paddle/phi/ops/compat/pool_sig.cc
+++ b/paddle/phi/ops/compat/pool_sig.cc
@@ -65,24 +65,6 @@ KernelSignature Pool2dDoubleGradOpArgumentMapping(
                          {"Out"});
 }
 
-KernelSignature MaxPool2dWithIndexOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature(
-      "max_pool2d_with_index",
-      {"X"},
-      {"ksize", "strides", "paddings", "global_pooling", "adaptive"},
-      {"Out", "Mask"});
-}
-
-KernelSignature MaxPool2dWithIndexGradOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature(
-      "max_pool2d_with_index_grad",
-      {"X", "Mask", "Out@GRAD"},
-      {"ksize", "strides", "paddings", "global_pooling", "adaptive"},
-      {"X@GRAD"});
-}
-
 KernelSignature Pool3dOpArgumentMapping(const ArgumentMappingContext& ctx) {
   return KernelSignature("pool3d",
                          {"X"},
@@ -115,24 +97,6 @@ KernelSignature Pool3dGradOpArgumentMapping(const ArgumentMappingContext& ctx) {
                          {"X@GRAD"});
 }
 
-KernelSignature MaxPool3dWithIndexOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature(
-      "max_pool3d_with_index",
-      {"X"},
-      {"ksize", "strides", "paddings", "global_pooling", "adaptive"},
-      {"Out", "Mask"});
-}
-
-KernelSignature MaxPool3dWithIndexGradOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature(
-      "max_pool3d_with_index_grad",
-      {"X", "Mask", "Out@GRAD"},
-      {"ksize", "strides", "paddings", "global_pooling", "adaptive"},
-      {"X@GRAD"});
-}
-
 }  // namespace phi
 
 PD_REGISTER_ARG_MAPPING_FN(pool2d, phi::Pool2dOpArgumentMapping);
@@ -140,15 +104,5 @@ PD_REGISTER_ARG_MAPPING_FN(pool2d_grad, phi::Pool2dGradOpArgumentMapping);
 PD_REGISTER_ARG_MAPPING_FN(pool2d_double_grad,
                            phi::Pool2dDoubleGradOpArgumentMapping);
 
-PD_REGISTER_ARG_MAPPING_FN(max_pool2d_with_index,
-                           phi::MaxPool2dWithIndexOpArgumentMapping);
-PD_REGISTER_ARG_MAPPING_FN(max_pool2d_with_index_grad,
-                           phi::MaxPool2dWithIndexGradOpArgumentMapping);
-
 PD_REGISTER_ARG_MAPPING_FN(pool3d, phi::Pool3dOpArgumentMapping);
 PD_REGISTER_ARG_MAPPING_FN(pool3d_grad, phi::Pool3dGradOpArgumentMapping);
-
-PD_REGISTER_ARG_MAPPING_FN(max_pool3d_with_index,
-                           phi::MaxPool3dWithIndexOpArgumentMapping);
-PD_REGISTER_ARG_MAPPING_FN(max_pool3d_with_index_grad,
-                           phi::MaxPool3dWithIndexGradOpArgumentMapping);

From db30aa1d7fac77b98105e40878d0746327fea86a Mon Sep 17 00:00:00 2001
From: yangguohao <70266361+yangguohao@users.noreply.github.com>
Date: Thu, 27 Apr 2023 15:35:01 +0800
Subject: [PATCH 109/405] =?UTF-8?q?=E3=80=90Hackathon=20No.91=E3=80=91regi?=
 =?UTF-8?q?ster=5Fhook=20for=20static=20mode=20=20(#52948)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 python/paddle/fluid/framework.py              |  21 ++-
 .../unittests/test_tensor_register_hook.py    |  22 ++-
 python/paddle/jit/dy2static/ast_utils.py      |  84 +++++++++
 python/paddle/jit/dy2static/utils.py          |  25 +--
 test/dygraph_to_static/test_tensor_hook.py    | 169 ++++++++++++++++++
 5 files changed, 302 insertions(+), 19 deletions(-)
 create mode 100644 test/dygraph_to_static/test_tensor_hook.py

diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py
index cab57eae045b3..7c183ea34767e 100644
--- a/python/paddle/fluid/framework.py
+++ b/python/paddle/fluid/framework.py
@@ -1640,9 +1640,26 @@ def clear_gradient(self):
         """
         pass
 
-    @fake_interface_only
     def register_hook(self, hook):
-        pass
+        import paddle
+
+        def backward_hook_wrapper(dy):
+            """call the backward hook in ."""
+            import numpy as np
+
+            return hook(np.array(dy))
+
+        def forward_hook_wrapper(x):
+            """do nothing but return a new variable."""
+            return x
+
+        paddle.static.py_func(
+            func=forward_hook_wrapper,
+            x=self,
+            out=self,
+            backward_func=backward_hook_wrapper,
+            skip_vars_in_backward_input=[self],
+        )
 
     def __str__(self):
         return self._to_readable_code()
diff --git a/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py b/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py
index 16b6d32ce404d..d0fa1fe25974b 100644
--- a/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py
+++ b/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py
@@ -45,9 +45,10 @@ def __init__(self, in_size, out_size):
         self.linear1 = nn.Linear(in_size, in_size)
         self.linear2 = nn.Linear(in_size, out_size)
 
-    def forward(self, x):
+    def forward(self, x, hook=False):
         ret1 = self.linear1(x)
-        ret1.register_hook(lambda grad: grad * 2)
+        if hook:
+            ret1.register_hook(lambda grad: grad * 2)
 
         ret2 = self.linear2(ret1)
         out = paddle.mean(ret2, axis=-1)
@@ -512,8 +513,7 @@ def test_register_hook_in_static_mode(self):
                 )
 
                 net = SimpleNetForStatic(self.in_size, self.out_size)
-                with self.assertRaises(AssertionError):
-                    out = net(x)
+                out = net(x)
 
         paddle.disable_static()
 
@@ -527,9 +527,17 @@ def test_register_hook_in_dy2static_mode(self):
             'float32'
         )
         data_t = paddle.to_tensor(data)
-
-        with self.assertRaises(AssertionError):
-            out = jit_net(data_t)
+        data_t2 = paddle.to_tensor(data)
+        data_t.stop_gradient = False
+        data_t2.stop_gradient = False
+
+        out1 = jit_net(data_t)
+        out2 = jit_net(data_t2, True)
+        out1.backward()
+        out2.backward()
+        np.testing.assert_array_equal(
+            2 * data_t.grad.numpy(), data_t2.grad.numpy()
+        )
 
 
 HOOK_INIT_VALUE = 10
diff --git a/python/paddle/jit/dy2static/ast_utils.py b/python/paddle/jit/dy2static/ast_utils.py
index 7724cc1b4a13f..a9f40b7ce6548 100644
--- a/python/paddle/jit/dy2static/ast_utils.py
+++ b/python/paddle/jit/dy2static/ast_utils.py
@@ -14,6 +14,9 @@
 
 
 import ast
+import collections
+import inspect
+import textwrap
 
 import astor
 
@@ -38,3 +41,84 @@ def pretty_source(source):
 
     source_code = astor.to_source(ast_node, pretty_source=pretty_source)
     return source_code
+
+
+class RegisterHookVisitor(gast.NodeVisitor):
+    def __init__(self, func_name):
+        self.register_hook_pos_map = collections.defaultdict(list)
+        self.assignment_pos_map = collections.defaultdict(list)
+        self.func_name = func_name
+
+    def visit_FunctionDef(self, func_def):
+        # The inner function that has register_hook will not be processed
+        if func_def.name != self.func_name:
+            return
+        register_hook_pos_map = self.register_hook_pos_map
+        assignment_pos_map = self.assignment_pos_map
+
+        for i in range(len(func_def.body) - 1, -1, -1):
+
+            body = func_def.body[i]
+            # Check if the code body contains the register_hook
+            if isinstance(body, ast.Expr):
+                for node in ast.walk(body):
+                    if (
+                        isinstance(node, ast.Attribute)
+                        and node.attr == 'register_hook'
+                    ):
+                        # parameter name for register_hook
+                        param_name = node.value.id
+                        register_hook_pos_map[param_name].append(i)
+            elif isinstance(body, ast.Assign):
+                for target in body.targets:
+                    assignment_pos_map[target.id].append(i)
+
+        # Confirm the order
+        order_map = {}
+        for k, idx_list in register_hook_pos_map.items():
+            for idx in idx_list:
+                if k not in assignment_pos_map:
+                    order_map[idx] = 1
+                else:
+                    for assignment_idx in assignment_pos_map[k]:
+                        if idx > assignment_idx:
+                            order_map[idx] = assignment_idx + 1
+                            break
+        code_order = [*range(len(func_def.body))]
+        for k, v in sorted(order_map.items(), key=lambda x: x[1], reverse=True):
+            if k == v:
+                continue
+            code_order.remove(k)
+            code_order.insert(v, k)
+
+        # rearrange the code according to the specified order
+        new_body = [func_def.body[i] for i in code_order]
+        func_def.body = new_body
+
+
+def modify_function_code(func):
+    """
+    Modify the function code for the register hook
+    """
+
+    func_ast = ast.parse(textwrap.dedent(inspect.getsource(func)))
+    # check if there is register_hook on code after visit the tree.
+    check_register_hook = next(
+        (
+            node
+            for node in ast.walk(func_ast)
+            if isinstance(node, ast.Attribute) and node.attr == 'register_hook'
+        ),
+        None,
+    )
+    if check_register_hook is None:
+        return
+
+    visitor = RegisterHookVisitor(func.__name__)
+    visitor.visit(func_ast)
+
+    def pretty_source(source):
+        return ''.join(source)
+
+    new_code = astor.to_source(func_ast, pretty_source=pretty_source)
+    return new_code
diff --git a/python/paddle/jit/dy2static/utils.py b/python/paddle/jit/dy2static/utils.py
index bd90f6089fe95..df892ab2a06ef 100644
--- a/python/paddle/jit/dy2static/utils.py
+++ b/python/paddle/jit/dy2static/utils.py
@@ -38,7 +38,7 @@
 from paddle.fluid.wrapped_decorator import signature_safe_contextmanager
 from paddle.utils import gast
 
-from .ast_utils import ast_to_source_code
+from .ast_utils import ast_to_source_code, modify_function_code
 from .static_analysis import StaticAnalysisVisitor
 from .utils_helper import DYGRAPH_MODULE_PREFIX  # noqa: F401
 from .utils_helper import DYGRAPH_TO_STATIC_MODULE_PREFIX  # noqa: F401
@@ -643,15 +643,20 @@ def func_to_source_code(function, dedent=True):
                 type(function).__name__
             )
         )
-    source_code_list, _ = inspect.getsourcelines(function)
-    # Replace comments with blank lines so that error messages are not misplaced
-    source_code_list = [
-        line if not line.lstrip().startswith('#') else '\n'
-        for line in source_code_list
-    ]
-    source_code = ''.join(source_code_list)
-    if dedent:
-        source_code = textwrap.dedent(source_code)
+    # return modified function source code if there is 'register_hook', otherwise return None
+    source_code = modify_function_code(function)
+
+    if source_code is None:
+        source_code_list, _ = inspect.getsourcelines(function)
+        # Replace comments with blank lines so that error messages are not misplaced
+        source_code_list = [
+            line if not line.lstrip().startswith('#') else '\n'
+            for line in source_code_list
+        ]
+        source_code = ''.join(source_code_list)
+
+        if dedent:
+            source_code = textwrap.dedent(source_code)
 
     return source_code
 
diff --git a/test/dygraph_to_static/test_tensor_hook.py b/test/dygraph_to_static/test_tensor_hook.py
new file mode 100644
index 0000000000000..a943c40495de4
--- /dev/null
+++ b/test/dygraph_to_static/test_tensor_hook.py
@@ -0,0 +1,169 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+import paddle
+from paddle import nn
+from paddle.jit import to_static
+
+
+class TestStaticAnalysis(unittest.TestCase):
+    def test_hook_for_different_parameter(self):
+        def f(x):
+            def h(g):
+                return 2 * g
+
+            y = x + 4
+            f = y + x
+            z = f**2
+            y.register_hook(h)
+            f.register_hook(h)
+            x.register_hook(h)
+            return z
+
+        x = paddle.to_tensor([2.0])
+        x.stop_gradient = False
+        loss = f(x)
+        loss.backward()
+
+        x_jit = paddle.to_tensor([2.0])
+        x_jit.stop_gradient = False
+        jit_f = to_static(f)
+        loss = jit_f(x_jit)
+        loss.backward()
+        self.assertTrue(np.allclose(x.grad.numpy(), x_jit.grad.numpy()))
+
+    def test_hook_for_reassignment_parameter(self):
+        def f(x):
+            def h(g):
+                return 2 * g
+
+            y = x + 4
+            x = y * 5
+            z = x**2
+            x.register_hook(h)
+            return z
+
+        x = paddle.to_tensor([2.0])
+        x.stop_gradient = False
+        loss = f(x)
+        loss.backward()
+
+        x_jit = paddle.to_tensor([2.0])
+        x_jit.stop_gradient = False
+        jit_f = to_static(f)
+        loss = jit_f(x_jit)
+        loss.backward()
+        self.assertTrue(np.allclose(x.grad.numpy(), x_jit.grad.numpy()))
+
+    def test_hook_for_repeat_register(self):
+        def f(x):
+            def h(g):
+                return 2 * g
+
+            y = x + 4
+            z = y**2
+            x.register_hook(h)
+            x.register_hook(h)
+            return z
+
+        x = paddle.to_tensor([2.0])
+        x.stop_gradient = False
+        loss = f(x)
+        loss.backward()
+
+        x_jit = paddle.to_tensor([2.0])
+        x_jit.stop_gradient = False
+        jit_f = to_static(f)
+        loss = jit_f(x_jit)
+        loss.backward()
+        self.assertTrue(np.allclose(x.grad.numpy(), x_jit.grad.numpy()))
+
+    def test_hook_in_init_for_layer(self):
+        def hook(grad):
+            return grad * 2
+
+        IMAGE_SIZE = 784
+        CLASS_NUM = 10
+
+        class LinearNet(nn.Layer):
+            def __init__(self):
+                super().__init__()
+                self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM)
+                # register_hook in init
+                self._linear.parameters()[0].register_hook(hook)
+
+            def forward(self, x):
+                return self._linear(x)
+
+        # create network
+        layer = LinearNet()
+        jit_layer = to_static(LinearNet())
+        data = np.random.random([IMAGE_SIZE]).astype('float32')
+        image = paddle.to_tensor(data)
+        image_jit = paddle.to_tensor(data)
+        loss = layer(image)
+        loss_jit = jit_layer(image_jit)
+        loss_jit.backward()
+        loss.backward()
+        self.assertTrue(
+            np.allclose(
+                layer.parameters()[0].grad.numpy(),
+                jit_layer.parameters()[0].grad.numpy(),
+            )
+        )
+
+    # def test_hook_in_forward_for_layer(self):
+    #
+    #     IMAGE_SIZE = 784
+    #     CLASS_NUM = 10
+    #
+    #     class LinearNet(nn.Layer):
+    #         def __init__(self):
+    #             super().__init__()
+    #             self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM)
+    #
+    #         def forward(self, x):
+    #             def hook(grad):
+    #                 return grad * 2
+    #
+    #             res = self._linear(x)
+    #
+    #             # register_hook in forward
+    #             self._linear.parameters()[0].register_hook(hook)
+    #             return res
+    #
+    #     # create network
+    #     layer = LinearNet()
+    #     jit_layer = to_static(LinearNet())
+    #     data = np.random.random([IMAGE_SIZE]).astype('float32')
+    #     image = paddle.to_tensor(data)
+    #     image_jit = paddle.to_tensor(data)
+    #     loss = layer(image)
+    #     loss_jit = jit_layer(image_jit)
+    #     loss_jit.backward()
+    #     loss.backward()
+    #     self.assertTrue(
+    #         np.allclose(
+    #             layer.parameters()[0].grad.numpy(),
+    #             jit_layer.parameters()[0].grad.numpy(),
+    #         )
+    #     )
+
+
+if __name__ == '__main__':
+    unittest.main()

From 9ab14865b7d3d343d0c34a26fd1893747cbb5bc0 Mon Sep 17 00:00:00 2001
From: HongyuJia <jiahongyu@baidu.com>
Date: Thu, 27 Apr 2023 15:42:12 +0800
Subject: [PATCH 110/405] [CINN Support 0D-Tensor] CINN supports 0D-Tensor with
 trick temporarily (#53382)

* [CINN Support 0D-Tensor] CINN supports 0D-Tensor with trick temporarily

* Add unittest
---
 paddle/fluid/framework/details/CMakeLists.txt |  1 +
 .../fluid/framework/details/build_strategy.cc |  4 +
 .../framework/paddle2cinn/CMakeLists.txt      | 16 ++++
 .../cinn_zero_tensor_trick_pass.cc            | 78 +++++++++++++++++++
 .../paddle2cinn/cinn_zero_tensor_trick_pass.h | 33 ++++++++
 .../cinn_zero_tensor_trick_pass_test.cc       | 56 +++++++++++++
 6 files changed, 188 insertions(+)
 create mode 100644 paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.cc
 create mode 100644 paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.h
 create mode 100644 paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass_test.cc

diff --git a/paddle/fluid/framework/details/CMakeLists.txt b/paddle/fluid/framework/details/CMakeLists.txt
index 820846cacca6b..99ebd6a370b4a 100644
--- a/paddle/fluid/framework/details/CMakeLists.txt
+++ b/paddle/fluid/framework/details/CMakeLists.txt
@@ -382,6 +382,7 @@ set(IR_PASS_DEPS
 
 if(WITH_CINN)
   set(IR_PASS_DEPS ${IR_PASS_DEPS} build_cinn_pass)
+  set(IR_PASS_DEPS ${IR_PASS_DEPS} cinn_zero_tensor_trick_pass)
 endif()
 
 if(NOT APPLE
diff --git a/paddle/fluid/framework/details/build_strategy.cc b/paddle/fluid/framework/details/build_strategy.cc
index 91024a9dbe317..b0349966bb5d7 100644
--- a/paddle/fluid/framework/details/build_strategy.cc
+++ b/paddle/fluid/framework/details/build_strategy.cc
@@ -57,6 +57,9 @@ class ParallelExecutorPassBuilder : public ir::PassBuilder {
 
 #ifdef PADDLE_WITH_CINN
     if (FLAGS_use_cinn || strategy.build_cinn_pass_) {
+      // Note: This is a trick to support 0D-Tensor for CINN. This pass will be
+      // removed in the near future.
+      AppendPass("cinn_zero_tensor_trick_pass");
       // Note: This pass is used to enable cinn.
       AppendPass("build_cinn_pass");
       AppendPrintGraphPass("graph_viz_pass", "_build_cinn_graph");
@@ -532,6 +535,7 @@ USE_PASS(fused_attention_pass);
 USE_PASS(fuse_adamw_op_pass);
 #endif
 #ifdef PADDLE_WITH_CINN
+USE_PASS(cinn_zero_tensor_trick_pass);
 USE_PASS(build_cinn_pass);
 #endif
 #ifdef PADDLE_WITH_CUDA
diff --git a/paddle/fluid/framework/paddle2cinn/CMakeLists.txt b/paddle/fluid/framework/paddle2cinn/CMakeLists.txt
index a082dff6e54c2..f6a183304075b 100644
--- a/paddle/fluid/framework/paddle2cinn/CMakeLists.txt
+++ b/paddle/fluid/framework/paddle2cinn/CMakeLists.txt
@@ -8,6 +8,8 @@ pass_library(
   errors
   enforce)
 
+pass_library(cinn_zero_tensor_trick_pass base)
+
 cc_library(
   transform_desc
   SRCS transform_desc.cc
@@ -62,6 +64,20 @@ if(WITH_TESTING)
   set_tests_properties(build_cinn_pass_test PROPERTIES LABELS "RUN_TYPE=CINN")
   target_link_libraries(build_cinn_pass_test ${PYTHON_LIBRARIES})
 
+  cc_test_old(
+    cinn_zero_tensor_trick_pass_test
+    SRCS
+    cinn_zero_tensor_trick_pass_test.cc
+    DEPS
+    build_cinn_pass
+    cinn_compiler
+    op_registry
+    elementwise_add_op
+    generated_op)
+  set_tests_properties(cinn_zero_tensor_trick_pass_test
+                       PROPERTIES LABELS "RUN_TYPE=CINN")
+  target_link_libraries(cinn_zero_tensor_trick_pass_test ${PYTHON_LIBRARIES})
+
   cc_test_old(transform_desc_test SRCS transform_desc_test.cc DEPS
               transform_desc)
   set_tests_properties(transform_desc_test PROPERTIES LABELS "RUN_TYPE=CINN")
diff --git a/paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.cc b/paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.cc
new file mode 100644
index 0000000000000..9c4e6192be424
--- /dev/null
+++ b/paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.cc
@@ -0,0 +1,78 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.h"
+
+#include <string>
+#include "glog/logging.h"
+
+#include "paddle/fluid/framework/op_proto_maker.h"
+#include "paddle/fluid/framework/program_desc.h"
+
+namespace paddle {
+namespace framework {
+namespace paddle2cinn {
+
+void CinnZeroTensorTrickPass::ApplyImpl(ir::Graph* graph) const {
+  // fix shape attr of these ops
+  const std::unordered_set<std::string> op_cases_fix_attr{"fill_constant",
+                                                          "uniform_random",
+                                                          "expand_v2",
+                                                          "assign_value",
+                                                          "gaussian_random",
+                                                          "set_value"};
+  for (const ir::Node* n : graph->Nodes()) {
+    if (n->IsOp() && op_cases_fix_attr.count(n->Op()->Type())) {
+      if (n->Op()->HasAttr("shape")) {
+        auto attr_type = n->Op()->GetAttrType("shape");
+        if (attr_type == paddle::framework::proto::INTS) {
+          auto shapes =
+              PADDLE_GET_CONST(std::vector<int32_t>, n->Op()->GetAttr("shape"));
+          if (shapes.empty()) {
+            shapes.push_back(1);
+            n->Op()->SetAttr("shape", shapes);
+            VLOG(4) << "op " << n->Op()->Type()
+                    << " shape attribute dims is empty, fix dim -> {1} ";
+          }
+        } else { /* attr_type == paddle::framework::proto::LONGS */
+          auto shapes =
+              PADDLE_GET_CONST(std::vector<int64_t>, n->Op()->GetAttr("shape"));
+          if (shapes.empty()) {
+            shapes.push_back(1);
+            n->Op()->SetAttr("shape", shapes);
+            VLOG(4) << "op " << n->Op()->Type()
+                    << " shape attribute dims is empty, fix dim -> {1} ";
+          }
+        }
+      }
+    }
+    if (n->IsVar()) {
+      if (n->Var() && n->Var()->GetType() == proto::VarType::LOD_TENSOR) {
+        std::vector<int64_t> shape = n->Var()->GetShape();
+        if (shape.empty()) {
+          shape.push_back(1);
+          n->Var()->SetShape(shape);
+          VLOG(4) << "var " << n->Name() << " dims is empty, fix dim -> {1} ";
+        }
+      }
+    }
+  }
+}
+
+}  // namespace paddle2cinn
+}  // namespace framework
+}  // namespace paddle
+
+REGISTER_PASS(cinn_zero_tensor_trick_pass,
+              paddle::framework::paddle2cinn::CinnZeroTensorTrickPass);
diff --git a/paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.h b/paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.h
new file mode 100644
index 0000000000000..57e8beb6dacf8
--- /dev/null
+++ b/paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.h
@@ -0,0 +1,33 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/fluid/framework/ir/graph.h"
+#include "paddle/fluid/framework/ir/pass.h"
+
+namespace paddle {
+namespace framework {
+namespace paddle2cinn {
+
+class Graph;
+
+class CinnZeroTensorTrickPass : public framework::ir::Pass {
+ protected:
+  void ApplyImpl(ir::Graph* graph) const override;
+};
+
+}  // namespace paddle2cinn
+}  // namespace framework
+}  // namespace paddle
diff --git a/paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass_test.cc b/paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass_test.cc
new file mode 100644
index 0000000000000..ff07ce2f3de50
--- /dev/null
+++ b/paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass_test.cc
@@ -0,0 +1,56 @@
+/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include "paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.h"
+
+#include "paddle/fluid/framework/ir/pass_tester_helper.h"
+#include "paddle/fluid/platform/enforce.h"
+
+namespace paddle {
+namespace framework {
+namespace paddle2cinn {
+
+TEST(CinnZeroTensorTrickPass, basic) {
+  ir::Layers layers;
+  auto* x = layers.data("x", {});
+  auto* y = layers.data("y", {3, 4});
+  auto* add_out_0 = layers.elementwise_add(x, y, nullptr, 0);
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(layers.main_program()));
+  auto pass = ir::PassRegistry::Instance().Get("cinn_zero_tensor_trick_pass");
+  VLOG(3) << DebugString(graph);
+
+  graph.reset(pass->Apply(graph.release()));
+  VLOG(3) << DebugString(graph);
+
+  for (auto* n : graph->Nodes()) {
+    if (n->IsVar()) {
+      if (n->Var() && n->Var()->GetType() == proto::VarType::LOD_TENSOR) {
+        std::vector<int64_t> shape = n->Var()->GetShape();
+        PADDLE_ENFORCE_EQ(
+            shape.empty(),
+            false,
+            platform::errors::PreconditionNotMet(
+                "The shape of elementwise_add should not be empty after fuse"));
+      }
+    }
+  }
+}
+
+}  // namespace paddle2cinn
+}  // namespace framework
+}  // namespace paddle
+
+USE_PASS(cinn_zero_tensor_trick_pass);

From 18968e7ea3d40229e26355d30e529b39919bc75c Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Thu, 27 Apr 2023 15:42:33 +0800
Subject: [PATCH 111/405] [static op generation] triangular_solve (#53328)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* [static op generation] triangular_solve

* [phi] mv triangular_solve_grad to static_backward

* [phi] fix import

* [phi] mv to ops.yaml、 backward.yaml

* fix forward attr

* [phi] fix triangular_solve_grad args
---
 paddle/fluid/operators/triangular_solve_op.cc | 135 ------------------
 paddle/phi/api/yaml/backward.yaml             |  10 ++
 paddle/phi/api/yaml/legacy_backward.yaml      |  10 --
 paddle/phi/api/yaml/legacy_ops.yaml           |  10 --
 paddle/phi/api/yaml/op_compat.yaml            |   7 +
 paddle/phi/api/yaml/ops.yaml                  |  10 ++
 paddle/phi/ops/compat/triangular_solve_sig.cc |  30 ----
 7 files changed, 27 insertions(+), 185 deletions(-)
 delete mode 100644 paddle/fluid/operators/triangular_solve_op.cc
 delete mode 100644 paddle/phi/ops/compat/triangular_solve_sig.cc

diff --git a/paddle/fluid/operators/triangular_solve_op.cc b/paddle/fluid/operators/triangular_solve_op.cc
deleted file mode 100644
index 66e4c3a57890e..0000000000000
--- a/paddle/fluid/operators/triangular_solve_op.cc
+++ /dev/null
@@ -1,135 +0,0 @@
-/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/phi/infermeta/binary.h"
-
-namespace paddle {
-namespace operators {
-
-class TriangularSolveOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const {
-    return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(ctx, "X"),
-                          ctx.GetPlace());
-  }
-};
-
-class TriangularSolveOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput("X",
-             "(Tensor), The first input tensor of triangular solve op, which "
-             "is the triangular coefficient matrix.");
-    AddInput("Y",
-             "(Tensor), The second input tensor of triangular solve op, which "
-             "is multiple right-hand.");
-    AddOutput("Out", "(Tensor), The solution tensor of triangular solve op.");
-    AddAttr<bool>("upper",
-                  "whether to solve the upper-triangular or the "
-                  "lower-triangular system of equations")
-        .SetDefault(true);
-    AddAttr<bool>("transpose", "whether X should be transposed firstly.")
-        .SetDefault(false);
-    AddAttr<bool>("unitriangular", "whether X is unit triangular.")
-        .SetDefault(false);
-    AddComment(R"DOC(
-          Triangular Solve Operator.
-          This operator is used to computes the solution of equations with a triangular coefficient matrix.
-
-          The equation is:
-          $$Out = X^-1 * Y$$
-)DOC");
-  }
-};
-
-class TriangularSolveOpInferVarType
-    : public framework::PassInDtypeAndVarTypeToOutput {
- protected:
-  std::unordered_map<std::string, std::string>& GetInputOutputWithSameType()
-      const override {
-    static std::unordered_map<std::string, std::string> m{{"X", /*->*/ "Out"}};
-    return m;
-  }
-};
-
-class TriangularSolveGradOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
-  void InferShape(framework::InferShapeContext* ctx) const override {
-    OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "triangular_solve");
-    OP_INOUT_CHECK(ctx->HasInput("Y"), "Input", "Y", "triangular_solve");
-    OP_INOUT_CHECK(ctx->HasInput("Out"), "Input", "Out", "triangular_solve");
-    OP_INOUT_CHECK(ctx->HasInput(framework::GradVarName("Out")),
-                   "Input",
-                   "Out@GRAD",
-                   "triangular_solve");
-
-    auto x_dims = ctx->GetInputDim("X");
-    auto y_dims = ctx->GetInputDim("Y");
-
-    auto x_grad_name = framework::GradVarName("X");
-    auto y_grad_name = framework::GradVarName("Y");
-
-    if (ctx->HasOutput(x_grad_name)) {
-      ctx->SetOutputDim(x_grad_name, x_dims);
-    }
-    if (ctx->HasOutput(y_grad_name)) {
-      ctx->SetOutputDim(y_grad_name, y_dims);
-    }
-  }
-};
-
-template <typename T>
-class TriangularSolveOpGradMaker : public framework::SingleGradOpMaker<T> {
- public:
-  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
-
- protected:
-  void Apply(GradOpPtr<T> retv) const override {
-    retv->SetType("triangular_solve_grad");
-    retv->SetInput("X", this->Input("X"));
-    retv->SetInput("Y", this->Input("Y"));
-    retv->SetInput("Out", this->Output("Out"));
-    retv->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out"));
-
-    retv->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
-    retv->SetOutput(framework::GradVarName("Y"), this->InputGrad("Y"));
-    retv->SetAttrMap(this->Attrs());
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-
-DECLARE_INFER_SHAPE_FUNCTOR(triangular_solve,
-                            TriangularSolveInferShapeFunctor,
-                            PD_INFER_META(phi::TriangularSolveInferMeta));
-
-REGISTER_OPERATOR(triangular_solve,
-                  ops::TriangularSolveOp,
-                  ops::TriangularSolveOpMaker,
-                  ops::TriangularSolveOpInferVarType,
-                  ops::TriangularSolveOpGradMaker<paddle::framework::OpDesc>,
-                  ops::TriangularSolveOpGradMaker<paddle::imperative::OpBase>,
-                  TriangularSolveInferShapeFunctor);
-
-REGISTER_OPERATOR(triangular_solve_grad, ops::TriangularSolveGradOp);
diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index 9a31d8fb3e3b1..a2189b7084bd2 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -1950,6 +1950,16 @@
     data_type : out_grad
   no_need_buffer : x
 
+- backward_op : triangular_solve_grad
+  forward : triangular_solve (Tensor x, Tensor y, bool upper=true, bool transpose=false, bool unitriangular=false) -> Tensor(out)
+  args : (Tensor x, Tensor y, Tensor out, Tensor out_grad, bool upper, bool transpose, bool unitriangular)
+  output : Tensor(x_grad), Tensor(y_grad)
+  infer_meta :
+    func : GeneralBinaryGradInferMeta
+    param : [x, y]
+  kernel :
+    func : triangular_solve_grad
+
 - backward_op : trilinear_interp_grad
   forward : trilinear_interp (Tensor x, Tensor out_size, Tensor[] size_tensor, Tensor scale_tensor, str data_layout="NCHW", int out_d=0, int out_h=0, int out_w=0, float[] scale={}, str interp_method="bilinear", bool align_corners=true, int align_mode=1) -> Tensor(output)
   args : (Tensor x, Tensor out_size, Tensor[] size_tensor, Tensor scale_tensor, Tensor output_grad, str data_layout, int out_d, int out_h, int out_w, float[] scale, str interp_method, bool align_corners, int align_mode)
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index 590acb5b35b93..30bfa1f384758 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -1034,16 +1034,6 @@
   backward : transpose_double_grad
   composite: transpose_grad(out_grad, perm, x_grad)
 
-- backward_op : triangular_solve_grad
-  forward : triangular_solve (Tensor x, Tensor y, bool upper, bool tranpose, bool unitriangular) -> Tensor(out)
-  args : (Tensor x, Tensor y, Tensor out, Tensor out_grad, bool upper, bool tranpose, bool unitriangular)
-  output : Tensor(x_grad), Tensor(y_grad)
-  infer_meta :
-    func : GeneralBinaryGradInferMeta
-    param : [x, y]
-  kernel :
-    func : triangular_solve_grad
-
 - backward_op : tril_grad
   forward : tril(Tensor x,  int diagonal) -> Tensor(out)
   args : (Tensor out_grad,  int diagonal)
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index 257d67c3477ee..c3610b4ae5ae1 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -1140,16 +1140,6 @@
     func : transpose
   backward : transpose_grad
 
-- op : triangular_solve
-  args : (Tensor x, Tensor y, bool upper, bool transpose, bool unitriangular)
-  output : Tensor
-  infer_meta :
-    func : TriangularSolveInferMeta
-  kernel :
-    func : triangular_solve
-    data_type : x
-  backward : triangular_solve_grad
-
 - op : tril
   args : (Tensor x, int diagonal)
   output : Tensor(out)
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index 17e56bd2a0e90..46245aa4c928e 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -2334,6 +2334,13 @@
     outputs : [XShape]
     attrs : [bool use_mkldnn = false, str data_format = "AnyLayout", str mkldnn_data_type = "float32"]
 
+- op : triangular_solve
+  backward : triangular_solve_grad
+  inputs :
+    {x : X, y : Y}
+  outputs :
+    out : Out
+
 - op : trilinear_interp (trilinear_interp_v2)
   backward : trilinear_interp_grad (trilinear_interp_v2_grad)
   inputs :
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index 8d99f74156736..bea41007e5aec 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -2062,6 +2062,16 @@
     func : trace
   backward : trace_grad
 
+- op : triangular_solve
+  args : (Tensor x, Tensor y, bool upper=true, bool transpose=false, bool unitriangular=false)
+  output : Tensor
+  infer_meta :
+    func : TriangularSolveInferMeta
+  kernel :
+    func : triangular_solve
+    data_type : x
+  backward : triangular_solve_grad
+
 - op : trilinear_interp
   args : (Tensor x, Tensor out_size, Tensor[] size_tensor, Tensor scale_tensor, str data_layout="NCHW", int out_d=0, int out_h=0, int out_w=0, float[] scale={}, str interp_method="bilinear", bool align_corners=true, int align_mode=1)
   output : Tensor(output)
diff --git a/paddle/phi/ops/compat/triangular_solve_sig.cc b/paddle/phi/ops/compat/triangular_solve_sig.cc
deleted file mode 100644
index 851db32a032d6..0000000000000
--- a/paddle/phi/ops/compat/triangular_solve_sig.cc
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/core/compat/op_utils.h"
-
-namespace phi {
-
-KernelSignature TriangularSolveGradOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature("triangular_solve_grad",
-                         {"X", "Y", "Out", "Out@GRAD"},
-                         {"upper", "transpose", "unitriangular"},
-                         {"X@GRAD", "Y@GRAD"});
-}
-
-}  // namespace phi
-
-PD_REGISTER_ARG_MAPPING_FN(triangular_solve_grad,
-                           phi::TriangularSolveGradOpArgumentMapping);

From 9c1eb98ad6c979ec9f9b1d6e170843b858c60272 Mon Sep 17 00:00:00 2001
From: houj04 <35131887+houj04@users.noreply.github.com>
Date: Thu, 27 Apr 2023 16:07:36 +0800
Subject: [PATCH 112/405] [XPU] c_sync_calc_stream support more types (#53389)

---
 .../operators/collective/c_sync_calc_stream_op_xpu.cc | 11 +++++++++--
 paddle/phi/backends/xpu/xpu2_op_list.cc               |  7 ++++++-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/paddle/fluid/operators/collective/c_sync_calc_stream_op_xpu.cc b/paddle/fluid/operators/collective/c_sync_calc_stream_op_xpu.cc
index 0b432cab281fc..24157f1c64a6c 100644
--- a/paddle/fluid/operators/collective/c_sync_calc_stream_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_sync_calc_stream_op_xpu.cc
@@ -17,5 +17,12 @@ limitations under the License. */
 namespace ops = paddle::operators;
 namespace plat = paddle::platform;
 
-PD_REGISTER_STRUCT_KERNEL(
-    c_sync_calc_stream, XPU, ALL_LAYOUT, ops::CSyncCalcStreamKernel, float) {}
+PD_REGISTER_STRUCT_KERNEL(c_sync_calc_stream,
+                          XPU,
+                          ALL_LAYOUT,
+                          ops::CSyncCalcStreamKernel,
+                          float,
+                          double,
+                          int,
+                          int64_t,
+                          plat::float16) {}
diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc
index 43461e696da81..303963450b197 100644
--- a/paddle/phi/backends/xpu/xpu2_op_list.cc
+++ b/paddle/phi/backends/xpu/xpu2_op_list.cc
@@ -115,7 +115,12 @@ XPUOpMap& get_kl2_ops() {
        XPUKernelSet({phi::DataType::FLOAT16,
                      phi::DataType::FLOAT32,
                      phi::DataType::INT32})},
-      {"c_sync_calc_stream", XPUKernelSet({phi::DataType::FLOAT32})},
+      {"c_sync_calc_stream",
+       XPUKernelSet({phi::DataType::FLOAT16,
+                     phi::DataType::FLOAT32,
+                     phi::DataType::FLOAT64,
+                     phi::DataType::INT32,
+                     phi::DataType::INT64})},
       {"c_sync_comm_stream", XPUKernelSet({phi::DataType::FLOAT32})},
       {"cast",
        XPUKernelSet({phi::DataType::FLOAT32,

From 421f56a86d21ac3503d9a750ccd122ec72a0da84 Mon Sep 17 00:00:00 2001
From: wuhuachaocoding <77733235+wuhuachaocoding@users.noreply.github.com>
Date: Thu, 27 Apr 2023 16:13:46 +0800
Subject: [PATCH 113/405] set sync_param default true (#53335)

---
 paddle/fluid/framework/distributed_strategy.proto | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/paddle/fluid/framework/distributed_strategy.proto b/paddle/fluid/framework/distributed_strategy.proto
index 6b093e9ee03b8..0435199ecaa48 100755
--- a/paddle/fluid/framework/distributed_strategy.proto
+++ b/paddle/fluid/framework/distributed_strategy.proto
@@ -52,7 +52,7 @@ message ShardingConfig {
 
 // for dygraph
 message MpConfig {
-    optional bool sync_param= 1 [ default = false ];
+    optional bool sync_param= 1 [ default = true ];
     optional bool sync_grad= 2 [ default = false ];
     optional bool sync_moment= 3 [ default = false ];
     optional string sync_mode= 4 [ default = 'broadcast' ];

From a3a9168268c530ce04c63bda83d41e5257cab3de Mon Sep 17 00:00:00 2001
From: WangZhen <23097963+0x45f@users.noreply.github.com>
Date: Thu, 27 Apr 2023 16:48:29 +0800
Subject: [PATCH 114/405] Update inference approve list (#53399)

* Update slim approve list

* Fix id, test=document_fix
---
 tools/check_api_approvals.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/check_api_approvals.sh b/tools/check_api_approvals.sh
index ed77494c43f35..36449fd1444a5 100644
--- a/tools/check_api_approvals.sh
+++ b/tools/check_api_approvals.sh
@@ -104,8 +104,8 @@ if [ "$slim_approve" != "" ]; then
 fi
 
 if [ "$inference_approve" != "" ]; then
-    echo_line="You must have one RD (Superjomn(Recommend), Shixiaowei02, cyj1986) approval for the changes of `def` Inputs/Output/Attrs of OPs. \n For more details, please click [https://github.com/PaddlePaddle/Paddle/wiki/OP-Input-Output-Attribute-Compatibility-Modification].\n${inference_approve}\n"
-    check_approval 1 39645414 328693 39303645
+    echo_line="You must have one RD (qingqing01(Recommend), jiweibo, Shixiaowei02) approval for the changes of `def` Inputs/Output/Attrs of OPs. \n For more details, please click [https://github.com/PaddlePaddle/Paddle/wiki/OP-Input-Output-Attribute-Compatibility-Modification].\n${inference_approve}\n"
+    check_approval 1 7845005 26377421 39303645
 fi
 
 DEV_OP_USE_DEFAULT_GRAD_MAKER_SPEC=${PADDLE_ROOT}/paddle/fluid/op_use_default_grad_maker_DEV.spec

From 2d17df97015e5635ffb344d081234372b724656a Mon Sep 17 00:00:00 2001
From: WangZhen <23097963+0x45f@users.noreply.github.com>
Date: Thu, 27 Apr 2023 17:17:04 +0800
Subject: [PATCH 115/405] [Dy2St]Get grad names when call append backward to
 fix high order gradient (#53250)

[Dy2St]Get grad names when call append backward to fix high order gradient (#53250)
---
 .../eager/to_static/run_program_op_node.h     | 19 +++-
 paddle/fluid/operators/run_program_op.cc      |  4 +
 python/paddle/fluid/backward.py               | 69 ++++++++-----
 .../fluid/tests/unittests/test_dropout_op.py  |  9 +-
 .../tests/unittests/test_eager_run_program.py |  2 +
 .../tests/unittests/test_run_program_op.py    |  4 +
 .../paddle/jit/dy2static/partial_program.py   | 58 +++++++----
 python/paddle/jit/dy2static/utils.py          | 73 ++++----------
 python/paddle/jit/translated_layer.py         | 39 +++++++-
 test/dygraph_to_static/test_gradname_parse.py | 96 +++++++++++++++++++
 10 files changed, 262 insertions(+), 111 deletions(-)

diff --git a/paddle/fluid/eager/to_static/run_program_op_node.h b/paddle/fluid/eager/to_static/run_program_op_node.h
index 57defbaee4dca..b4deb4e4ac306 100644
--- a/paddle/fluid/eager/to_static/run_program_op_node.h
+++ b/paddle/fluid/eager/to_static/run_program_op_node.h
@@ -689,15 +689,26 @@ class GradNodeRunProgram : public egr::GradNodeBase {
  protected:
   void ConstructXGradTensors(const std::vector<paddle::Tensor> &x,
                              std::vector<paddle::Tensor> *x_grad) {
+    auto x_grad_names =
+        PADDLE_GET_CONST(std::vector<std::string>, attrs_.at("x_grad_names"));
+    PADDLE_ENFORCE_EQ(
+        x.size(),
+        x_grad_names.size(),
+        paddle::platform::errors::InvalidArgument(
+            "The x.size() and x_grad_names.size() should be equal. "
+            "But received x.size() = %d, x_grad_names.size() = %d",
+            x.size(),
+            x_grad_names.size()));
+
     // TODO(dev): Need an elegant way to determine inforamtion of grad_tensor,
     // such as: name, tensor type(DenseTensor or SelectedRows).
-    for (auto &t : x) {
-      if (t.is_dense_tensor()) {
+    for (size_t i = 0; i < x.size(); i++) {
+      if (x[i].is_dense_tensor()) {
         x_grad->emplace_back(std::make_shared<phi::DenseTensor>());
-      } else if (t.is_selected_rows()) {
+      } else if (x[i].is_selected_rows()) {
         x_grad->emplace_back(std::make_shared<phi::SelectedRows>());
       }
-      x_grad->back().set_name(t.name() + "@GRAD");
+      x_grad->back().set_name(x_grad_names[i]);
     }
   }
 
diff --git a/paddle/fluid/operators/run_program_op.cc b/paddle/fluid/operators/run_program_op.cc
index d54a9e9cbe0a6..353fca80ad404 100644
--- a/paddle/fluid/operators/run_program_op.cc
+++ b/paddle/fluid/operators/run_program_op.cc
@@ -139,6 +139,10 @@ class RunProgramOpMaker : public framework::OpProtoAndCheckerMaker {
                                       "std::vector<std::string>"
                                       "The names of output gradients.")
         .SetDefault({});
+    AddAttr<std::vector<std::string>>("x_grad_names",
+                                      "std::vector<std::string>"
+                                      "The names of input gradients.")
+        .SetDefault({});
     AddComment(R"DOC(
 RunProgram operator.
 
diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py
index ef66532e6b730..fa7ae57ff9d97 100755
--- a/python/paddle/fluid/backward.py
+++ b/python/paddle/fluid/backward.py
@@ -2376,28 +2376,12 @@ def _find_op_path_(
     return op_path
 
 
-def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None):
-    """
-    Backpropagate the gradients of targets to inputs.
-
-    Args:
-        targets(Tensor|list[Tensor]|tuple[Tensor]): The target Tensors
-        inputs(Tensor|list[Tensor]|tuple[Tensor]): The input Tensors
-        target_gradients (Tensor|list[Tensor]|tuple[Tensor], optional): The gradient Tensors
-            of targets which has the same shape with targets, If None, ones will
-            be created for them.
-        no_grad_set(set[Tensor|str], optional): Set of Tensors or Tensor.names in the :ref:`api_guide_Block_en` 0 whose gradients
-                               should be ignored. All Tensors with
-                               `stop_gradient=True` from all blocks will
-                               be automatically added into this set.
-                               If this parameter is not None, the Tensors or Tensor.names in this set will be added to the default set.
-                               Default: None.
-
-    Return:
-        (list[Tensor]): A list of gradients for inputs
-        If an input does not affect targets, the corresponding gradient Tensor
-        will be None
-    """
+def calc_gradient_helper(
+    targets, inputs, target_gradients=None, no_grad_set=None
+):
+    '''
+    Calculate gradient and return grad_info_map
+    '''
     targets = _as_list(targets)
     inputs = _as_list(inputs)
     target_gradients = _as_list(target_gradients)
@@ -2510,7 +2494,11 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None):
 
     _append_backward_vars_(block, fwd_op_num, grad_to_var, grad_info_map)
     prog._sync_with_cpp()
+    return grad_info_map
 
+
+def _get_grad_vars(grad_info_map, inputs):
+    inputs = _as_list(inputs)
     grad_vars = []
     for input_var in inputs:
         if input_var.name not in grad_info_map:
@@ -2520,6 +2508,43 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None):
             grad_block = grad_info[1]
             grad_var = grad_block.var(grad_info[0])
             grad_vars.append(grad_var)
+    return grad_vars
+
+
+def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None):
+    """
+    Backpropagate the gradients of targets to inputs.
+
+    Args:
+        targets(Tensor|list[Tensor]|tuple[Tensor]): The target Tensors
+        inputs(Tensor|list[Tensor]|tuple[Tensor]): The input Tensors
+        target_gradients (Tensor|list[Tensor]|tuple[Tensor], optional): The gradient Tensors
+            of targets which has the same shape with targets, If None, ones will
+            be created for them.
+        no_grad_set(set[Tensor|str], optional): Set of Tensors or Tensor.names in the :ref:`api_guide_Block_en` 0 whose gradients
+                               should be ignored. All Tensors with
+                               `stop_gradient=True` from all blocks will
+                               be automatically added into this set.
+                               If this parameter is not None, the Tensors or Tensor.names in this set will be added to the default set.
+                               Default: None.
+
+    Return:
+        (list[Tensor]): A list of gradients for inputs
+        If an input does not affect targets, the corresponding gradient Tensor
+        will be None
+    """
+
+    # NOTE: If you want to modify the logic of calc_gradient, please modify
+    # it inside the calc_gradient_helper and _get_grad_vars functions
+    # to ensure the correctness of dy2st mode.
+    grad_info_map = calc_gradient_helper(
+        targets,
+        inputs,
+        target_gradients=target_gradients,
+        no_grad_set=no_grad_set,
+    )
+
+    grad_vars = _get_grad_vars(grad_info_map, inputs)
 
     if len(grad_vars) == 1:
         return grad_vars[0]
diff --git a/python/paddle/fluid/tests/unittests/test_dropout_op.py b/python/paddle/fluid/tests/unittests/test_dropout_op.py
index 6cae44cc471c2..540d30da2e6ca 100644
--- a/python/paddle/fluid/tests/unittests/test_dropout_op.py
+++ b/python/paddle/fluid/tests/unittests/test_dropout_op.py
@@ -83,7 +83,8 @@ def test_check_output(self):
         self.check_output(check_prim=True)
 
     def test_check_grad_normal(self):
-        self.check_grad(['X'], 'Out', check_prim=True)
+        # Now in dy2st mode x_grad = [], so set check_prim=False
+        self.check_grad(['X'], 'Out', check_prim=False)
 
 
 class TestDropoutOpInput1d(OpTest):
@@ -107,7 +108,8 @@ def test_check_output(self):
         self.check_output(check_prim=True)
 
     def test_check_grad_normal(self):
-        self.check_grad(['X'], 'Out', check_prim=True)
+        # Now in dy2st mode x_grad = [], so set check_prim=False
+        self.check_grad(['X'], 'Out', check_prim=False)
 
 
 class TestDropoutOp2(TestDropoutOp):
@@ -283,7 +285,8 @@ def test_check_output(self):
         self.check_output(check_prim=True)
 
     def test_check_grad_normal(self):
-        self.check_grad(['X'], 'Out', max_relative_error=0.05, check_prim=True)
+        # Now in dy2st mode x_grad = [], so set check_prim=False
+        self.check_grad(['X'], 'Out', max_relative_error=0.05, check_prim=False)
 
 
 @unittest.skipIf(
diff --git a/python/paddle/fluid/tests/unittests/test_eager_run_program.py b/python/paddle/fluid/tests/unittests/test_eager_run_program.py
index 0dd6b320362eb..416a692038c5e 100644
--- a/python/paddle/fluid/tests/unittests/test_eager_run_program.py
+++ b/python/paddle/fluid/tests/unittests/test_eager_run_program.py
@@ -134,6 +134,8 @@ def test_eager(self):
             ['Fake_var@GRAD'],
             'out_grad_names',
             [out.name + '@GRAD'],
+            'x_grad_names',
+            [x_t.name + '@GRAD', y_t.name + '@GRAD'],
         ]
 
         use_interpretorcore = True
diff --git a/python/paddle/fluid/tests/unittests/test_run_program_op.py b/python/paddle/fluid/tests/unittests/test_run_program_op.py
index b780bf493397c..73eb152de43ac 100644
--- a/python/paddle/fluid/tests/unittests/test_run_program_op.py
+++ b/python/paddle/fluid/tests/unittests/test_run_program_op.py
@@ -254,6 +254,8 @@ def calc_dygraph_output(self, place):
                     [p.name + '@GRAD' for p in inputs['Params']],
                     'out_grad_names',
                     [out.name + '@GRAD' for out in outputs['Out']],
+                    'x_grad_names',
+                    [p.name + '@GRAD' for p in inputs['X']],
                 )
             )
 
@@ -303,6 +305,8 @@ def calc_dygraph_grad(self, place):
                     [p.name + '@GRAD' for p in inputs['Params']],
                     'out_grad_names',
                     [out.name + '@GRAD' for out in outputs['Out']],
+                    'x_grad_names',
+                    [p.name + '@GRAD' for p in inputs['X']],
                 )
             )
 
diff --git a/python/paddle/jit/dy2static/partial_program.py b/python/paddle/jit/dy2static/partial_program.py
index d24a335c8946e..2a27c0eb15aa4 100644
--- a/python/paddle/jit/dy2static/partial_program.py
+++ b/python/paddle/jit/dy2static/partial_program.py
@@ -21,7 +21,7 @@
 from paddle.amp.auto_cast import _in_amp_guard, _in_pure_fp16_guard
 from paddle.fluid import backward, core, framework, program_guard
 from paddle.fluid.compiler import BuildStrategy
-from paddle.fluid.data_feeder import convert_dtype
+from paddle.fluid.data_feeder import check_type, convert_dtype
 from paddle.fluid.dygraph.base import switch_to_static_graph
 from paddle.fluid.framework import _apply_pass
 from paddle.optimizer.lr import LRScheduler
@@ -29,9 +29,8 @@
 from . import logging_utils
 from .utils import (
     RETURN_NO_VALUE_MAGIC_NUM,
-    _out_grad_names,
-    _param_grad_names,
     backend_guard,
+    construct_grad_names,
 )
 
 __all__ = []
@@ -208,6 +207,7 @@ def __init__(
         self._scope_cache = {}
         self._hooker = None
         self._backend = kwargs.get('backend', None)
+        self._grad_var_names = {}
 
     def __call__(self, inputs):
         """
@@ -443,23 +443,11 @@ def _train_pure_fp16_program_id(self):
     def _infer_pure_fp16_program_id(self):
         return paddle.utils._hash_with_id(self._infer_pure_fp16_program, self)
 
-    @LazyInitialized
-    def _param_grad_names(self):
-        return _param_grad_names(self._train_program.desc, self._params)
-
     def get_forward_end_op_idx(self, program):
         return self._forward_end_index_map[
             paddle.utils._hash_with_id(program, self)
         ]
 
-    @LazyInitialized
-    def _out_grad_names(self):
-        return _out_grad_names(
-            self._train_program.desc,
-            self.get_forward_end_op_idx(self._train_program),
-            len(self._outputs.var_ids),
-        )
-
     @property
     def program(self):
         """
@@ -649,7 +637,33 @@ def _append_backward_desc(self, main_program):
         if targets:
             start_idx = len(program.block(0).ops) + len(self._outputs.tolist())
             with backend_guard(self._backend):
-                backward.gradients(targets=targets, inputs=[])
+                check_type(
+                    targets,
+                    'targets',
+                    (framework.Variable, list, tuple),
+                    'paddle.static.gradients',
+                )
+                grad_info_map = backward.calc_gradient_helper(
+                    targets=targets, inputs=[]
+                )
+
+                x_vars = [
+                    program.block(0).var(var.name)
+                    for var in self._inputs
+                    if isinstance(var, framework.Variable)
+                ]
+                param_vars = [
+                    program.block(0).var(param.name) for param in self._params
+                ]
+                out_vars = [
+                    program.block(0).var(var.name)
+                    for var in self._outputs
+                    if isinstance(var, framework.Variable)
+                ]
+
+                self._grad_var_names = construct_grad_names(
+                    grad_info_map, x_vars, param_vars, out_vars
+                )
 
             if self._hooker:
                 program, start_idx = self._hooker.after_append_backward(
@@ -720,9 +734,11 @@ def _prepare_attributes(self):
             attrs.extend(
                 (
                     'param_grad_names',
-                    self._param_grad_names,
+                    self._grad_var_names.get('param', []),
                     'out_grad_names',
-                    self._out_grad_names,
+                    self._grad_var_names.get('out', []),
+                    'x_grad_names',
+                    self._grad_var_names.get('x', []),
                 )
             )
         if self._cuda_graph_capture_mode:
@@ -761,9 +777,9 @@ def _get_forward_backward_program_form(
         backward_end_op_index = whole_program.desc.block(0).op_size()
         # For Backward process in CINN, all param@GRAD shoule be skipped for GC, because
         # they will be shared in scope and used by optimizer.
-        backward_skip_vars = (
-            self._parse_skip_gc_vars(whole_program) + self._param_grad_names
-        )
+        backward_skip_vars = self._parse_skip_gc_vars(
+            whole_program
+        ) + self._grad_var_names.get('param', [])
         backward_builded_program = add_build_strategy_for(
             whole_program,
             backward_start_op_index,
diff --git a/python/paddle/jit/dy2static/utils.py b/python/paddle/jit/dy2static/utils.py
index df892ab2a06ef..1dbbf3669ca4b 100644
--- a/python/paddle/jit/dy2static/utils.py
+++ b/python/paddle/jit/dy2static/utils.py
@@ -32,7 +32,7 @@
 
 import paddle
 from paddle import fluid  # noqa: F401
-from paddle.fluid import core, unique_name
+from paddle.fluid import backward, core, framework, unique_name
 from paddle.fluid.data_feeder import convert_dtype
 from paddle.fluid.layer_helper import LayerHelper
 from paddle.fluid.wrapped_decorator import signature_safe_contextmanager
@@ -1462,61 +1462,6 @@ def create_name_str(name_ids):
     return "(%s, )" % ','.join(names_str)
 
 
-def _param_grad_names(program_desc, params):
-    """
-    Parse PARAM@GARD name from original train and infer program.
-    """
-    names = []
-    # NOTE: `names` and `params` must be in the same order so that
-    # the param grad name can be set correctly in the run_program.
-    for param in params:
-        candidate = []
-        for var in program_desc.block(0).all_vars():
-            var_name = var.name()
-            if param.name not in var_name:
-                continue
-            suf_count = var_name.count(GRAD_SUFFIX)
-            if suf_count > 0:
-                suffix = param.name + GRAD_SUFFIX * suf_count
-                pre_count = var_name.count(GRAD_PREFIX)
-                if GRAD_PREFIX * pre_count + suffix == var_name:
-                    candidate.append(var_name)
-
-        if candidate:
-            names.append(
-                max(
-                    candidate,
-                    key=lambda name: name.count(GRAD_PREFIX)
-                    if GRAD_PREFIX in name
-                    else name.count(GRAD_SUFFIX),
-                )
-            )
-        else:
-            names.append(param.name + GRAD_SUFFIX)
-    return names
-
-
-def _out_grad_names(program_desc, fwd_end_op_index, out_size):
-    """
-    Parse Out@GARD name from original train and infer program.
-    """
-    names = []
-    for i in range(
-        fwd_end_op_index,
-        min(fwd_end_op_index + out_size, program_desc.block(0).op_size()),
-    ):
-        op = program_desc.block(0).op(i)
-        # If prim forward op, fill_any_like will be decomposite as fill_constant.
-        if core._is_fwd_prim_enabled():
-            target = ('fill_any_like', 'fill_constant')
-        else:
-            target = 'fill_any_like'
-        if op.type() in target:
-            var_name = op.output('Out')[0]
-            names.append(var_name)
-    return names
-
-
 def prim_or_cinn_is_enabled(build_strategy, backend):
     if backend == 'CINN':
         return True
@@ -1571,3 +1516,19 @@ def backend_guard(backend):
     finally:
         core._set_prim_forward_enabled(orign_fwd)
         core._set_prim_backward_enabled(orign_bwd)
+
+
+def construct_grad_names(grad_info_map, x_vars, param_vars, out_vars):
+    grad_var_names = {}
+    fn = (
+        lambda grad_var: grad_var.name
+        if isinstance(grad_var, framework.Variable)
+        else framework.EMPTY_VAR_NAME
+    )
+    x_grad_vars = backward._get_grad_vars(grad_info_map, x_vars)
+    grad_var_names['x'] = list(map(fn, x_grad_vars))
+    param_grad_vars = backward._get_grad_vars(grad_info_map, param_vars)
+    grad_var_names['param'] = list(map(fn, param_grad_vars))
+    out_grad_vars = backward._get_grad_vars(grad_info_map, out_vars)
+    grad_var_names['out'] = list(map(fn, out_grad_vars))
+    return grad_var_names
diff --git a/python/paddle/jit/translated_layer.py b/python/paddle/jit/translated_layer.py
index c56608d6bb66e..9f2fe5ee41b0d 100644
--- a/python/paddle/jit/translated_layer.py
+++ b/python/paddle/jit/translated_layer.py
@@ -20,16 +20,16 @@
 import paddle
 from paddle import _legacy_C_ops
 from paddle.fluid import backward, core, framework, unique_name
+from paddle.fluid.data_feeder import check_type
 from paddle.fluid.dygraph.base import switch_to_static_graph
 from paddle.fluid.framework import OpProtoHolder, _non_static_mode
 from paddle.jit.dy2static.partial_program import (
     LazyInitialized,
     add_build_strategy_for,
 )
+from paddle.jit.dy2static.utils import construct_grad_names
 from paddle.nn.layer import layers
 
-from .dy2static.utils import _out_grad_names, _param_grad_names
-
 __all__ = []
 
 INFER_MODEL_SUFFIX = ".pdmodel"
@@ -349,6 +349,7 @@ def __init__(self, program_desc):
         self._train_program_desc = self._append_backward_desc(
             self._infer_program_desc
         )
+        self._grad_var_names = {}
 
     # forward:
     @switch_to_static_graph
@@ -419,6 +420,10 @@ def double_grad_descs(self):
     def scope(self):
         return self._inner_scope
 
+    @property
+    def grad_var_names(self):
+        return self._grad_var_names
+
     def _preprocess(self, program_desc):
         # rename persistable variables of 'program_desc'
         list_persistable_var = _get_persistable_var_names(program_desc)
@@ -599,7 +604,29 @@ def _append_backward_desc(self, infer_program_desc):
             targets.append(program.global_block().var(out.name()))
 
         # 3. append backward
-        backward.gradients(targets=targets, inputs=[])
+        check_type(
+            targets,
+            'targets',
+            (framework.Variable, list, tuple),
+            'paddle.static.gradients',
+        )
+        grad_info_map = backward.calc_gradient_helper(
+            targets=targets, inputs=[]
+        )
+        x_vars = [
+            program.block(0).var(desc.name()) for desc in self._input_descs
+        ]
+        param_vars = [
+            program.block(0).var(name) for name in self._persistable_names
+        ]
+        out_vars = [
+            program.block(0).var(desc.name()) for desc in self._output_descs
+        ]
+
+        self._grad_var_names = construct_grad_names(
+            grad_info_map, x_vars, param_vars, out_vars
+        )
+
         return program.desc
 
 
@@ -964,9 +991,11 @@ def _run_dygraph(instance, input, program_holder):
         attrs.extend(
             (
                 'param_grad_names',
-                _param_grad_names(trace_program, persistable_vars),
+                program_holder.grad_var_names.get('param', []),
                 'out_grad_names',
-                _out_grad_names(trace_program, end_op_index, len(output_vars)),
+                program_holder.grad_var_names.get('out', []),
+                'x_grad_names',
+                program_holder.grad_var_names.get('x', []),
             )
         )
 
diff --git a/test/dygraph_to_static/test_gradname_parse.py b/test/dygraph_to_static/test_gradname_parse.py
index d51bcbf068423..743f92f758f4f 100644
--- a/test/dygraph_to_static/test_gradname_parse.py
+++ b/test/dygraph_to_static/test_gradname_parse.py
@@ -15,6 +15,8 @@
 
 import unittest
 
+import numpy as np
+
 import paddle
 from paddle import ParamAttr
 from paddle.nn import BatchNorm, Linear
@@ -65,5 +67,99 @@ def test_grad_name_parse(self):
         opt.minimize(loss)
 
 
+def tanh_high_order_grad(x):
+    y = paddle.tanh(x)
+    return paddle.grad(y, x, create_graph=True)[0]
+
+
+class TestTanhHighOrderGrad(unittest.TestCase):
+    def setUp(self):
+        self.func = tanh_high_order_grad
+
+        x1 = paddle.ones((1,))
+        x1.stop_gradient = False
+        self.dy_input = (x1,)
+        self.dy_grad_input = (x1,)
+
+        x2 = paddle.ones((1,))
+        x2.stop_gradient = False
+        self.dy2st_input = (x2,)
+        self.dy2st_grad_input = (x2,)
+
+    def test_run(self):
+        try:
+            dy_out = self.func(*self.dy_input)
+            dy_grad = paddle.grad(dy_out, self.dy_grad_input)
+        except:
+            dy_grad = [None for i in self.dy_grad_input]
+        dy_grad = [
+            t.numpy() if isinstance(t, paddle.Tensor) else t for t in dy_grad
+        ]
+
+        dy2st_out = paddle.jit.to_static(self.func)(*self.dy2st_input)
+        dy2st_grad = paddle.grad(dy2st_out, self.dy2st_grad_input)
+        dy2st_grad = [
+            t.numpy() if isinstance(t, paddle.Tensor) else t for t in dy_grad
+        ]
+        np.testing.assert_equal(dy_grad, dy2st_grad)
+
+        dy_input_grad = [
+            t.grad.numpy() if isinstance(t.grad, paddle.Tensor) else None
+            for t in self.dy_input
+        ]
+        dy2st_input_grad = [
+            t.grad.numpy() if isinstance(t.grad, paddle.Tensor) else None
+            for t in self.dy2st_input
+        ]
+        np.testing.assert_equal(dy_input_grad, dy2st_input_grad)
+
+
+def matmul_high_order_grad(x, y):
+    z = paddle.matmul(x, y)
+    g = paddle.grad(z, [x, y], create_graph=False)
+    return g[0]
+
+
+class TestMatMulHighOrderGrad1(TestTanhHighOrderGrad):
+    def setUp(self):
+        self.func = matmul_high_order_grad
+
+        x1 = paddle.ones([1])
+        x1.stop_gradient = False
+        y1 = paddle.ones([1])
+        y1.stop_gradient = False
+        self.dy_input = (x1, y1)
+        self.dy_grad_input = (x1,)
+
+        x2 = paddle.ones([1])
+        x2.stop_gradient = False
+        y2 = paddle.ones([1])
+        y2.stop_gradient = False
+        self.dy2st_input = (x2, y2)
+        self.dy2st_grad_input = (x2,)
+
+
+class TestMatMulHighOrderGrad2(TestTanhHighOrderGrad):
+    def setUp(self):
+        self.func = matmul_high_order_grad
+
+        x = np.random.randn(5, 5)
+        y = np.random.randn(5, 5)
+
+        x1 = paddle.to_tensor(x)
+        x1.stop_gradient = False
+        y1 = paddle.to_tensor(y)
+        y1.stop_gradient = True
+        self.dy_input = (x1, y1)
+        self.dy_grad_input = (x1,)
+
+        x2 = paddle.to_tensor(x)
+        x2.stop_gradient = False
+        y2 = paddle.to_tensor(y)
+        y2.stop_gradient = True
+        self.dy2st_input = (x2, y2)
+        self.dy2st_grad_input = (x2,)
+
+
 if __name__ == "__main__":
     unittest.main()

From 1c97aa69632f2d8aae7622a178c225f8164bbe26 Mon Sep 17 00:00:00 2001
From: zhupengyang <zhu_py@qq.com>
Date: Thu, 27 Apr 2023 19:15:17 +0800
Subject: [PATCH 116/405] xpu quant weight only (#53306)

---
 .../fused_multi_transformer_xpu_quant_pass.cc | 65 ++++++++++---------
 paddle/fluid/framework/ir/xpu/pass_utils.cc   |  7 ++
 paddle/fluid/framework/ir/xpu/quant_utils.cc  | 13 ++++
 paddle/fluid/inference/analysis/argument.h    |  6 ++
 .../inference/analysis/ir_pass_manager.cc     |  8 +++
 paddle/fluid/inference/api/analysis_config.cc | 21 ++++++
 .../fluid/inference/api/analysis_predictor.cc |  4 ++
 .../inference/api/paddle_analysis_config.h    | 14 ++++
 paddle/fluid/pybind/inference_api.cc          |  5 ++
 9 files changed, 113 insertions(+), 30 deletions(-)

diff --git a/paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_quant_pass.cc b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_quant_pass.cc
index fab466a50637e..b9868a81135d4 100644
--- a/paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_quant_pass.cc
+++ b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_quant_pass.cc
@@ -280,6 +280,8 @@ int FusedMultiTransformerXPUQuantPass::ApplyImpl(ir::Graph* graph,
                                                  with_time_step,
                                                  with_seq_lengths,
                                                  with_src_mask);
+  int quant_weight_bits =
+      Has("quant_weight_bits") ? Get<int>("quant_weight_bits") : -1;
 
   int found_subgraph_count = 0;
   auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph,
@@ -312,36 +314,39 @@ int FusedMultiTransformerXPUQuantPass::ApplyImpl(ir::Graph* graph,
     // quant weight nodes
     // w_nodes_vec: [QKVW, OutLinearW, FFN1Weight, FFN2Weight]
     std::vector<std::vector<Node*>> w_nodes_vec(4);
-    std::vector<std::vector<Node*>> w_int16_nodes_vec(4);
+    std::vector<std::vector<Node*>> w_intx_nodes_vec(4);
     std::vector<std::vector<Node*>> w_max_nodes_vec(4);
-    std::vector<std::vector<std::string>> w_int16_names_vec(4);
+    std::vector<std::vector<std::string>> w_intx_names_vec(4);
     std::vector<std::vector<std::string>> w_max_names_vec(4);
     auto quant_func = [&](const std::string& input_name,
                           std::vector<Node*>* w_nodes,
-                          std::vector<Node*>* w_int16_nodes,
+                          std::vector<Node*>* w_intx_nodes,
                           std::vector<Node*>* w_max_nodes,
-                          std::vector<std::string>* w_int16_names,
+                          std::vector<std::string>* w_intx_names,
                           std::vector<std::string>* w_max_names,
                           bool need_transpose) {
-      typedef int16_t TW;
-
       auto w_names = fused_mt->Op()->Input(input_name);
       for (auto w_name : w_names) {
         Node* w_node = FindNodeWithName(graph, w_name);
-        Node* w_int16 = nullptr;
+        Node* w_intx = nullptr;
         Node* w_max = nullptr;
         PADDLE_ENFORCE_NE(
             w_node,
             nullptr,
             platform::errors::Fatal("w node should not be nullptr"));
-        PrepareWeight<TW>(
-            graph, scope, block, w_node, &w_int16, &w_max, need_transpose);
+        if (quant_weight_bits == 8) {
+          PrepareWeight<int8_t>(
+              graph, scope, block, w_node, &w_intx, &w_max, need_transpose);
+        } else {
+          PrepareWeight<int16_t>(
+              graph, scope, block, w_node, &w_intx, &w_max, need_transpose);
+        }
         w_nodes->push_back(w_node);
-        w_int16_nodes->push_back(w_int16);
+        w_intx_nodes->push_back(w_intx);
         w_max_nodes->push_back(w_max);
       }
       for (size_t i = 0; i < w_names.size(); ++i) {
-        w_int16_names->push_back(w_int16_nodes->at(i)->Name());
+        w_intx_names->push_back(w_intx_nodes->at(i)->Name());
         w_max_names->push_back(w_max_nodes->at(i)->Name());
       }
       PADDLE_ENFORCE_EQ(
@@ -353,11 +358,11 @@ int FusedMultiTransformerXPUQuantPass::ApplyImpl(ir::Graph* graph,
               static_cast<int>(w_nodes->size())));
       PADDLE_ENFORCE_EQ(
           w_names.size(),
-          w_int16_nodes->size(),
+          w_intx_nodes->size(),
           platform::errors::Fatal(
-              "The size of w_names(%d) should be equal to w_int16_nodes(%d)",
+              "The size of w_names(%d) should be equal to w_intx_nodes(%d)",
               static_cast<int>(w_names.size()),
-              static_cast<int>(w_int16_nodes->size())));
+              static_cast<int>(w_intx_nodes->size())));
       PADDLE_ENFORCE_EQ(
           w_names.size(),
           w_max_nodes->size(),
@@ -367,11 +372,11 @@ int FusedMultiTransformerXPUQuantPass::ApplyImpl(ir::Graph* graph,
               static_cast<int>(w_max_nodes->size())));
       PADDLE_ENFORCE_EQ(
           w_names.size(),
-          w_int16_names->size(),
+          w_intx_names->size(),
           platform::errors::Fatal(
-              "The size of w_names(%d) should be equal to w_int16_names(%d)",
+              "The size of w_names(%d) should be equal to w_intx_names(%d)",
               static_cast<int>(w_names.size()),
-              static_cast<int>(w_int16_names->size())));
+              static_cast<int>(w_intx_names->size())));
       PADDLE_ENFORCE_EQ(
           w_names.size(),
           w_max_names->size(),
@@ -382,30 +387,30 @@ int FusedMultiTransformerXPUQuantPass::ApplyImpl(ir::Graph* graph,
     };
     quant_func("QKVW",
                &(w_nodes_vec[0]),
-               &(w_int16_nodes_vec[0]),
+               &(w_intx_nodes_vec[0]),
                &(w_max_nodes_vec[0]),
-               &(w_int16_names_vec[0]),
+               &(w_intx_names_vec[0]),
                &(w_max_names_vec[0]),
                false);
     quant_func("OutLinearW",
                &(w_nodes_vec[1]),
-               &(w_int16_nodes_vec[1]),
+               &(w_intx_nodes_vec[1]),
                &(w_max_nodes_vec[1]),
-               &(w_int16_names_vec[1]),
+               &(w_intx_names_vec[1]),
                &(w_max_names_vec[1]),
                true);
     quant_func("FFN1Weight",
                &(w_nodes_vec[2]),
-               &(w_int16_nodes_vec[2]),
+               &(w_intx_nodes_vec[2]),
                &(w_max_nodes_vec[2]),
-               &(w_int16_names_vec[2]),
+               &(w_intx_names_vec[2]),
                &(w_max_names_vec[2]),
                true);
     quant_func("FFN2Weight",
                &(w_nodes_vec[3]),
-               &(w_int16_nodes_vec[3]),
+               &(w_intx_nodes_vec[3]),
                &(w_max_nodes_vec[3]),
-               &(w_int16_names_vec[3]),
+               &(w_intx_names_vec[3]),
                &(w_max_names_vec[3]),
                true);
 
@@ -482,13 +487,13 @@ int FusedMultiTransformerXPUQuantPass::ApplyImpl(ir::Graph* graph,
                                     name_caches.at("CacheKVOut"));
     fused_mt_xpu_op_desc->SetOutput("out", name_caches.at("Out"));
 
-    fused_mt_xpu_op_desc->SetInput("qkvw", w_int16_names_vec[0]);
+    fused_mt_xpu_op_desc->SetInput("qkvw", w_intx_names_vec[0]);
     fused_mt_xpu_op_desc->SetInput("qkvw_max", w_max_names_vec[0]);
-    fused_mt_xpu_op_desc->SetInput("out_linear_w", w_int16_names_vec[1]);
+    fused_mt_xpu_op_desc->SetInput("out_linear_w", w_intx_names_vec[1]);
     fused_mt_xpu_op_desc->SetInput("out_linear_wmax", w_max_names_vec[1]);
-    fused_mt_xpu_op_desc->SetInput("ffn1_weight", w_int16_names_vec[2]);
+    fused_mt_xpu_op_desc->SetInput("ffn1_weight", w_intx_names_vec[2]);
     fused_mt_xpu_op_desc->SetInput("ffn1_weight_max", w_max_names_vec[2]);
-    fused_mt_xpu_op_desc->SetInput("ffn2_weight", w_int16_names_vec[3]);
+    fused_mt_xpu_op_desc->SetInput("ffn2_weight", w_intx_names_vec[3]);
     fused_mt_xpu_op_desc->SetInput("ffn2_weight_max", w_max_names_vec[3]);
     if (!fused_mt_xpu_op_desc->HasAttr("rotary_emb_dims")) {
       fused_mt_xpu_op_desc->SetAttr("rotary_emb_dims", 0);
@@ -501,7 +506,7 @@ int FusedMultiTransformerXPUQuantPass::ApplyImpl(ir::Graph* graph,
     }
     // link int16 format of QKVW/OutLinearW/FFN1Weight/FFN2Weight to
     // fused_mt_xpu
-    for (auto nodes : w_int16_nodes_vec) {
+    for (auto nodes : w_intx_nodes_vec) {
       for (auto node : nodes) {
         IR_NODE_LINK_TO(node, fused_mt);
       }
diff --git a/paddle/fluid/framework/ir/xpu/pass_utils.cc b/paddle/fluid/framework/ir/xpu/pass_utils.cc
index bec10d421c7c2..aaa117d363a5f 100644
--- a/paddle/fluid/framework/ir/xpu/pass_utils.cc
+++ b/paddle/fluid/framework/ir/xpu/pass_utils.cc
@@ -193,6 +193,13 @@ template void PrepareWeight<int16_t>(Graph* graph,
                                      Node** dst,
                                      Node** dst_max,
                                      bool transpose);
+template void PrepareWeight<int8_t>(Graph* graph,
+                                    Scope* scope,
+                                    BlockDesc* block,
+                                    Node* src,
+                                    Node** dst,
+                                    Node** dst_max,
+                                    bool transpose);
 
 void PrepareBias(
     Graph* graph, Scope* scope, BlockDesc* block, Node* src, Node** dst) {
diff --git a/paddle/fluid/framework/ir/xpu/quant_utils.cc b/paddle/fluid/framework/ir/xpu/quant_utils.cc
index ae697708ddcd7..d075d42d29506 100644
--- a/paddle/fluid/framework/ir/xpu/quant_utils.cc
+++ b/paddle/fluid/framework/ir/xpu/quant_utils.cc
@@ -207,6 +207,16 @@ void QuantFP32ToIntX<int16_t>(const float* src_ptr,
   }
 }
 
+template <>
+void QuantFP32ToIntX<int8_t>(const float* src_ptr,
+                             int8_t* dst_ptr,
+                             float max_val,
+                             int numel) {
+  for (int i = 0; i < numel; i++) {
+    dst_ptr[i] = Fp32ToIntx<int8_t, 127>(src_ptr[i], max_val);
+  }
+}
+
 template <typename T>
 void PrepareWeight(phi::DenseTensor* weight,
                    phi::DenseTensor* weight_max,
@@ -253,6 +263,9 @@ void PrepareWeight(phi::DenseTensor* weight,
 template void PrepareWeight<int16_t>(phi::DenseTensor* weight,
                                      phi::DenseTensor* weight_max,
                                      bool transpose);
+template void PrepareWeight<int8_t>(phi::DenseTensor* weight,
+                                    phi::DenseTensor* weight_max,
+                                    bool transpose);
 
 }  // namespace ir
 }  // namespace framework
diff --git a/paddle/fluid/inference/analysis/argument.h b/paddle/fluid/inference/analysis/argument.h
index 014f4e828efc7..88e6749223008 100644
--- a/paddle/fluid/inference/analysis/argument.h
+++ b/paddle/fluid/inference/analysis/argument.h
@@ -289,6 +289,12 @@ struct Argument {
   DECL_ARGUMENT_FIELD(xpu_adaptive_seqlen, XpuAdaptiveSeqlen, bool);
   DECL_ARGUMENT_FIELD(xpu_device_id, XpuDeviceId, int);
   DECL_ARGUMENT_FIELD(xpu_enable_multi_stream, XpuEnableMultiStream, bool);
+  DECL_ARGUMENT_FIELD(xpu_quant_post_dynamic_weight_bits,
+                      XpuQuantPostDynamicWeightBits,
+                      int);
+  DECL_ARGUMENT_FIELD(xpu_quant_post_dynamic_op_types,
+                      XpuQuantPostDynamicOpTypss,
+                      std::vector<std::string>);
 
   DECL_ARGUMENT_FIELD(use_opencl, UseOpenCL, bool);
 
diff --git a/paddle/fluid/inference/analysis/ir_pass_manager.cc b/paddle/fluid/inference/analysis/ir_pass_manager.cc
index c3c2fb6d80ffd..1d87edcd3404c 100644
--- a/paddle/fluid/inference/analysis/ir_pass_manager.cc
+++ b/paddle/fluid/inference/analysis/ir_pass_manager.cc
@@ -308,6 +308,14 @@ void IRPassManager::CreatePasses(Argument *argument,
       }
       bool use_fc_padding = !fc_mkldnn_pass && argument->use_fc_padding();
       pass->Set("use_fc_padding", new bool(use_fc_padding));
+    } else if (pass_name == "fused_multi_transformer_xpu_quant_pass") {
+      auto op_types = argument->xpu_quant_post_dynamic_op_types();
+      if (std::count(op_types.begin(),
+                     op_types.end(),
+                     "fused_multi_transformer") > 0) {
+        pass->Set("quant_weight_bits",
+                  new int(argument->xpu_quant_post_dynamic_weight_bits()));
+      }
     }
     pre_pass = pass_name;
 
diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc
index afb2dcd981fa8..3fa947ce27daf 100644
--- a/paddle/fluid/inference/api/analysis_config.cc
+++ b/paddle/fluid/inference/api/analysis_config.cc
@@ -196,6 +196,14 @@ void AnalysisConfig::SetXpuDeviceId(int device_id) {
   Update();
 }
 
+void AnalysisConfig::SetXpuConfig(
+    int quant_post_dynamic_weight_bits,
+    const std::vector<std::string> &quant_post_dynamic_op_types) {
+  xpu_quant_post_dynamic_weight_bits_ = quant_post_dynamic_weight_bits;
+  xpu_quant_post_dynamic_op_types_ = quant_post_dynamic_op_types;
+  Update();
+}
+
 void AnalysisConfig::EnableCustomDevice(const std::string &device_type,
                                         int device_id,
                                         Precision precision_mode) {
@@ -489,6 +497,8 @@ AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) {
   CP_MEMBER(xpu_precision_);
   CP_MEMBER(xpu_adaptive_seqlen_);
   CP_MEMBER(xpu_enable_multi_stream_);
+  CP_MEMBER(xpu_quant_post_dynamic_weight_bits_);
+  CP_MEMBER(xpu_quant_post_dynamic_op_types_);
 
   // Lite OpenCL Related
   CP_MEMBER(use_opencl_);
@@ -1091,6 +1101,10 @@ std::string AnalysisConfig::SerializeInfoCache() {
   ss << xpu_precision_;
   ss << xpu_adaptive_seqlen_;
   ss << xpu_enable_multi_stream_;
+  ss << xpu_quant_post_dynamic_weight_bits_;
+  for (auto op_type : xpu_quant_post_dynamic_op_types_) {
+    ss << op_type;
+  }
 
   ss << use_npu_;
   ss << npu_device_id_;
@@ -1331,6 +1345,13 @@ std::string AnalysisConfig::Summary() {
     os.InsertRow({"xpu_device_id", std::to_string(xpu_device_id_)});
     os.InsertRow(
         {"xpu_l3_workspace_size", std::to_string(xpu_l3_workspace_size_)});
+    os.InsertRow({"xpu_quant_post_dynamic_weight_bits",
+                  std::to_string(xpu_quant_post_dynamic_weight_bits_)});
+    std::vector<std::string> op_types{"xpu_quant_post_dynamic_op_types"};
+    for (auto op_type : xpu_quant_post_dynamic_op_types_) {
+      op_types.push_back(op_type);
+    }
+    os.InsertRow(op_types);
   }
   os.InsetDivider();
 
diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc
index 219c3c2754c68..4ef15fbbc15dc 100644
--- a/paddle/fluid/inference/api/analysis_predictor.cc
+++ b/paddle/fluid/inference/api/analysis_predictor.cc
@@ -1426,6 +1426,10 @@ void AnalysisPredictor::PrepareArgument() {
   argument_->SetXpuAdaptiveSeqlen(config_.xpu_adaptive_seqlen_);
   argument_->SetXpuDeviceId(config_.xpu_device_id_);
   argument_->SetXpuEnableMultiStream(config_.xpu_enable_multi_stream_);
+  argument_->SetXpuQuantPostDynamicWeightBits(
+      config_.xpu_quant_post_dynamic_weight_bits_);
+  argument_->SetXpuQuantPostDynamicOpTypss(
+      config_.xpu_quant_post_dynamic_op_types_);
 #endif
 
   auto *pass_builder = config_.pass_builder();
diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h
index 10b92debdace1..3300fe8a9b8dc 100644
--- a/paddle/fluid/inference/api/paddle_analysis_config.h
+++ b/paddle/fluid/inference/api/paddle_analysis_config.h
@@ -288,6 +288,18 @@ struct PD_INFER_DECL AnalysisConfig {
                  bool adaptive_seqlen = false,
                  bool enable_multi_stream = false);
 
+  ///
+  /// \brief configs of XPU
+  ///
+  /// \param quant_post_dynamic_weight_bits Weight bits used in dynamic post
+  /// quantization. Optional value: -1, 8, 16. Default value is -1, means using
+  /// the recommended way. \param quant_post_dynamic_op_types Ops used in
+  /// dynamic post quantization.
+  ///
+  void SetXpuConfig(
+      int quant_post_dynamic_weight_bits = -1,
+      const std::vector<std::string>& quant_post_dynamic_op_types = {});
+
   ///
   /// \brief configs of IPU
   ///
@@ -1181,6 +1193,8 @@ struct PD_INFER_DECL AnalysisConfig {
   std::string xpu_precision_;
   bool xpu_adaptive_seqlen_;
   bool xpu_enable_multi_stream_;
+  int xpu_quant_post_dynamic_weight_bits_{-1};
+  std::vector<std::string> xpu_quant_post_dynamic_op_types_;
 
   // LITE OPENCL SETTINGS
   bool use_opencl_{false};
diff --git a/paddle/fluid/pybind/inference_api.cc b/paddle/fluid/pybind/inference_api.cc
index bce79c27c0585..e00c22423eb28 100644
--- a/paddle/fluid/pybind/inference_api.cc
+++ b/paddle/fluid/pybind/inference_api.cc
@@ -767,6 +767,11 @@ void BindAnalysisConfig(py::module *m) {
       .def("set_xpu_device_id",
            &AnalysisConfig::SetXpuDeviceId,
            py::arg("device_id") = 0)
+      .def(
+          "set_xpu_config",
+          &AnalysisConfig::SetXpuConfig,
+          py::arg("quant_post_dynamic_weight_bits") = -1,
+          py::arg("quant_post_dynamic_op_types") = std::vector<std::string>({}))
       .def("enable_custom_device",
            &AnalysisConfig::EnableCustomDevice,
            py::arg("device_type"),

From 182b6f8385b17f880652782924e131bbfcc7f9f7 Mon Sep 17 00:00:00 2001
From: Yuanle Liu <yuanlehome@163.com>
Date: Thu, 27 Apr 2023 19:44:10 +0800
Subject: [PATCH 117/405] scale trt converter support int64 (#53388)

---
 paddle/fluid/inference/tensorrt/convert/scale_op.cc | 2 +-
 paddle/fluid/inference/tensorrt/op_teller.cc        | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/convert/scale_op.cc b/paddle/fluid/inference/tensorrt/convert/scale_op.cc
index c6ed50501f766..054964419d2d9 100644
--- a/paddle/fluid/inference/tensorrt/convert/scale_op.cc
+++ b/paddle/fluid/inference/tensorrt/convert/scale_op.cc
@@ -19,7 +19,7 @@ namespace inference {
 namespace tensorrt {
 
 /*
- * ConcatOp
+ * Scale Op
  */
 class ScaleOpConverter : public OpConverter {
  public:
diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index bb0fbdf6ca848..6a94e14d7e68b 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -1184,10 +1184,12 @@ struct SimpleOpTypeSetTeller : public Teller {
           return false;
         }
       } else {
-        // At present, only support float32 or float16 or int32 into trt.
+        // At present, only support float32 or float16 or int32 or int64 into
+        // trt.
         if (!(dtype == framework::proto::VarType::FP32 ||
               dtype == framework::proto::VarType::FP16 ||
-              dtype == framework::proto::VarType::INT32)) {
+              dtype == framework::proto::VarType::INT32 ||
+              dtype == framework::proto::VarType::INT64)) {
           return false;
         }
       }

From 6b9b49ceac9d5b15e41c90b8eb70f4caa5fb6d3f Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Thu, 27 Apr 2023 19:48:29 +0800
Subject: [PATCH 118/405] add scale for AF2 flash_attn, much thanks to xreki
 and shaojie for debug these codes

---
 paddle/fluid/operators/fused/fused_gate_attention.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index cf4fbdd3a1739..5dd6f26a802d6 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -1010,7 +1010,7 @@ class FlashAttnWithGating {
 
     // 7. flas_attn part one, get temp worksapce size.
     float p_dropout = 0.f;
-    float softmax_scale = static_cast<float>(1);
+    float softmax_scale = static_cast<float>(1.0f / std::sqrt(head_size_));
     cudaStream_t stream = dev_ctx_.stream();
     uint64_t workspace_size;
     bool succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
@@ -1212,7 +1212,7 @@ class FlashAttnWithGating {
     // 7. flas_attn part one, get temp worksapce size.
     uint64_t workspace_size;
     float p_dropout = 0.f;
-    float softmax_scale = static_cast<float>(1);
+    float softmax_scale = static_cast<float>(1.0f / std::sqrt(head_size_));
     cudaStream_t stream = dev_ctx_.stream();
     int num_splits = 0;  // 0 for an internal heuristic, which is optimal
     bool succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(

From fe053396734c58e445e3e8c9973e6ca8c125682a Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Thu, 27 Apr 2023 21:23:35 +0800
Subject: [PATCH 119/405] =?UTF-8?q?[phi]=20Move=20sequence=5Fpool=20to=20p?=
 =?UTF-8?q?hi=20-=20Step=203=20=EF=BC=9Asequence=5Fpool=5Fgrad=5Fop=20(#52?=
 =?UTF-8?q?680)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* [phi] move sequence_pool kernel to phi

* mv kernels impl

* fix parameter error

* clean include

* fix compat filename

* [phi] move fluid sequence_pool_grad to phi

* [phi][compat] sig rm GradVarName

* [phi] fix sequence_pool out type

* [phi] rm impl, add const string

* [phi] fix const str

* fix sequence_pooling cmake

* [phi] mv sequence_pooling_test

* [phi] fix grad sig

* [phi] fix sequence_pool is_test error

* [phi] fix sequence_pooling gpu include

* [phi] mv to impl

* [phi] fix SequencePoolFunctor cu include

* [phi] modify out max_index int32_t

* [phi] add pooltype mapping determine

* [phi] fix sequence_pool_sig

* [phi] fix sequence_pool_sig sum

* [phi] try ci

* [phi] fix max_index optional
---
 paddle/fluid/operators/math/CMakeLists.txt    |  1 -
 .../sequence_ops/sequence_pool_op.cc          | 12 +----
 .../operators/sequence_ops/sequence_pool_op.h | 49 -------------------
 .../sequence_ops/unity_build_rule.cmake       |  1 -
 .../kernels/cpu/sequence_pool_grad_kernel.cc  | 26 ++++++++++
 .../kernels/gpu/sequence_pool_grad_kernel.cu} | 14 ++++--
 .../impl/sequence_pool_grad_kernel_impl.h     | 39 +++++++++++++++
 .../kernels/impl/sequence_pool_kernel_impl.h  |  2 +-
 .../phi/kernels/sequence_pool_grad_kernel.h   | 30 ++++++++++++
 paddle/phi/ops/compat/sequence_pool_sig.cc    | 10 ++++
 10 files changed, 116 insertions(+), 68 deletions(-)
 delete mode 100644 paddle/fluid/operators/sequence_ops/sequence_pool_op.h
 create mode 100644 paddle/phi/kernels/cpu/sequence_pool_grad_kernel.cc
 rename paddle/{fluid/operators/sequence_ops/sequence_pool_op.cu => phi/kernels/gpu/sequence_pool_grad_kernel.cu} (59%)
 create mode 100644 paddle/phi/kernels/impl/sequence_pool_grad_kernel_impl.h
 create mode 100644 paddle/phi/kernels/sequence_pool_grad_kernel.h

diff --git a/paddle/fluid/operators/math/CMakeLists.txt b/paddle/fluid/operators/math/CMakeLists.txt
index 405661f3a1ab9..c67703ba52814 100644
--- a/paddle/fluid/operators/math/CMakeLists.txt
+++ b/paddle/fluid/operators/math/CMakeLists.txt
@@ -14,7 +14,6 @@ math_library(sample_prob)
 math_library(sampler DEPS generator)
 
 # math_library(math_function DEPS blas dense_tensor tensor)
-
 if(WITH_XPU)
   math_library(beam_search DEPS math_function beam_search_xpu)
 else()
diff --git a/paddle/fluid/operators/sequence_ops/sequence_pool_op.cc b/paddle/fluid/operators/sequence_ops/sequence_pool_op.cc
index f51cc7d903664..d616bca2c4e3b 100644
--- a/paddle/fluid/operators/sequence_ops/sequence_pool_op.cc
+++ b/paddle/fluid/operators/sequence_ops/sequence_pool_op.cc
@@ -12,10 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/operators/sequence_ops/sequence_pool_op.h"
-
-#include <memory>
-#include <string>
+#include "paddle/fluid/framework/op_registry.h"
 
 namespace paddle {
 namespace operators {
@@ -196,10 +193,3 @@ REGISTER_OPERATOR(sequence_pool,
 REGISTER_OPERATOR(sequence_pool_grad,
                   ops::SequencePoolGradOp,
                   ops::SequencePoolGradOpNoNeedBufferVarsInferer);
-
-PD_REGISTER_STRUCT_KERNEL(sequence_pool_grad,
-                          CPU,
-                          ALL_LAYOUT,
-                          ops::SequencePoolGradKernel,
-                          float,
-                          double) {}
diff --git a/paddle/fluid/operators/sequence_ops/sequence_pool_op.h b/paddle/fluid/operators/sequence_ops/sequence_pool_op.h
deleted file mode 100644
index 7b00395bb6bb2..0000000000000
--- a/paddle/fluid/operators/sequence_ops/sequence_pool_op.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#pragma once
-#include <string>
-
-#include "paddle/fluid/framework/eigen.h"
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/phi/kernels/funcs/math_function.h"
-#include "paddle/phi/kernels/funcs/sequence_pooling.h"
-
-namespace paddle {
-namespace operators {
-
-template <typename T, typename DeviceContext>
-class SequencePoolGradKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext& context) const override {
-    auto* out_g =
-        context.Input<phi::DenseTensor>(framework::GradVarName("Out"));
-    auto* in_g = context.Output<phi::DenseTensor>(framework::GradVarName("X"));
-    std::string pooltype = context.Attr<std::string>("pooltype");
-    const phi::DenseTensor* index = nullptr;
-    if (pooltype == "MAX") {
-      index = context.Input<phi::DenseTensor>("MaxIndex");
-    }
-    in_g->mutable_data<T>(context.GetPlace());
-    phi::funcs::SequencePoolGradFunctor<DeviceContext, T> pool;
-    pool(context.template device_context<DeviceContext>(),
-         pooltype,
-         *out_g,
-         in_g,
-         index);
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
diff --git a/paddle/fluid/operators/sequence_ops/unity_build_rule.cmake b/paddle/fluid/operators/sequence_ops/unity_build_rule.cmake
index b23035082e4aa..b256a227a9646 100644
--- a/paddle/fluid/operators/sequence_ops/unity_build_rule.cmake
+++ b/paddle/fluid/operators/sequence_ops/unity_build_rule.cmake
@@ -30,7 +30,6 @@ register_unity_group(
   sequence_expand_op.cu
   sequence_mask_op.cu
   sequence_pad_op.cu
-  sequence_pool_op.cu
   sequence_expand_as_op.cu
   sequence_reshape_op.cu
   sequence_reverse_op.cu
diff --git a/paddle/phi/kernels/cpu/sequence_pool_grad_kernel.cc b/paddle/phi/kernels/cpu/sequence_pool_grad_kernel.cc
new file mode 100644
index 0000000000000..9f15ed0147031
--- /dev/null
+++ b/paddle/phi/kernels/cpu/sequence_pool_grad_kernel.cc
@@ -0,0 +1,26 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/sequence_pool_grad_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/impl/sequence_pool_grad_kernel_impl.h"
+
+PD_REGISTER_KERNEL(sequence_pool_grad,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::SequencePoolGradKernel,
+                   float,
+                   double) {}
diff --git a/paddle/fluid/operators/sequence_ops/sequence_pool_op.cu b/paddle/phi/kernels/gpu/sequence_pool_grad_kernel.cu
similarity index 59%
rename from paddle/fluid/operators/sequence_ops/sequence_pool_op.cu
rename to paddle/phi/kernels/gpu/sequence_pool_grad_kernel.cu
index 796b02cb03e32..fe991a1fef431 100644
--- a/paddle/fluid/operators/sequence_ops/sequence_pool_op.cu
+++ b/paddle/phi/kernels/gpu/sequence_pool_grad_kernel.cu
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -11,8 +11,12 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
-#include "paddle/fluid/operators/sequence_ops/sequence_pool_op.h"
 
-namespace ops = paddle::operators;
-PD_REGISTER_STRUCT_KERNEL(
-    sequence_pool_grad, GPU, ALL_LAYOUT, ops::SequencePoolGradKernel, float) {}
+#include "paddle/phi/kernels/sequence_pool_grad_kernel.h"
+#include "paddle/phi/kernels/impl/sequence_pool_grad_kernel_impl.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+PD_REGISTER_KERNEL(
+    sequence_pool_grad, GPU, ALL_LAYOUT, phi::SequencePoolGradKernel, float) {}
diff --git a/paddle/phi/kernels/impl/sequence_pool_grad_kernel_impl.h b/paddle/phi/kernels/impl/sequence_pool_grad_kernel_impl.h
new file mode 100644
index 0000000000000..da9bdc1f1fdf5
--- /dev/null
+++ b/paddle/phi/kernels/impl/sequence_pool_grad_kernel_impl.h
@@ -0,0 +1,39 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/phi/kernels/funcs/sequence_pooling.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void SequencePoolGradKernel(const Context& dev_ctx,
+                            const DenseTensor& x,
+                            const paddle::optional<DenseTensor>& max_index,
+                            const DenseTensor& out_grad,
+                            bool is_test,
+                            const std::string& pooltype,
+                            float pad_value,
+                            DenseTensor* x_grad) {
+  const phi::DenseTensor* index = nullptr;
+  if (pooltype == "MAX") {
+    index = max_index.get_ptr();
+  }
+  dev_ctx.template Alloc<T>(x_grad);
+  phi::funcs::SequencePoolGradFunctor<Context, T> pool;
+  pool(dev_ctx, pooltype, out_grad, x_grad, index);
+}
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/impl/sequence_pool_kernel_impl.h b/paddle/phi/kernels/impl/sequence_pool_kernel_impl.h
index 56633b6b54270..e448820516afe 100644
--- a/paddle/phi/kernels/impl/sequence_pool_kernel_impl.h
+++ b/paddle/phi/kernels/impl/sequence_pool_kernel_impl.h
@@ -68,7 +68,7 @@ void SequencePoolKernel(const Context& ctx,
       (is_test == false || (ctx.GetPlace() == phi::CPUPlace()) == false)) {
     index = max_index;
     index->Resize({dims});
-    ctx.template Alloc<int>(index);
+    ctx.template Alloc<int32_t>(index);
   }
   phi::funcs::SequencePoolFunctor<Context, T> pool;
   pool(ctx, pooltype, pad_value_, x, out, is_test, index);
diff --git a/paddle/phi/kernels/sequence_pool_grad_kernel.h b/paddle/phi/kernels/sequence_pool_grad_kernel.h
new file mode 100644
index 0000000000000..a88f9ceb4b1ae
--- /dev/null
+++ b/paddle/phi/kernels/sequence_pool_grad_kernel.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+template <typename T, typename Context>
+void SequencePoolGradKernel(const Context& dev_ctx,
+                            const DenseTensor& x,
+                            const paddle::optional<DenseTensor>& max_index,
+                            const DenseTensor& out_grad,
+                            bool is_test,
+                            const std::string& pooltype,
+                            float pad_value,
+                            DenseTensor* x_grad);
+
+}  // namespace phi
diff --git a/paddle/phi/ops/compat/sequence_pool_sig.cc b/paddle/phi/ops/compat/sequence_pool_sig.cc
index 6c4d6691db4bb..224f83d1f32f9 100644
--- a/paddle/phi/ops/compat/sequence_pool_sig.cc
+++ b/paddle/phi/ops/compat/sequence_pool_sig.cc
@@ -21,6 +21,16 @@ KernelSignature SequencePoolOpArgumentMapping(
                          {"Out", "MaxIndex"});
 }
 
+KernelSignature SequencePoolGradOpArgumentMapping(
+    const ArgumentMappingContext& ctx) {
+  return KernelSignature("sequence_pool_grad",
+                         {"X", "MaxIndex", "Out@GRAD"},
+                         {"is_test", "pooltype", "pad_value"},
+                         {"X@GRAD"});
+}
+
 }  // namespace phi
 
 PD_REGISTER_ARG_MAPPING_FN(sequence_pool, phi::SequencePoolOpArgumentMapping);
+PD_REGISTER_ARG_MAPPING_FN(sequence_pool_grad,
+                           phi::SequencePoolGradOpArgumentMapping);

From 2e31f245c4fd0a028f86f0ad80adc2d8df8a5bf6 Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Thu, 27 Apr 2023 22:00:02 +0800
Subject: [PATCH 120/405] much thanks to xreki and shaojie for debuging this
 flash_attn in AlphaFold2 model plugin

---
 cmake/external/flashattn.cmake                |  2 +-
 .../operators/fused/fused_gate_attention.h    | 82 +++++++++----------
 2 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index c94717ff9f5ff..29c0b8efb426c 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -20,7 +20,7 @@ set(FLASHATTN_PREFIX_DIR ${THIRD_PARTY_PATH}/flashattn)
 set(FLASHATTN_SOURCE_SUBDIR csrc/flash_attn)
 set(FLASHATTN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flashattn)
 set(FLASHATTN_REPOSITORY ${GIT_URL}/JamesLim-sy/flash-attention.git)
-set(FLASHATTN_TAG opitmization_for_alphafold2)
+set(FLASHATTN_TAG 782a98a1fc02fb36d07129a97750430e4fdf2022)
 
 set(FLASHATTN_INCLUDE_DIR
     "${FLASHATTN_INSTALL_DIR}/include"
diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 5dd6f26a802d6..5126d1a78f13a 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -70,9 +70,24 @@ inline std::string TensorDebugString(const phi::DenseTensor* t) {
   }
 }
 
-inline std::string WaitWithDebugInfo(const phi::GPUContext& dev_ctx) {
-  dev_ctx.Wait();
-  return "[Synchronize] ";
+inline void WaitWithDebugInfo(const phi::GPUContext& dev_ctx) {
+  if (VLOG_IS_ON(5)) {
+    dev_ctx.Wait();
+    VLOG(5) << "[Flash attn Synchronize] ";
+  }
+}  
+
+template<typename T>
+inline void TypeDebugInfo() {
+  if (VLOG_IS_ON(4)) {
+    if (std::is_same<T, phi::dtype::float16>::value) {
+      VLOG(4) << "[Grad]: T is phi::dtype::float16.";
+    } else if (std::is_same<T, phi::dtype::bfloat16>::value) {
+      VLOG(4) << "[Grad]: T is phi::dtype::bfloat16.";
+    } else if (std::is_same<T, float>::value) {
+      VLOG(4) << "[Grad]: T is float.";
+    }
+  }
 }
 
 template <typename T>
@@ -902,15 +917,7 @@ class FlashAttnWithGating {
                       GateAttentionConfig<T>* config) {
     bool is_bf16 =
         qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
-
-    if (std::is_same<T, phi::dtype::float16>::value) {
-      VLOG(4) << "T is phi::dtype::float16.";
-    } else if (std::is_same<T, phi::dtype::bfloat16>::value) {
-      VLOG(4) << "T is phi::dtype::bfloat16.";
-    } else if (std::is_same<T, float>::value) {
-      VLOG(4) << "T is float.";
-    }
-
+    TypeDebugInfo<T>();
     PADDLE_ENFORCE_NOT_NULL(
         qkv_transpose_out,
         platform::errors::NotFound("The input qkv_transpose_out can not be "
@@ -920,6 +927,7 @@ class FlashAttnWithGating {
     phi::DenseTensor* qkv_out = config->GetQKVOut();
     ComputeQKVTransposeForwardForFlashAttn(*qkv_out, qkv_transpose_out);
     config->ClearQKVOut();
+    WaitWithDebugInfo(dev_ctx_);
 
     int seq_batch_size = static_cast<int>(config->batch_size) *
                          static_cast<int>(config->seq_len_m);
@@ -928,7 +936,7 @@ class FlashAttnWithGating {
          seq_batch_size * static_cast<int>(config->seq_len_r),
          static_cast<int>(config->num_heads),
          static_cast<int>(config->head_dim)});
-    VLOG(5) << WaitWithDebugInfo(dev_ctx_) << "Reshape qkv_transpose_out: ["
+    VLOG(5) << "Reshape qkv_transpose_out: ["
             << config->qkv_transpose_out_dims << "] -> ["
             << qkv_transpose_out->dims() << "]";
 
@@ -988,14 +996,15 @@ class FlashAttnWithGating {
     int softmax_lse_last_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
     softmax_lse->Resize({batch_size_, num_heads_, softmax_lse_last_dim});
     AllocWithDebugInfo<float>(dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
-    VLOG(5) << WaitWithDebugInfo(dev_ctx_) << "Allocate softmax_lse: shape=["
+    WaitWithDebugInfo(dev_ctx_);
+    VLOG(5) << "Allocate softmax_lse: shape=["
             << softmax_lse->dims() << "]";
 
     // 6. construct random seed
     auto seed_offset_pair = GenerateSeedOffsetPair(batch_size_, num_heads_);
     uint64_t seed = seed_offset_pair.first;
     uint64_t offset = seed_offset_pair.second;
-    VLOG(5) << WaitWithDebugInfo(dev_ctx_) << "Construct random seed";
+    VLOG(5) << "Construct random seed";
 
     VLOG(6) << "cu_seq_q: " << TensorDebugString(&cu_seq_q);
     VLOG(6) << "cu_seq_k: " << TensorDebugString(&cu_seq_k);
@@ -1047,8 +1056,8 @@ class FlashAttnWithGating {
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
-    VLOG(5) << WaitWithDebugInfo(dev_ctx_)
-            << "Get workspace_size=" << workspace_size;
+    WaitWithDebugInfo(dev_ctx_);
+    VLOG(5) << "Get workspace_size=" << workspace_size;
 
     // 8. Run flash-attention kernel.
     phi::DenseTensor workspace = CreateWorkspace(workspace_size);
@@ -1086,8 +1095,8 @@ class FlashAttnWithGating {
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
-    VLOG(5) << "[ComputeForward]" << WaitWithDebugInfo(dev_ctx_)
-            << "Run SUCCESS!!!";
+    WaitWithDebugInfo(dev_ctx_);
+    VLOG(5) << "[ComputeForward]: Run SUCCESS!!!";
 
     if (config->has_gating) {
       gate_out->Resize(config->gate_out_dims);
@@ -1105,14 +1114,7 @@ class FlashAttnWithGating {
                        GateAttentionGradConfig<T>* config) {
     bool is_bf16 =
         qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
-
-    if (std::is_same<T, phi::dtype::float16>::value) {
-      VLOG(4) << "[Grad]: T is phi::dtype::float16.";
-    } else if (std::is_same<T, phi::dtype::bfloat16>::value) {
-      VLOG(4) << "[Grad]: T is phi::dtype::bfloat16.";
-    } else if (std::is_same<T, float>::value) {
-      VLOG(4) << "[Grad]: T is float.";
-    }
+    TypeDebugInfo<T>();
 
     PADDLE_ENFORCE_NOT_NULL(
         qkv_transpose_out,
@@ -1135,7 +1137,7 @@ class FlashAttnWithGating {
 
     int seq_batch_size = static_cast<int>(config->batch_size) *
                          static_cast<int>(config->seq_len_m);
-    VLOG(5) << WaitWithDebugInfo(dev_ctx_);
+    WaitWithDebugInfo(dev_ctx_);
 
     // 2. Init with cu_seq_q and cu_seq_k for flash_attn.
     phi::DenseTensor cu_seq_q, cu_seq_k;
@@ -1174,8 +1176,8 @@ class FlashAttnWithGating {
     int total_k_ = qkv_dims[1];             // q.dims()[0]
     int num_heads_ = qkv_dims[2];           // q.dims()[1]
     int head_size_ = qkv_dims[3];           // q.dims()[2]
-    int max_seqlen_q_ = config->seq_len_r;  // batch_size_;
-    int max_seqlen_k_ = config->m_size;     // batch_size_;
+    int max_seqlen_q_ = config->seq_len_r;
+    int max_seqlen_k_ = config->m_size;
     VLOG(6) << "batch_size : " << batch_size_;
     VLOG(6) << "total_q   : " << total_q_;
     VLOG(6) << "total_k   : " << total_k_;
@@ -1188,20 +1190,19 @@ class FlashAttnWithGating {
     phi::DenseTensor softmax_d;
     softmax_d.Resize(softmax_lse->dims());
     AllocWithDebugInfo<float>(dev_ctx_, "d_softmax_lse", &softmax_d);
-    VLOG(5) << WaitWithDebugInfo(dev_ctx_);
+    WaitWithDebugInfo(dev_ctx_);
 
     phi::DenseTensor bias_d;
     if (nonbatched_bias) {
       bias_d = phi::Empty<T, phi::GPUContext>(
           dev_ctx_, {batch_size_, num_heads_, max_seqlen_q_, max_seqlen_k_});
     }
-    VLOG(5) << WaitWithDebugInfo(dev_ctx_);
+    WaitWithDebugInfo(dev_ctx_);
 
     // 6. construct random seed
     auto seed_offset_pair = GenerateSeedOffsetPair(batch_size_, num_heads_);
     uint64_t seed = seed_offset_pair.first;
     uint64_t offset = seed_offset_pair.second;
-
     VLOG(6) << "fmha_out: " << TensorDebugString(fmha_out);
     VLOG(6) << "fmha_out_grad: " << TensorDebugString(fmha_out_grad);
     VLOG(6) << "softmax_lse: " << TensorDebugString(softmax_lse);
@@ -1254,8 +1255,7 @@ class FlashAttnWithGating {
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
-    VLOG(5) << WaitWithDebugInfo(dev_ctx_)
-            << "Get workspace_size=" << workspace_size;
+    VLOG(5) << "Get workspace_size=" << workspace_size;
 
     phi::DenseTensor workspace = CreateWorkspace(workspace_size);
     succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
@@ -1297,7 +1297,8 @@ class FlashAttnWithGating {
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
-    VLOG(5) << WaitWithDebugInfo(dev_ctx_);
+    WaitWithDebugInfo(dev_ctx_);
+    VLOG(5) << "[ComputeBackward]: Run SUCCESS!!!";
 
     if (nonbatched_bias) {
       // compare block reduce
@@ -1344,9 +1345,8 @@ class FlashAttnWithGating {
     int64_t grid = (seq_size + block - 1) / block;
     FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
         start, step, end, cu_seq_q->data<int32_t>(), cu_seq_k->data<int32_t>());
-
-    VLOG(5) << WaitWithDebugInfo(dev_ctx_)
-            << "AllocAndInit cu_seq_q and cu_seq_k: start=" << start
+    WaitWithDebugInfo(dev_ctx_);
+    VLOG(5) << "AllocAndInit cu_seq_q and cu_seq_k: start=" << start
             << ", step=" << step << ", end=" << end;
   }
 
@@ -1356,8 +1356,8 @@ class FlashAttnWithGating {
       workspace = phi::Empty<float, phi::GPUContext>(
           dev_ctx_, {int64_t(workspace_size / sizeof(float))});
     }
-    VLOG(5) << WaitWithDebugInfo(dev_ctx_)
-            << "Allocate workspace: workspace_size=" << workspace_size;
+    WaitWithDebugInfo(dev_ctx_);
+    VLOG(5) << "Allocate workspace: workspace_size=" << workspace_size;
     return workspace;
   }
 

From 3262172d5db5d09b844cd8d04d9a6ca110c5451a Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Thu, 27 Apr 2023 22:00:40 +0800
Subject: [PATCH 121/405] much thanks to xreki and shaojie for debuging this
 flash_attn in AlphaFold2 model plugin

---
 .../operators/fused/fused_gate_attention.h    | 21 ++++++++-----------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 5126d1a78f13a..319889246890a 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -75,9 +75,9 @@ inline void WaitWithDebugInfo(const phi::GPUContext& dev_ctx) {
     dev_ctx.Wait();
     VLOG(5) << "[Flash attn Synchronize] ";
   }
-}  
+}
 
-template<typename T>
+template <typename T>
 inline void TypeDebugInfo() {
   if (VLOG_IS_ON(4)) {
     if (std::is_same<T, phi::dtype::float16>::value) {
@@ -922,7 +922,6 @@ class FlashAttnWithGating {
         qkv_transpose_out,
         platform::errors::NotFound("The input qkv_transpose_out can not be "
                                    "nullptr when merge_qkv is true."));
-
     // 1. Dealing with qkv_out for flash_attn.
     phi::DenseTensor* qkv_out = config->GetQKVOut();
     ComputeQKVTransposeForwardForFlashAttn(*qkv_out, qkv_transpose_out);
@@ -936,9 +935,8 @@ class FlashAttnWithGating {
          seq_batch_size * static_cast<int>(config->seq_len_r),
          static_cast<int>(config->num_heads),
          static_cast<int>(config->head_dim)});
-    VLOG(5) << "Reshape qkv_transpose_out: ["
-            << config->qkv_transpose_out_dims << "] -> ["
-            << qkv_transpose_out->dims() << "]";
+    VLOG(5) << "Reshape qkv_transpose_out: [" << config->qkv_transpose_out_dims
+            << "] -> [" << qkv_transpose_out->dims() << "]";
 
     // q_size == k_size
     int64_t q_size = config->GetQuerySize();
@@ -997,8 +995,7 @@ class FlashAttnWithGating {
     softmax_lse->Resize({batch_size_, num_heads_, softmax_lse_last_dim});
     AllocWithDebugInfo<float>(dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
     WaitWithDebugInfo(dev_ctx_);
-    VLOG(5) << "Allocate softmax_lse: shape=["
-            << softmax_lse->dims() << "]";
+    VLOG(5) << "Allocate softmax_lse: shape=[" << softmax_lse->dims() << "]";
 
     // 6. construct random seed
     auto seed_offset_pair = GenerateSeedOffsetPair(batch_size_, num_heads_);
@@ -1172,10 +1169,10 @@ class FlashAttnWithGating {
                         static_cast<int>(config->num_heads),
                         static_cast<int>(config->head_dim)});
     int batch_size_ = seq_batch_size;
-    int total_q_ = qkv_dims[1];             // q.dims()[0]
-    int total_k_ = qkv_dims[1];             // q.dims()[0]
-    int num_heads_ = qkv_dims[2];           // q.dims()[1]
-    int head_size_ = qkv_dims[3];           // q.dims()[2]
+    int total_q_ = qkv_dims[1];    // q.dims()[0]
+    int total_k_ = qkv_dims[1];    // q.dims()[0]
+    int num_heads_ = qkv_dims[2];  // q.dims()[1]
+    int head_size_ = qkv_dims[3];  // q.dims()[2]
     int max_seqlen_q_ = config->seq_len_r;
     int max_seqlen_k_ = config->m_size;
     VLOG(6) << "batch_size : " << batch_size_;

From 3474e09cc6face797d866f27efe3c8605e3af883 Mon Sep 17 00:00:00 2001
From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com>
Date: Thu, 27 Apr 2023 22:09:44 +0800
Subject: [PATCH 122/405] Support different dtypes of inputs for broadcast for
 dropout optimization  (#52093)

* change judgement for DropoutGradGPUKernelDriver

* add UnrollerWithoutVecSize and after this Loaddata to be refined

* pass unittest

* use same unroller with XPU

* BroadcastWithInt64Index

* BroadcastDataLoader template partial specialization

* fix compile errs in ROCms

* PR comment
---
 paddle/phi/kernels/funcs/broadcast_function.h | 620 ++++++++----------
 paddle/phi/kernels/funcs/dropout_impl.cu.h    |  58 +-
 paddle/phi/kernels/funcs/elementwise_base.h   | 106 +--
 .../kernels/primitive/datamover_primitives.h  |  96 ++-
 4 files changed, 425 insertions(+), 455 deletions(-)

diff --git a/paddle/phi/kernels/funcs/broadcast_function.h b/paddle/phi/kernels/funcs/broadcast_function.h
index f96a1764c24a5..b4125ab555079 100644
--- a/paddle/phi/kernels/funcs/broadcast_function.h
+++ b/paddle/phi/kernels/funcs/broadcast_function.h
@@ -31,20 +31,49 @@ namespace funcs {
 
 enum BroadcastLoadType { kMixed = 1, kBroadcast = 2, kElementwise = 3 };
 
-template <typename InT, typename OutT, int Arity>
+template <int Index>
+struct UseBroadcast {
+  template <typename ArgsT, typename Array1, typename Array2>
+  static HOSTDEVICE void Apply(
+      const std::vector<const DenseTensor *> &ins_tensor,
+      const ArgsT &args,
+      int64_t numel,
+      Array1 *ins_data,
+      Array2 *use_broadcast,
+      int *broadcast_num,
+      bool *all_elementwise) {
+    (*ins_data)[Index] = (const _ptr_ char *)(ins_tensor[Index]->data());
+    bool is_same_dim = ins_tensor[Index]->numel() == numel;
+    if (is_same_dim) {
+      (*use_broadcast)[Index] = false;
+    } else {
+      (*use_broadcast)[Index] = true;
+      (*broadcast_num)++;
+    }
+    *all_elementwise &= is_same_dim;
+  }
+};
+
+template <typename OutT, int Arity, typename Functor>
 struct LoaderTypeClassifier {
  public:
   int64_t numel{0};
-  int vec_size{1};
+  int vec_size{4};
   int broadcast_num{0};
   bool all_elementwise{true};
-  phi::Array<int, Arity> use_broadcast;
-  phi::Array<const _ptr_ InT *__restrict__, Arity> ins_data;
+  phi::Array<bool, Arity> use_broadcast;
+  phi::Array<const _ptr_ char *__restrict__, Arity> ins_data;
 
   LoaderTypeClassifier() {}
   LoaderTypeClassifier(const std::vector<const DenseTensor *> &ins,
                        std::vector<DenseTensor *> *outs) {
+    using Traits = phi::funcs::FunctionTraits<Functor>;
+    using ArgsT = typename Traits::ArgsTuple;
+    ArgsT arg;
     uint64_t out_addr = reinterpret_cast<uint64_t>((*outs)[0]->data<OutT>());
+
+    UnrollerWithoutVecSize<VecSizeGetter, Arity>::step(ins, arg, &vec_size);
+
     for (auto i = 1; i < outs->size(); ++i) {
       PADDLE_ENFORCE_EQ(
           (*outs)[i]->dims(),
@@ -56,165 +85,185 @@ struct LoaderTypeClassifier {
       out_addr =
           (out_addr | reinterpret_cast<uint64_t>((*outs)[i]->data<OutT>()));
     }
-    int out_vec_size =
-        phi::GetVectorizedSize<OutT>(reinterpret_cast<OutT *>(out_addr));
 
-    uint64_t in_addr = static_cast<uint64_t>(0);
+    vec_size = std::min(
+        vec_size,
+        phi::GetVectorizedSize<OutT>(reinterpret_cast<OutT *>(out_addr)));
     numel = (*outs)[0]->numel();
-    for (int i = 0; i < Arity; ++i) {
-      auto in_data = ins[i]->data<InT>();
-      ins_data[i] = (const _ptr_ InT *)(in_data);
-
-      bool is_same_dim = ins[i]->numel() == numel;
-      if (is_same_dim) {
-        use_broadcast[i] = false;
-        in_addr = (in_addr | reinterpret_cast<uint64_t>(in_data));
-      } else {
-        use_broadcast[i] = true;
-        broadcast_num++;
-      }
-      all_elementwise &= is_same_dim;
-    }
-    int in_vec_size = std::min(
-        4, phi::GetVectorizedSize<InT>(reinterpret_cast<InT *>(in_addr)));
-    vec_size = std::min(out_vec_size, in_vec_size);
+    UnrollerWithoutVecSize<UseBroadcast, Arity>::step(ins,
+                                                      arg,
+                                                      numel,
+                                                      &ins_data,
+                                                      &use_broadcast,
+                                                      &broadcast_num,
+                                                      &all_elementwise);
   }
 };
 
-#ifndef PADDLE_WITH_XPU_KP
 // Common broadcast/elementwise Loader.
-template <typename T, int VecSize, int Arity, bool IsBoundary, int LoadType>
+template <int Index, int VecSize, bool IsBoundary, int LoadType>
 struct BroadcastDataLoader {
-  __device__ __forceinline__ void operator()(
-      T args[Arity][VecSize],
-      const phi::Array<const _ptr_ T *__restrict__, Arity> &ins,
-      const phi::Array<kps::details::BroadcastConfig, Arity> &configs,
-      const phi::Array<int, Arity> &use_broadcast,
-      const int block_offset,
-      const int num,
-      const uint32_t numel) {
-#pragma unroll
-    for (int i = 0; i < Arity; ++i) {
-      kps::Init<T, VecSize>(args[i], static_cast<T>(1.0f));
-      if (use_broadcast[i]) {
-        kps::ReadDataBc<T, VecSize, 1, IsBoundary>(
-            args[i], ins[i], block_offset, configs[i], numel, VecSize);
-      } else {
-        kps::ReadData<T, VecSize, 1, IsBoundary>(
-            args[i], ins[i] + block_offset, num, VecSize);
-      }
+  template <typename Array1, typename Array2, typename Array3, typename ArgsT>
+  static __device__ __forceinline__ void Apply(const Array1 &ins,
+                                               ArgsT *args,
+                                               const Array2 &configs,
+                                               const Array3 &use_broadcast,
+                                               const int block_offset,
+                                               const int num,
+                                               const uint32_t numel) {
+    using Type = std::tuple_element_t<Index, ArgsT>;
+    kps::Init<Type, ArgsT, Index, VecSize>(args, static_cast<Type>(1.0f));
+
+    if (use_broadcast[Index]) {
+      kps::ReadDataBc<Type, VecSize, 1, ArgsT, Index, IsBoundary>(
+          args,
+          reinterpret_cast<const _ptr_ Type *>(ins[Index]),
+          block_offset,
+          configs[Index],
+          numel,
+          VecSize);
+    }
+    // NOTE: If use if...else... with condition `use_broadcast[Index]` here,
+    // there will be some errs with clang12 while compiling in ROCm.
+    // When the compiler is upgraded, if...else... may be used.
+    if (!use_broadcast[Index]) {
+      kps::ReadData<Type, VecSize, 1, ArgsT, Index, IsBoundary>(
+          args,
+          reinterpret_cast<const _ptr_ Type *>(ins[Index]) + block_offset,
+          num,
+          VecSize);
     }
   }
 };
 
+/* BroadcastDataLoaders Partial specialization */
+#ifndef PADDLE_WITH_XPU_KP
 // Scalar elementwise Loader with consideration of IsBoundary.
-template <typename T, int VecSize, int Arity>
-struct BroadcastDataLoader<T, VecSize, Arity, true, kElementwise> {
-  __device__ __forceinline__ void operator()(
-      T args[Arity][VecSize],
-      const phi::Array<const _ptr_ T *__restrict__, Arity> &ins,
-      const phi::Array<kps::details::BroadcastConfig, Arity> &configs,
-      const phi::Array<int, Arity> &use_broadcast,
-      const int block_offset,
-      const int num,
-      const uint32_t numel) {
+template <int Index, int VecSize>
+struct BroadcastDataLoader<Index, VecSize, true, kElementwise> {
+  template <typename Array1, typename Array2, typename Array3, typename ArgsT>
+  static __device__ __forceinline__ void Apply(const Array1 &ins,
+                                               ArgsT *args,
+                                               const Array2 &configs,
+                                               const Array3 &use_broadcast,
+                                               const int block_offset,
+                                               const int num,
+                                               const uint32_t numel) {
+    using Type = std::tuple_element_t<Index, ArgsT>;
     int thread_offset = threadIdx.x * VecSize + block_offset;
 #pragma unroll
-    for (int i = 0; i < Arity; ++i) {
-#pragma unroll
-      for (int idx = 0; idx < VecSize; ++idx) {
-        args[i][idx] = static_cast<T>(1);
-        int index = thread_offset + idx;
-        if (index < numel) {
-          args[i][idx] = ins[i][index];
-        }
+    for (int idx = 0; idx < VecSize; ++idx) {
+      std::get<Index>(args[idx]) = static_cast<Type>(1);
+      int index = thread_offset + idx;
+      if (index < numel) {
+        std::get<Index>(args[idx]) =
+            reinterpret_cast<const _ptr_ Type *>(ins[Index])[index];
       }
     }
   }
 };
 
 // Vectorized elementwise Loader without consideration of IsBoundary.
-template <typename T, int VecSize, int Arity>
-struct BroadcastDataLoader<T, VecSize, Arity, false, kElementwise> {
-  __device__ __forceinline__ void operator()(
-      T args[Arity][VecSize],
-      const phi::Array<const _ptr_ T *__restrict__, Arity> &ins,
-      const phi::Array<kps::details::BroadcastConfig, Arity> &configs,
-      const phi::Array<int, Arity> &use_broadcast,
-      const int block_offset,
-      const int num,
-      const uint32_t numel) {
-    using VecType = phi::kps::details::VectorType<T, VecSize>;
-    VecType vec_temp[Arity];
+template <int Index, int VecSize>
+struct BroadcastDataLoader<Index, VecSize, false, kElementwise> {
+  template <typename Array1, typename Array2, typename Array3, typename ArgsT>
+  static __device__ __forceinline__ void Apply(const Array1 &ins,
+                                               ArgsT *args,
+                                               const Array2 &configs,
+                                               const Array3 &use_broadcast,
+                                               const int block_offset,
+                                               const int num,
+                                               const uint32_t numel) {
+    using Type = std::tuple_element_t<Index, ArgsT>;
+    using VecType = phi::kps::details::VectorType<Type, VecSize>;
+    VecType vec_temp;
 
     int thread_offset = threadIdx.x + blockIdx.x * blockDim.x;
+    const VecType *__restrict__ vec_input =
+        reinterpret_cast<const VecType *__restrict__>(ins[Index]);
+    vec_temp = vec_input[thread_offset];
 #pragma unroll
-    for (int i = 0; i < Arity; ++i) {
-      const VecType *__restrict__ vec_input =
-          reinterpret_cast<const VecType *__restrict__>(ins[i]);
-      vec_temp[i] = vec_input[thread_offset];
-#pragma unroll
-      for (int idx = 0; idx < VecSize; ++idx) {
-        args[i][idx] = vec_temp[i].val[idx];
-      }
+    for (int idx = 0; idx < VecSize; ++idx) {
+      std::get<Index>(args[idx]) = vec_temp.val[idx];
     }
   }
 };
 
 // Common broadcast data loader.
-template <typename T, int VecSize, int Arity, bool IsBoundary>
-struct BroadcastDataLoader<T, VecSize, Arity, IsBoundary, kBroadcast> {
-  __device__ __forceinline__ void operator()(
-      T args[Arity][VecSize],
-      const phi::Array<const _ptr_ T *__restrict__, Arity> &ins,
-      const phi::Array<kps::details::BroadcastConfig, Arity> &configs,
-      const phi::Array<int, Arity> &use_broadcast,
-      const int block_offset,
-      const int num,
-      const uint32_t numel) {
-    uint32_t index_bc[Arity][VecSize];
-#pragma unroll
-    for (int j = 0; j < Arity; ++j) {
+template <int Index, int VecSize, bool IsBoundary>
+struct BroadcastDataLoader<Index, VecSize, IsBoundary, kBroadcast> {
+  template <typename Array1, typename Array2, typename Array3, typename ArgsT>
+  static __device__ __forceinline__ void Apply(const Array1 &ins,
+                                               ArgsT *args,
+                                               const Array2 &configs,
+                                               const Array3 &use_broadcast,
+                                               const int block_offset,
+                                               const int num,
+                                               const uint32_t numel) {
+    using Type = std::tuple_element_t<Index, ArgsT>;
+    uint32_t index_bc[VecSize];
 #pragma unroll
-      for (int k = 0; k < VecSize; ++k) {
-        index_bc[j][k] = 0;
-        args[j][k] = static_cast<T>(1);
-      }
+    for (int k = 0; k < VecSize; ++k) {
+      index_bc[k] = 0;
+      std::get<Index>(args[k]) = static_cast<Type>(1);
     }
 
     uint32_t thread_offset = block_offset + threadIdx.x * VecSize;
 #pragma unroll
     for (int k = 0; k < VecSize; ++k) {
       uint32_t idx = thread_offset + k;
-      if (IsBoundary) {
-        if (idx == numel) break;
+      if (IsBoundary && idx == numel) {
+        break;
       }
-
 #pragma unroll
       for (int i = 0; i < phi::DDim::kMaxRank; ++i) {
         if (i == configs[0].rank) break;
         auto fast_divmoder = configs[0].divmoders[i].Divmod(idx);
         idx = fast_divmoder.val[0];
-#pragma unroll
-        for (int j = 0; j < Arity; ++j) {
-          index_bc[j][k] += fast_divmoder.val[1] * configs[j].strides[i];
-        }
+        index_bc[k] += fast_divmoder.val[1] * configs[Index].strides[i];
       }
     }
 
 #pragma unroll
-    for (int j = 0; j < Arity; ++j) {
-#pragma unroll
-      for (int k = 0; k < VecSize; ++k) {
-        args[j][k] = ins[j][index_bc[j][k]];
-      }
+    for (int k = 0; k < VecSize; ++k) {
+      std::get<Index>(args[k]) =
+          reinterpret_cast<const _ptr_ Type *>(ins[Index])[index_bc[k]];
     }
   }
 };
+
 #endif
 
-template <typename InT,
-          typename OutT,
+// static broadcast unroller
+template <template <int Index, int VecSize, bool IsBoundary, int LoadType>
+          typename Func,
+          bool IsBoundary,
+          int LoadType,
+          int VecSize,
+          int End,
+          int Begin = 0>
+struct BcUnroller {
+  template <typename... Args>
+  static HOSTDEVICE inline void step(Args &&...args) {
+    Func<Begin, VecSize, IsBoundary, LoadType>::Apply(
+        std::forward<Args>(args)...);
+    BcUnroller<Func, IsBoundary, LoadType, VecSize, End, Begin + 1>::step(
+        args...);
+  }
+};
+
+template <template <int Index, int VecSize, bool IsBoundary, int LoadType>
+          typename Func,
+          bool IsBoundary,
+          int LoadType,
+          int VecSize,
+          int End>
+struct BcUnroller<Func, IsBoundary, LoadType, VecSize, End, End> {
+  template <typename... Args>
+  static HOSTDEVICE inline void step(Args &&...args) {}
+};
+
+template <typename OutT,
           typename Functor,
           int Arity,
           int NumOuts,
@@ -222,59 +271,43 @@ template <typename InT,
           bool IsBoundary,
           int LoadType>
 __device__ void VectorizedBroadcastKernelImpl(
-    const phi::Array<const _ptr_ InT *__restrict__, Arity> &ins,
+    const phi::Array<const _ptr_ char *__restrict__, Arity> &ins,
     phi::Array<_ptr_ OutT *, NumOuts> outs,
-    const phi::Array<int, Arity> &use_broadcast,
+    const phi::Array<bool, Arity> &use_broadcast,
     const uint32_t numel,
     const phi::Array<kps::details::BroadcastConfig, Arity> &configs,
     int num,
     int block_offset,
     int read_lens,
     Functor func) {
-  __simd__ InT args[Arity][VecSize];
+  using Traits = phi::funcs::FunctionTraits<Functor>;
+  using ArgsT = typename Traits::ArgsTuple;
+  __simd__ ArgsT args[VecSize];
   __simd__ ConditionalT<OutT, NumOuts> result[VecSize];
-#ifdef PADDLE_WITH_XPU_KP
-#pragma unroll
-  for (int i = 0; i < Arity; ++i) {
-    kps::Init<InT, VecSize>(args[i], static_cast<InT>(1.0f), read_lens);
-    if (use_broadcast[i]) {
-      kps::ReadDataBc<InT, VecSize, 1, IsBoundary>(
-          args[i], ins[i], block_offset, configs[i], numel, read_lens);
-    } else {
-      kps::ReadData<InT, VecSize, 1, IsBoundary>(
-          args[i], ins[i] + block_offset, num, read_lens);
-    }
-  }
-#else
-  BroadcastDataLoader<InT, VecSize, Arity, IsBoundary, LoadType>()(
-      args, ins, configs, use_broadcast, block_offset, num, numel);
-#endif
 
-  constexpr bool kCallElementwiseAny =
-      phi::funcs::FunctionTraits<Functor>::has_pointer_args;
-  phi::funcs::ElementwisePrimitiveCaller<InT,
-                                         ConditionalT<OutT, NumOuts>,
-                                         VecSize,
-                                         Functor,
-                                         Arity,
-                                         kCallElementwiseAny>()(
-      func, args, result, read_lens);
+  BcUnroller<BroadcastDataLoader, IsBoundary, LoadType, VecSize, Arity>::step(
+      ins, args, configs, use_broadcast, block_offset, num, numel);
+
+  SameDimsElementwisePrimitiveCaller<ConditionalT<OutT, NumOuts>,
+                                     VecSize,
+                                     Functor,
+                                     ArgsT,
+                                     Arity>()(func, args, result, read_lens);
   phi::funcs::
       ElementwiseWriteDataCallerBc<OutT, VecSize, IsBoundary, NumOuts>()(
           outs, result, block_offset, num, read_lens);
 }
 
 template <typename Functor,
-          typename InT,
           typename OutT,
           int Arity,
           int NumOuts,
           int VecSize,
           int LoadType>
 __global__ void VectorizedBroadcastKernel(
-    phi::Array<const _ptr_ InT *__restrict__, Arity> ins,
+    phi::Array<const _ptr_ char *__restrict__, Arity> ins,
     phi::Array<_ptr_ OutT *, NumOuts> outs,
-    phi::Array<int, Arity> use_broadcast,
+    phi::Array<bool, Arity> use_broadcast,
     uint32_t numel,
     phi::Array<kps::details::BroadcastConfig, Arity> configs,
     int main_offset,
@@ -285,8 +318,7 @@ __global__ void VectorizedBroadcastKernel(
   int block_offset = BLOCK_ID_X * BLOCK_NUM_X * read_lens;
   int stride = BLOCK_NUM_X * GRID_NUM_X * read_lens;
   for (; block_offset < main_offset; block_offset += stride) {
-    VectorizedBroadcastKernelImpl<InT,
-                                  OutT,
+    VectorizedBroadcastKernelImpl<OutT,
                                   Functor,
                                   Arity,
                                   NumOuts,
@@ -304,8 +336,7 @@ __global__ void VectorizedBroadcastKernel(
   }
   int num = numel - block_offset;
   if (num > 0) {
-    VectorizedBroadcastKernelImpl<InT,
-                                  OutT,
+    VectorizedBroadcastKernelImpl<OutT,
                                   Functor,
                                   Arity,
                                   NumOuts,
@@ -324,8 +355,7 @@ __global__ void VectorizedBroadcastKernel(
 #else
   int block_offset = BLOCK_ID_X * BLOCK_NUM_X * VecSize;
   if (block_offset < main_offset) {
-    VectorizedBroadcastKernelImpl<InT,
-                                  OutT,
+    VectorizedBroadcastKernelImpl<OutT,
                                   Functor,
                                   Arity,
                                   NumOuts,
@@ -341,8 +371,7 @@ __global__ void VectorizedBroadcastKernel(
                                             read_lens,
                                             func);
   } else {
-    VectorizedBroadcastKernelImpl<InT,
-                                  OutT,
+    VectorizedBroadcastKernelImpl<OutT,
                                   Functor,
                                   Arity,
                                   NumOuts,
@@ -361,19 +390,14 @@ __global__ void VectorizedBroadcastKernel(
 #endif
 }
 
-template <typename InT,
-          typename OutT,
-          typename Func,
-          int Arity,
-          int NumOuts,
-          int VecSize>
+template <typename OutT, typename Functor, int Arity, int NumOuts, int VecSize>
 void LaunchBroadcastKernel(
     const KPDevice &ctx,
     const std::vector<const DenseTensor *> &ins,
     std::vector<DenseTensor *> *outs,
-    Func func,
+    Functor func,
     const phi::Array<kps::details::BroadcastConfig, Arity> &configs,
-    const LoaderTypeClassifier<InT, OutT, Arity> &loader_classifier) {
+    const LoaderTypeClassifier<OutT, Arity, Functor> &loader_classifier) {
   phi::Array<_ptr_ OutT *, NumOuts> outs_data;
   for (int i = 0; i < NumOuts; ++i) {
     outs_data[i] = (_ptr_ OutT *)(ctx.Alloc<OutT>((*outs)[i]));
@@ -388,7 +412,7 @@ void LaunchBroadcastKernel(
   int main_offset = (numel / (read_lens * threads)) * read_lens * threads;
   int tail_tid = numel % (read_lens * threads);
 
-  VectorizedBroadcastKernel<Func, InT, OutT, Arity, NumOuts, VecSize, false>
+  VectorizedBroadcastKernel<Functor, OutT, Arity, NumOuts, VecSize, false>
       <<<blocks, threads, 0, stream>>>(loader_classifier.ins_data,
                                        outs_data,
                                        loader_classifier.use_broadcast,
@@ -409,8 +433,7 @@ void LaunchBroadcastKernel(
   int tail_tid = numel % (VecSize * threads);
 
   if (loader_classifier.all_elementwise) {
-    VectorizedBroadcastKernel<Func,
-                              InT,
+    VectorizedBroadcastKernel<Functor,
                               OutT,
                               Arity,
                               NumOuts,
@@ -427,7 +450,7 @@ void LaunchBroadcastKernel(
                                          func);
   } else if (loader_classifier.broadcast_num > (Arity >> 1)) {
     constexpr BroadcastLoadType type_ = (Arity > 1) ? kBroadcast : kMixed;
-    VectorizedBroadcastKernel<Func, InT, OutT, Arity, NumOuts, VecSize, type_>
+    VectorizedBroadcastKernel<Functor, OutT, Arity, NumOuts, VecSize, type_>
         <<<blocks, threads, 0, stream>>>(loader_classifier.ins_data,
                                          outs_data,
                                          loader_classifier.use_broadcast,
@@ -438,7 +461,7 @@ void LaunchBroadcastKernel(
                                          VecSize,
                                          func);
   } else {
-    VectorizedBroadcastKernel<Func, InT, OutT, Arity, NumOuts, VecSize, kMixed>
+    VectorizedBroadcastKernel<Functor, OutT, Arity, NumOuts, VecSize, kMixed>
         <<<blocks, threads, 0, stream>>>(loader_classifier.ins_data,
                                          outs_data,
                                          loader_classifier.use_broadcast,
@@ -471,94 +494,49 @@ HOSTDEVICE static int64_t ConvertSrcIdxToDstIdx(
   return dst_idx;
 }
 
-template <typename T, int VecSize, bool IsBoundary>
-HOSTDEVICE static void ReadVecDataWithInt64Index(
-    const T *in,
-    int64_t idx,
-    bool need_broadcast,
-    const phi::Array<int64_t, phi::DDim::kMaxRank + 1> &src_strides,
-    const phi::Array<int64_t, phi::DDim::kMaxRank + 1> &dst_strides,
-    int rank,
-    int n,
-    phi::AlignedVector<T, VecSize> *out) {
-  if (IsBoundary) {
-    for (int i = 0; i < n; ++i) {
-      (*out)[i] =
-          in[ConvertSrcIdxToDstIdx(idx + i, src_strides, dst_strides, rank)];
-    }
-  } else {
-    if (!need_broadcast) {
-      phi::Load<T, VecSize>(in + idx, out);
-    } else {
+template <int N>
+struct MaxWithOne {
+  static constexpr auto kValue = (N >= 1 ? N : 1);
+};
+
+template <int Index, int VecSize>
+struct ReadVecDataWithInt64Index {
+  template <typename Array1, typename Array2, typename Array3, typename ArgsT>
+  static __device__ __forceinline__ void Apply(
+      const Array1 &in,
+      ArgsT *args,
+      int64_t idx,
+      const Array2 &need_broadcast,
+      const phi::Array<int64_t, phi::DDim::kMaxRank + 1> &src_strides,
+      const Array3 &dst_strides,
+      int rank,
+      bool is_boundary) {
+    using Type = std::tuple_element_t<Index, ArgsT>;
+    if (is_boundary) {
 #pragma unroll
       for (int i = 0; i < VecSize; ++i) {
-        (*out)[i] =
-            in[ConvertSrcIdxToDstIdx(idx + i, src_strides, dst_strides, rank)];
+        std::get<Index>(args[i]) = in[Index][ConvertSrcIdxToDstIdx(
+            idx + i, src_strides, dst_strides[Index], rank)];
+      }
+    } else {
+      if (!need_broadcast[Index]) {
+        kps::ReadData<Type, VecSize, 1, ArgsT, Index, false>(
+            args, reinterpret_cast<const _ptr_ Type *>(in[Index]) + idx, 1);
+      } else {
+#pragma unroll
+        for (int i = 0; i < VecSize; ++i) {
+          std::get<Index>(args[i]) = in[Index][ConvertSrcIdxToDstIdx(
+              idx + i, src_strides, dst_strides[Index], rank)];
+        }
       }
     }
   }
-}
-
-template <typename InT,
-          typename OutT,
-          typename Functor,
-          int VecSize,
-          int NumIns>
-struct ApplyFunctorWithInt64IndexHelper {
-  HOSTDEVICE static OutT Run(const phi::AlignedVector<InT, VecSize> *ins_vec,
-                             Functor functor,
-                             int i);
-};
-
-template <typename InT, typename OutT, typename Functor, int VecSize>
-struct ApplyFunctorWithInt64IndexHelper<InT, OutT, Functor, VecSize, 0> {
-  HOSTDEVICE static OutT Run(const phi::AlignedVector<InT, VecSize> *ins_vec,
-                             Functor functor,
-                             int i) {
-    return static_cast<OutT>(functor());
-  }
-};
-
-template <typename InT, typename OutT, typename Functor, int VecSize>
-struct ApplyFunctorWithInt64IndexHelper<InT, OutT, Functor, VecSize, 1> {
-  HOSTDEVICE static OutT Run(const phi::AlignedVector<InT, VecSize> *ins_vec,
-                             Functor functor,
-                             int i) {
-    return static_cast<OutT>(functor(ins_vec[0][i]));
-  }
-};
-
-template <typename InT, typename OutT, typename Functor, int VecSize>
-struct ApplyFunctorWithInt64IndexHelper<InT, OutT, Functor, VecSize, 2> {
-  HOSTDEVICE static OutT Run(const phi::AlignedVector<InT, VecSize> *ins_vec,
-                             Functor functor,
-                             int i) {
-    return static_cast<OutT>(functor(ins_vec[0][i], ins_vec[1][i]));
-  }
 };
 
-template <typename InT, typename OutT, typename Functor, int VecSize>
-struct ApplyFunctorWithInt64IndexHelper<InT, OutT, Functor, VecSize, 3> {
-  HOSTDEVICE static OutT Run(const phi::AlignedVector<InT, VecSize> *ins_vec,
-                             Functor functor,
-                             int i) {
-    return static_cast<OutT>(
-        functor(ins_vec[0][i], ins_vec[1][i], ins_vec[2][i]));
-  }
-};
-
-template <int N>
-struct MaxWithOne {
-  static constexpr auto kValue = (N >= 1 ? N : 1);
-};
-
-template <typename InT,
-          typename OutT,
-          typename Functor,
-          int VecSize,
-          int NumIns>
+template <typename OutT, typename Functor, int VecSize, int NumIns>
 __global__ void BroadcastKernelWithInt64Index(
-    phi::Array<const InT *, MaxWithOne<NumIns>::kValue> ins,
+    const phi::Array<const _ptr_ char *__restrict__, MaxWithOne<NumIns>::kValue>
+        &ins,
     OutT *out,
     phi::Array<phi::Array<int64_t, phi::DDim::kMaxRank + 1>,
                MaxWithOne<NumIns>::kValue> ins_strides,
@@ -572,70 +550,34 @@ __global__ void BroadcastKernelWithInt64Index(
   int64_t stride = static_cast<int64_t>(blockDim.x) * gridDim.x * VecSize;
   int64_t limit = numel - VecSize;
 
-  phi::Array<phi::AlignedVector<InT, VecSize>, MaxWithOne<NumIns>::kValue>
-      ins_vec;
+  using Traits = phi::funcs::FunctionTraits<Functor>;
+  using ArgsT = typename Traits::ArgsTuple;
+
+  ArgsT args[VecSize];
   phi::AlignedVector<OutT, VecSize> out_vec;
   for (; idx <= limit; idx += stride) {
-#pragma unroll
-    for (int i = 0; i < NumIns; ++i) {
-      ReadVecDataWithInt64Index<InT, VecSize, false>(ins[i],
-                                                     idx,
-                                                     need_broadcasts[i],
-                                                     out_strides,
-                                                     ins_strides[i],
-                                                     rank,
-                                                     VecSize,
-                                                     &ins_vec[i]);
-    }
+    Unroller<ReadVecDataWithInt64Index, VecSize, NumIns>::step(
+        ins, args, idx, need_broadcasts, out_strides, ins_strides, rank, false);
 
 #pragma unroll
     for (int i = 0; i < VecSize; ++i) {
-      out_vec[i] = ApplyFunctorWithInt64IndexHelper<InT,
-                                                    OutT,
-                                                    Functor,
-                                                    VecSize,
-                                                    NumIns>::Run(ins_vec.Get(),
-                                                                 functor,
-                                                                 i);
+      out_vec[i] = static_cast<OutT>(Apply(functor, args[i]));
     }
-
     phi::Store<OutT, VecSize>(out_vec, out + idx);
   }
 
   if (idx < numel) {
     int remain = numel - idx;  // remain is always less than VecSize, therefore
                                // `int` is enough here
-#pragma unroll
-    for (int i = 0; i < NumIns; ++i) {
-      ReadVecDataWithInt64Index<InT, VecSize, true>(ins[i],
-                                                    idx,
-                                                    need_broadcasts[i],
-                                                    out_strides,
-                                                    ins_strides[i],
-                                                    rank,
-                                                    remain,
-                                                    &ins_vec[i]);
-    }
-
+    Unroller<ReadVecDataWithInt64Index, VecSize, NumIns>::step(
+        ins, args, idx, need_broadcasts, out_strides, ins_strides, rank, true);
     for (int i = 0; i < remain; ++i) {
-      out[idx + i] =
-          ApplyFunctorWithInt64IndexHelper<InT,
-                                           OutT,
-                                           Functor,
-                                           VecSize,
-                                           NumIns>::Run(ins_vec.Get(),
-                                                        functor,
-                                                        i);
+      out_vec[idx + i] = static_cast<OutT>(Apply(functor, args[i]));
     }
   }
 }
 
-template <typename InT,
-          typename OutT,
-          typename Functor,
-          int Arity,
-          int NumOuts,
-          int VecSize>
+template <typename OutT, typename Functor, int Arity, int NumOuts, int VecSize>
 struct LaunchBroadcastKernelWithInt64IndexHelper {
   static void Run(const KPDevice &ctx,
                   const std::vector<const DenseTensor *> &ins,
@@ -647,9 +589,8 @@ struct LaunchBroadcastKernelWithInt64IndexHelper {
   }
 };
 
-template <typename InT, typename OutT, typename Functor, int Arity, int VecSize>
-struct LaunchBroadcastKernelWithInt64IndexHelper<InT,
-                                                 OutT,
+template <typename OutT, typename Functor, int Arity, int VecSize>
+struct LaunchBroadcastKernelWithInt64IndexHelper<OutT,
                                                  Functor,
                                                  Arity,
                                                  /*NumOuts=*/1,
@@ -659,10 +600,9 @@ struct LaunchBroadcastKernelWithInt64IndexHelper<InT,
                   std::vector<DenseTensor *> *outs,
                   int axis,
                   Functor functor) {
-    phi::Array<const InT *, MaxWithOne<Arity>::kValue> ins_ptrs;
-    for (int i = 0; i < Arity; ++i) {
-      ins_ptrs[i] = ins[i]->data<InT>();
-    }
+    phi::Array<const _ptr_ char *__restrict__, MaxWithOne<Arity>::kValue>
+        ins_ptrs;
+    UnrollerWithoutVecSize<InputSetter, Arity>::step(ins, &ins_ptrs);
     auto *out_tensor = (*outs)[0];
     auto *out_ptr = ctx.Alloc<OutT>(out_tensor);
 
@@ -734,7 +674,7 @@ struct LaunchBroadcastKernelWithInt64IndexHelper<InT,
     auto gpu_config =
         phi::backends::gpu::GetGpuLaunchConfig1D(ctx, numel, VecSize);
 
-    BroadcastKernelWithInt64Index<InT, OutT, Functor, VecSize, Arity>
+    BroadcastKernelWithInt64Index<OutT, Functor, VecSize, Arity>
         <<<gpu_config.block_per_grid,
            gpu_config.thread_per_block,
            0,
@@ -844,9 +784,9 @@ struct LaunchBroadcastKernelWithInt64IndexHelper<InT,
 #endif
 
 template <ElementwiseType ET,
-          typename InT,
           typename OutT,
           typename Functor,
+          int kArity,
           int NumOuts = 1>
 void BroadcastKernelForDifferentVecSize(
     const KPDevice &ctx,
@@ -854,47 +794,17 @@ void BroadcastKernelForDifferentVecSize(
     std::vector<DenseTensor *> *outs,
     int axis,
     Functor func) {
-  using Traits = phi::funcs::FunctionTraits<Functor>;
-  const int kArity =
-      Traits::has_pointer_args ? static_cast<int>(ET) : Traits::arity;
-  PADDLE_ENFORCE_EQ(
-      ins.size(),
-      kArity,
-      phi::errors::InvalidArgument("The number of inputs is expected to be "
-                                   "equal to the "
-                                   "arity of functor. But received: the "
-                                   "number of inputs "
-                                   "is %d, the arity of functor is %d.",
-                                   ins.size(),
-                                   kArity));
-  PADDLE_ENFORCE_LE(
-      kArity,
-      3,
-      phi::errors::InvalidArgument("Currently only broadcast of ternary is "
-                                   "supported "
-                                   "and verified, but received %d.",
-                                   kArity));
-  PADDLE_ENFORCE_EQ(
-      outs->size(),
-      NumOuts,
-      phi::errors::InvalidArgument("Number of outputs shall equal to number "
-                                   "of functions, "
-                                   "but number of outputs is %d, of "
-                                   "functions is %d.",
-                                   outs->size(),
-                                   NumOuts));
-
 #ifndef PADDLE_WITH_XPU_KP
   constexpr bool kEnabledInt64IndexKernel = (NumOuts == 1 && kArity <= 3);
   bool use_int64_index_kernel =
       kEnabledInt64IndexKernel &&
       (*outs)[0]->numel() >= std::numeric_limits<int32_t>::max();
   if (use_int64_index_kernel) {
-    auto loader_classifier = LoaderTypeClassifier<InT, OutT, kArity>(ins, outs);
+    auto loader_classifier =
+        LoaderTypeClassifier<OutT, kArity, Functor>(ins, outs);
     switch (loader_classifier.vec_size) {
       case VecSizeL: {
-        LaunchBroadcastKernelWithInt64IndexHelper<InT,
-                                                  OutT,
+        LaunchBroadcastKernelWithInt64IndexHelper<OutT,
                                                   Functor,
                                                   kArity,
                                                   NumOuts,
@@ -906,8 +816,7 @@ void BroadcastKernelForDifferentVecSize(
         break;
       }
       case VecSizeM: {
-        LaunchBroadcastKernelWithInt64IndexHelper<InT,
-                                                  OutT,
+        LaunchBroadcastKernelWithInt64IndexHelper<OutT,
                                                   Functor,
                                                   kArity,
                                                   NumOuts,
@@ -919,8 +828,7 @@ void BroadcastKernelForDifferentVecSize(
         break;
       }
       case VecSizeS: {
-        LaunchBroadcastKernelWithInt64IndexHelper<InT,
-                                                  OutT,
+        LaunchBroadcastKernelWithInt64IndexHelper<OutT,
                                                   Functor,
                                                   kArity,
                                                   NumOuts,
@@ -949,7 +857,7 @@ void BroadcastKernelForDifferentVecSize(
       phi::errors::InvalidArgument(
           "XPU only support inputs is 2, but received %d", ins.size()));
 
-  auto loader_classifier = LoaderTypeClassifier<InT, OutT, kArity>();
+  auto loader_classifier = LoaderTypeClassifier<OutT, kArity, Functor>();
   const auto dims_simplifier =
       BroadcastDimsSimplifier(ins, (*outs)[0]->dims(), axis);
   if (VLOG_IS_ON(6)) {
@@ -968,7 +876,8 @@ void BroadcastKernelForDifferentVecSize(
   bool is_optimize = configs[0].cmp_type != type;
   int vec_size = is_optimize ? VecSizeL : VecSizeM;
 #else
-  auto loader_classifier = LoaderTypeClassifier<InT, OutT, kArity>(ins, outs);
+  auto loader_classifier =
+      LoaderTypeClassifier<OutT, kArity, Functor>(ins, outs);
   if (!loader_classifier.all_elementwise) {
     const auto dims_simplifier =
         BroadcastDimsSimplifier(ins, (*outs)[0]->dims(), axis);
@@ -991,17 +900,17 @@ void BroadcastKernelForDifferentVecSize(
 #endif
   switch (loader_classifier.vec_size) {
     case VecSizeL: {
-      LaunchBroadcastKernel<InT, OutT, Functor, kArity, NumOuts, VecSizeL>(
+      LaunchBroadcastKernel<OutT, Functor, kArity, NumOuts, VecSizeL>(
           ctx, ins, outs, func, configs, loader_classifier);
       break;
     }
     case VecSizeM: {
-      LaunchBroadcastKernel<InT, OutT, Functor, kArity, NumOuts, VecSizeM>(
+      LaunchBroadcastKernel<OutT, Functor, kArity, NumOuts, VecSizeM>(
           ctx, ins, outs, func, configs, loader_classifier);
       break;
     }
     case VecSizeS: {
-      LaunchBroadcastKernel<InT, OutT, Functor, kArity, NumOuts, VecSizeS>(
+      LaunchBroadcastKernel<OutT, Functor, kArity, NumOuts, VecSizeS>(
           ctx, ins, outs, func, configs, loader_classifier);
       break;
     }
@@ -1025,6 +934,28 @@ void BroadcastKernel(const KPDevice &ctx,
                      Functor func) {
   // When there are multiple inputs, the outputs's rank should be equal the
   // maximum rank of all inputs.
+  using Traits = phi::funcs::FunctionTraits<Functor>;
+  const int kArity = Traits::arity;
+  PADDLE_ENFORCE_EQ(
+      ins.size(),
+      kArity,
+      phi::errors::InvalidArgument("The number of inputs is expected to be "
+                                   "equal to the "
+                                   "arity of functor. But received: the "
+                                   "number of inputs "
+                                   "is %d, the arity of functor is %d.",
+                                   ins.size(),
+                                   kArity));
+  PADDLE_ENFORCE_EQ(
+      outs->size(),
+      NumOuts,
+      phi::errors::InvalidArgument("Number of outputs shall equal to number "
+                                   "of functions, "
+                                   "but number of outputs is %d, of "
+                                   "functions is %d.",
+                                   outs->size(),
+                                   NumOuts));
+
   int max_rank = 0;
   int min_rank = phi::DDim::kMaxRank;
   for (auto *in : ins) {
@@ -1037,7 +968,7 @@ void BroadcastKernel(const KPDevice &ctx,
     max_rank = std::max(max_rank, (*outs)[0]->dims().size());
   }
   axis = axis == -1 ? max_rank - min_rank : axis;
-  BroadcastKernelForDifferentVecSize<ET, InT, OutT, Functor, NumOuts>(
+  BroadcastKernelForDifferentVecSize<ET, OutT, Functor, kArity, NumOuts>(
       ctx, ins, outs, axis, func);
 }
 
@@ -1051,6 +982,7 @@ void ElementwiseCompute(const GPUContext &dev_ctx,
   std::vector<const DenseTensor *> ins = {&x, &y};
   std::vector<DenseTensor *> outs = {z};
   dev_ctx.template Alloc<OutType>(z);
+
   BroadcastKernel<ElementwiseType::kBinary, T, OutType, Functor, 1>(
       dev_ctx, ins, &outs, axis, func);
 }
diff --git a/paddle/phi/kernels/funcs/dropout_impl.cu.h b/paddle/phi/kernels/funcs/dropout_impl.cu.h
index 006784eb64c76..2ae9d3c02f63d 100644
--- a/paddle/phi/kernels/funcs/dropout_impl.cu.h
+++ b/paddle/phi/kernels/funcs/dropout_impl.cu.h
@@ -459,41 +459,43 @@ void DropoutGradGPUKernelDriver(const phi::GPUContext& dev_ctx,
     // y = factor * x
     ScaleByDropoutFactor<T, MT>(dev_ctx, grad_y, grad_x, factor);
   } else {
-    phi::DenseTensor broadcasted_mask;
-    if (is_dropout_nd) {
-      broadcasted_mask.Resize(grad_y.dims());
-      dev_ctx.template Alloc<uint8_t>(&broadcasted_mask);
-
-      std::vector<const phi::DenseTensor*> broadcast_ins = {&mask};
-      std::vector<phi::DenseTensor*> broadcast_outs = {&broadcasted_mask};
-      phi::funcs::BroadcastKernel<phi::ElementwiseType::kUnary,
-                                  uint8_t,
-                                  uint8_t>(dev_ctx,
-                                           broadcast_ins,
-                                           &broadcast_outs,
-                                           -1,
-                                           kps::IdentityFunctor<uint8_t>());
-    }
-
-    std::vector<const phi::DenseTensor*> ins = {
-        &grad_y, is_dropout_nd ? &broadcasted_mask : &mask};
-    std::vector<phi::DenseTensor*> outs = {grad_x};
-    if (upscale_in_train) {
-      if (dropout_prob == 1.0f) {
+    if (upscale_in_train && dropout_prob == 1.0f) {
 #ifdef PADDLE_WITH_HIP
-        hipMemset(grad_x->data<T>(), 0, grad_x->numel() * sizeof(T));
+      hipMemset(grad_x->data<T>(), 0, grad_x->numel() * sizeof(T));
 #else
-        cudaMemset(grad_x->data<T>(), 0, grad_x->numel() * sizeof(T));
+      cudaMemset(grad_x->data<T>(), 0, grad_x->numel() * sizeof(T));
 #endif
+    } else {
+      MT factor = upscale_in_train
+                      ? static_cast<MT>(1.0f / (1.0f - dropout_prob))
+                      : static_cast<MT>(1.0f);
+      if (is_dropout_nd) {
+        phi::DenseTensor broadcasted_mask;
+
+        broadcasted_mask.Resize(grad_y.dims());
+        dev_ctx.template Alloc<uint8_t>(&broadcasted_mask);
+
+        std::vector<const phi::DenseTensor*> broadcast_ins = {&mask};
+        std::vector<phi::DenseTensor*> broadcast_outs = {&broadcasted_mask};
+        phi::funcs::BroadcastKernel<phi::ElementwiseType::kUnary,
+                                    uint8_t,
+                                    uint8_t>(dev_ctx,
+                                             broadcast_ins,
+                                             &broadcast_outs,
+                                             -1,
+                                             kps::IdentityFunctor<uint8_t>());
+
+        std::vector<const phi::DenseTensor*> ins = {&grad_y, &broadcasted_mask};
+        std::vector<phi::DenseTensor*> outs = {grad_x};
+        phi::funcs::ElementwiseKernel<T>(
+            dev_ctx, ins, &outs, CudaDropoutGradFunctor<T>(factor));
+
       } else {
-        MT factor = static_cast<MT>(1.0f / (1.0f - dropout_prob));
+        std::vector<const phi::DenseTensor*> ins = {&grad_y, &mask};
+        std::vector<phi::DenseTensor*> outs = {grad_x};
         phi::funcs::ElementwiseKernel<T>(
             dev_ctx, ins, &outs, CudaDropoutGradFunctor<T>(factor));
       }
-    } else {
-      MT factor = static_cast<MT>(1.0f);
-      phi::funcs::ElementwiseKernel<T>(
-          dev_ctx, ins, &outs, CudaDropoutGradFunctor<T>(factor));
     }
   }
 }
diff --git a/paddle/phi/kernels/funcs/elementwise_base.h b/paddle/phi/kernels/funcs/elementwise_base.h
index b2c747d5348f8..0f6c225d89b2a 100644
--- a/paddle/phi/kernels/funcs/elementwise_base.h
+++ b/paddle/phi/kernels/funcs/elementwise_base.h
@@ -35,7 +35,7 @@ namespace kps = phi::kps;
 
 namespace phi {
 
-enum ElementwiseType { kUnary = 1, kBinary = 2, kTernary = 3, kAny = -1 };
+enum ElementwiseType { kUnary = 1, kBinary = 2, kTernary = 3 };
 /* Packing scalar type T(float, int etc.) into Array<T, NumOuts> type
    for supporting multiple-output feature in elementwise system.*/
 template <class T, int Num>
@@ -508,15 +508,31 @@ struct Unroller<Func, VecSize, End, End> {
   static HOSTDEVICE inline void step(Args &&...args) {}
 };
 
+// static unroller without VecSize for broadcast
+template <template <int Index> typename Func, int End, int Begin = 0>
+struct UnrollerWithoutVecSize {
+  template <typename... Args>
+  static HOSTDEVICE inline void step(Args &&...args) {
+    Func<Begin>::Apply(std::forward<Args>(args)...);
+    UnrollerWithoutVecSize<Func, End, Begin + 1>::step(args...);
+  }
+};
+
+template <template <int Index> typename Func, int End>
+struct UnrollerWithoutVecSize<Func, End, End> {
+  template <typename... Args>
+  static HOSTDEVICE inline void step(Args &&...args) {}
+};
+
 template <int Index, int VecSize>
 struct Loader {
   template <typename Array, typename ArgsT>
-  static __device__ void Apply(const Array &in,
-                               ArgsT *args,
-                               kps::IndexType offset,
-                               int num,
-                               int read_lens,
-                               bool is_boundary) {
+  static __device__ __forceinline__ void Apply(const Array &in,
+                                               ArgsT *args,
+                                               kps::IndexType offset,
+                                               int num,
+                                               int read_lens,
+                                               bool is_boundary) {
     using Type = std::tuple_element_t<Index, ArgsT>;
     kps::Init<Type, ArgsT, Index, VecSize>(
         args, static_cast<Type>(1.0f), read_lens);
@@ -536,7 +552,7 @@ struct Loader {
   }
 };
 
-template <int Index, int VecSize>
+template <int Index>
 struct InputSetter {
   template <typename Array>
   static HOSTDEVICE void Apply(
@@ -545,7 +561,7 @@ struct InputSetter {
   }
 };
 
-template <int Index, int VecSize>
+template <int Index>
 struct VecSizeGetter {
   template <typename ArgsT>
   static HOSTDEVICE void Apply(const std::vector<const DenseTensor *> &ins,
@@ -569,8 +585,7 @@ int GetVectorizedSizeForTensors(const std::vector<const DenseTensor *> &ins,
   int vec_size = 4;
   uint64_t addr = static_cast<uint64_t>(0);
   ArgsT arg;
-  // The Arg VecSize=1 is to match the Unroller template.
-  Unroller<VecSizeGetter, 1, Arity>::step(ins, arg, &vec_size);
+  UnrollerWithoutVecSize<VecSizeGetter, Arity>::step(ins, arg, &vec_size);
   for (auto iter = outs.begin(); iter != outs.end(); ++iter) {
     addr = (addr | reinterpret_cast<uint64_t>((*iter)->data<OutT>()));
   }
@@ -580,73 +595,6 @@ int GetVectorizedSizeForTensors(const std::vector<const DenseTensor *> &ins,
   return vec_size;
 }
 
-template <typename InT,
-          typename OutT,
-          int VecSize,
-          typename Functor,
-          int Arity,
-          bool CallElementwiseAny = false>
-struct ElementwisePrimitiveCaller {
-  __device__ inline void operator()(Functor func,
-                                    InT (*args)[VecSize],
-                                    OutT *result,
-                                    int read_lens);
-};
-
-template <typename InT, typename OutT, int VecSize, typename Functor, int Arity>
-struct ElementwisePrimitiveCaller<InT, OutT, VecSize, Functor, Arity, true> {
-  __device__ inline void operator()(Functor func,
-                                    InT (*args)[VecSize],
-                                    OutT *result,
-                                    int read_lens) {
-    kps::ElementwiseAny<InT, OutT, VecSize, 1, Arity, Functor>(
-        result, args, func);
-  }
-};
-
-template <typename InT, typename OutT, int VecSize, typename Functor>
-struct ElementwisePrimitiveCaller<InT, OutT, VecSize, Functor, 0, false> {
-  __device__ inline void operator()(Functor func,
-                                    InT (*args)[VecSize],
-                                    OutT *result,
-                                    int read_lens) {
-    kps::ElementwiseConstant<InT, OutT, VecSize, 1, Functor>(result, func);
-  }
-};
-
-template <typename InT, typename OutT, int VecSize, typename Functor>
-struct ElementwisePrimitiveCaller<InT, OutT, VecSize, Functor, 1, false> {
-  __device__ inline void operator()(Functor func,
-                                    InT (*args)[VecSize],
-                                    OutT *result,
-                                    int read_lens) {
-    kps::ElementwiseUnary<InT, OutT, VecSize, 1, Functor>(
-        result, args[0], func);
-  }
-};
-
-template <typename InT, typename OutT, int VecSize, typename Functor>
-struct ElementwisePrimitiveCaller<InT, OutT, VecSize, Functor, 2, false> {
-  __device__ inline void operator()(Functor func,
-                                    InT (*args)[VecSize],
-                                    OutT *result,
-                                    int read_lens) {
-    kps::ElementwiseBinary<InT, OutT, VecSize, 1, Functor>(
-        result, args[0], args[1], func, read_lens);
-  }
-};
-
-template <typename InT, typename OutT, int VecSize, typename Functor>
-struct ElementwisePrimitiveCaller<InT, OutT, VecSize, Functor, 3, false> {
-  __device__ inline void operator()(Functor func,
-                                    InT (*args)[VecSize],
-                                    OutT *result,
-                                    int read_lens) {
-    kps::ElementwiseTernary<InT, OutT, VecSize, 1, Functor>(
-        result, args[0], args[1], args[2], func);
-  }
-};
-
 namespace detail {
 template <class F, class Tuple, std::size_t... Index>
 // GCC/Clang need the decltype() return type
@@ -802,7 +750,7 @@ void LaunchElementwiseCudaKernel(const KPDevice &ctx,
   phi::Array<const _ptr_ char *__restrict__, Arity> ins_data;
   phi::Array<_ptr_ OutT *, NumOuts> outs_data;
 
-  Unroller<InputSetter, VecSize, Arity>::step(ins, &ins_data);
+  UnrollerWithoutVecSize<InputSetter, Arity>::step(ins, &ins_data);
   for (int i = 0; i < NumOuts; ++i) {
     outs_data[i] = (_ptr_ OutT *)(ctx.Alloc<OutT>((*outs)[i]));
   }
diff --git a/paddle/phi/kernels/primitive/datamover_primitives.h b/paddle/phi/kernels/primitive/datamover_primitives.h
index ac27916196105..2a3579d99cfe6 100644
--- a/paddle/phi/kernels/primitive/datamover_primitives.h
+++ b/paddle/phi/kernels/primitive/datamover_primitives.h
@@ -255,6 +255,18 @@ __device__ __forceinline__ void Init(ArgsT* dst, T init_data, int read_lens) {
   }
 }
 
+/**
+ * The difference from the above function is that
+ * it supports different data types of inputs.
+ */
+template <typename T, typename ArgsT, int Index, int NX>
+__device__ __forceinline__ void Init(ArgsT* dst, T init_data) {
+#pragma unroll
+  for (int i = 0; i < NX; i++) {
+    std::get<Index>(dst[i]) = init_data;
+  }
+}
+
 /**
  * @brief Read 1D data from global memory to register. When IsBoundary = true
  * and (NX % 4 == 0 or Nx % 2 == 0), vectorized load data will be used to
@@ -307,6 +319,23 @@ __device__ __forceinline__ void ReadData(T* dst,
   }
 }
 
+/**
+ * @brief Read 1D data from global memory to register.
+ * @template paraments
+ * T: The type of data.
+ * NX: Each thread load NX data from global memory continuously.
+ * NY: Each thread need to load NY rows, only NY = 1 was supported.
+ * IsBoundary: Whether to make an out-of-bounds judgment on access to memory.
+ * When the number of data processed by this block is less than
+ * NX x NY x blockDim.x, boundary judgment is required to avoid memory access
+ * crossing the boundary.
+ *
+ * @param：
+ * dst: The register pointer of the thread, the size is NX * NY.
+ * src: The data pointer of the current block.
+ * size: The current block needs to load size data continuously.
+ */
+
 template <typename T, int NX, int NY, bool IsBoundary = false>
 __device__ __forceinline__ void ReadData(T* dst,
                                          const T* __restrict__ src,
@@ -347,9 +376,8 @@ __device__ __forceinline__ void ReadData(T* dst,
  * T: The type of data.
  * NX: Each thread load NX data from global memory continuously.
  * NY: Each thread need to load NY rows, only NY = 1 was supported.
- * ArgsT: The Type if dst, ArgsT can be std::tuple<T> or std::tuple<Args>
+ * ArgsT: The Type of dst, ArgsT can be std::tuple<T> or std::tuple<Args>
  * Index: The index of data stored in dst.
- * threadIdx.x is used as the thread index. Currently only GPU was supported.
  * IsBoundary: Whether to make an out-of-bounds judgment on access to memory.
  * When the number of data processed by this block is less than
  * NX x NY x blockDim.x, boundary judgment is required to avoid memory access
@@ -369,7 +397,7 @@ template <typename T,
 __device__ __forceinline__ void ReadData(ArgsT* dst,
                                          const T* __restrict__ src,
                                          int num,
-                                         int read_lens) {
+                                         int read_lens = 0) {
   if (IsBoundary) {  // blockDim.x * NX > num
     int thread_offset = threadIdx.x * NX;
 #pragma unroll
@@ -743,7 +771,6 @@ __device__ __forceinline__ void Init(T* dst, T* init_data, int num) {
  * NX: The number of data continuously loaded by each thread.
  * NY: The number of data rows loaded by each thread, only NY = 1 was supported.
  * threadIdx.x is used as the thread index. Currently only GPU was supported.
- * Rank: The shape size of out. eg in[1, 35], out[32, 35] then shape size is 2.
  * IsBoundary: Indicates whether to perform block access storage out-of-bounds
  * judgment. When the number of data processed by the block is less than
  * NX x NY x blockDim.x, boundary judgment is required to avoid memory access
@@ -788,6 +815,67 @@ __device__ __forceinline__ void ReadDataBc(
   }
 }
 
+/**
+ * @brief Read 1D data from global memory to register with broadcast form.
+ * The difference from the above function is that it supports different data
+ * types of inputs.
+ *
+ * @template paraments
+ * T: The type of data stored in the global memory.
+ * NX: The number of data continuously loaded by each thread.
+ * NY: The number of data rows loaded by each thread, only NY = 1 was supported.
+ * ArgsT: The Type of dst, ArgsT can be std::tuple<T> or std::tuple<Args>
+ * Index: The index of data stored in dst.
+ * IsBoundary: Indicates whether to perform block access storage out-of-bounds
+ * judgment. When the number of data processed by the block is less than
+ * NX x NY x blockDim.x, boundary judgment is required to avoid memory access
+ * crossing the boundary.
+ *
+ * @param：
+ * dst: The register pointer of the thread, the size is NX * NY.
+ * src: The original input data pointer of kernel.
+ * block_offset: The data offset of this block, blockDim.x * blockIdx.x * NX;
+ * config: Calculation configuration of broadcast. It is used to calculate the
+ * coordinate mapping relationship between output data and input data.
+ * total_num_output: Total number of original output.
+ */
+
+template <typename T,
+          int NX,
+          int NY,
+          typename ArgsT,
+          int Index,
+          bool IsBoundary = false>
+__device__ __forceinline__ void ReadDataBc(
+    ArgsT* dst,
+    const T* __restrict__ src,
+    uint32_t block_offset,
+    const details::BroadcastConfig& config,
+    int total_num_output,
+    int read_lens = NX) {
+  uint32_t thread_offset = block_offset + threadIdx.x * NX;
+  uint32_t index_src = 0;
+
+#pragma unroll
+  for (uint32_t nx = 0; nx < NX; ++nx) {
+    uint32_t index_output = thread_offset + nx;
+    index_src = 0;
+    if (IsBoundary) {
+      if (index_output >= total_num_output) {
+        break;
+      }
+    }
+#pragma unroll
+    for (int i = 0; i < phi::DDim::kMaxRank; ++i) {
+      if (i >= config.rank) break;
+      auto fast_divmoder = config.divmoders[i].Divmod(index_output);
+      index_output = fast_divmoder.val[0];
+      index_src += fast_divmoder.val[1] * config.strides[i];
+    }
+    std::get<Index>(dst[nx]) = src[index_src];
+  }
+}
+
 /**
  * @brief Initialize register with data index.
  *

From 8c02766bff4c548e0e257aab7a92be84cc16630d Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Fri, 28 Apr 2023 00:54:45 +0800
Subject: [PATCH 123/405] decrease the effect of debug print on performance

---
 .../operators/fused/fused_gate_attention.h    | 137 +++++++++---------
 .../fused/fused_gate_attention_op.cu          |   1 -
 2 files changed, 67 insertions(+), 71 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 319889246890a..7fb539558e388 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -14,9 +14,8 @@ limitations under the License. */
 
 #pragma once
 
-#include "paddle/phi/kernels/arange_kernel.h"
-// #include "paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h"
 #include "paddle/phi/backends/dynload/flashattn.h"
+#include "paddle/phi/kernels/arange_kernel.h"
 #include "paddle/phi/kernels/funcs/broadcast_function.h"
 #include "paddle/phi/kernels/funcs/elementwise_functor.h"
 #include "paddle/phi/kernels/funcs/range_function.h"
@@ -53,7 +52,9 @@ void AllocWithDebugInfo(const phi::GPUContext& dev_ctx,
                         const std::string& info,
                         phi::DenseTensor* t) {
   dev_ctx.Alloc<T>(t, t->numel() * sizeof(T));
-  VLOG(4) << info << ": " << MemoryDebugString(*t);
+  if (VLOG_IS_ON(4)) {
+    VLOG(4) << info << ": " << MemoryDebugString(*t);
+  }
 }
 
 inline std::string TensorDebugString(const phi::DenseTensor* t) {
@@ -918,6 +919,7 @@ class FlashAttnWithGating {
     bool is_bf16 =
         qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
     TypeDebugInfo<T>();
+
     PADDLE_ENFORCE_NOT_NULL(
         qkv_transpose_out,
         platform::errors::NotFound("The input qkv_transpose_out can not be "
@@ -926,17 +928,19 @@ class FlashAttnWithGating {
     phi::DenseTensor* qkv_out = config->GetQKVOut();
     ComputeQKVTransposeForwardForFlashAttn(*qkv_out, qkv_transpose_out);
     config->ClearQKVOut();
-    WaitWithDebugInfo(dev_ctx_);
 
     int seq_batch_size = static_cast<int>(config->batch_size) *
                          static_cast<int>(config->seq_len_m);
-    qkv_transpose_out->Resize(
-        {3,
-         seq_batch_size * static_cast<int>(config->seq_len_r),
-         static_cast<int>(config->num_heads),
-         static_cast<int>(config->head_dim)});
-    VLOG(5) << "Reshape qkv_transpose_out: [" << config->qkv_transpose_out_dims
-            << "] -> [" << qkv_transpose_out->dims() << "]";
+    if (VLOG_IS_ON(5)) {
+      qkv_transpose_out->Resize(
+          {3,
+           seq_batch_size * static_cast<int>(config->seq_len_r),
+           static_cast<int>(config->num_heads),
+           static_cast<int>(config->head_dim)});
+      VLOG(5) << "Reshape qkv_transpose_out: ["
+              << config->qkv_transpose_out_dims << "] -> ["
+              << qkv_transpose_out->dims() << "]";
+    }
 
     // q_size == k_size
     int64_t q_size = config->GetQuerySize();
@@ -973,29 +977,21 @@ class FlashAttnWithGating {
     dims_merge_func(nonbatched_bias, &temp_bias, "temp_bias");
 
     // 4. flash_attn parameter setting.
-    auto& qkv_dims = qkv_transpose_out->dims();
+    // auto& qkv_dims = qkv_transpose_out->dims();
     int batch_size_ = seq_batch_size;
-    int total_q_ = qkv_dims[1];             // q.dims()[0]
-    int total_k_ = qkv_dims[1];             // q.dims()[0]
-    int num_heads_ = qkv_dims[2];           // q.dims()[1]
-    int head_size_ = qkv_dims[3];           // q.dims()[2]
-    int max_seqlen_q_ = config->seq_len_r;  // batch_size_;
-    int max_seqlen_k_ = config->m_size;     // batch_size_;
+    int total_q_ =
+        seq_batch_size * static_cast<int>(config->seq_len_r);  // qkv_dims[1];
+    int total_k_ = total_q_;                                   // qkv_dims[1];
+    int num_heads_ = static_cast<int>(config->num_heads);      // qkv_dims[2];
+    int head_size_ = static_cast<int>(config->head_dim);       // qkv_dims[3];
+    int max_seqlen_q_ = config->seq_len_r;                     // batch_size_;
+    int max_seqlen_k_ = config->m_size;                        // batch_size_;
     int num_splits = 0;  // 0 for an internal heuristic, which is optimal
-    VLOG(6) << "batch_size : " << batch_size_;
-    VLOG(6) << "total_q   : " << total_q_;
-    VLOG(6) << "total_k   : " << total_k_;
-    VLOG(6) << "num_heads : " << num_heads_;
-    VLOG(6) << "head_size : " << head_size_;
-    VLOG(6) << "max_seqlen_q : " << max_seqlen_q_;
-    VLOG(6) << "max_seqlen_k : " << max_seqlen_k_;
 
     // 5. construct softmax_lse
     int softmax_lse_last_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
     softmax_lse->Resize({batch_size_, num_heads_, softmax_lse_last_dim});
     AllocWithDebugInfo<float>(dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
-    WaitWithDebugInfo(dev_ctx_);
-    VLOG(5) << "Allocate softmax_lse: shape=[" << softmax_lse->dims() << "]";
 
     // 6. construct random seed
     auto seed_offset_pair = GenerateSeedOffsetPair(batch_size_, num_heads_);
@@ -1003,16 +999,26 @@ class FlashAttnWithGating {
     uint64_t offset = seed_offset_pair.second;
     VLOG(5) << "Construct random seed";
 
-    VLOG(6) << "cu_seq_q: " << TensorDebugString(&cu_seq_q);
-    VLOG(6) << "cu_seq_k: " << TensorDebugString(&cu_seq_k);
-    VLOG(6) << "temp_bias: " << TensorDebugString(&temp_bias);
-    VLOG(6) << "temp_mask: " << TensorDebugString(&temp_mask);
-    VLOG(6) << "nonbatched_bias: " << TensorDebugString(nonbatched_bias);
-    VLOG(6) << "src_mask: " << TensorDebugString(src_mask);
-    VLOG(6) << "qkv_transpose_out: " << TensorDebugString(qkv_transpose_out);
-    VLOG(6) << "softmax_lse: " << TensorDebugString(softmax_lse);
-    VLOG(6) << "fmha_out: " << TensorDebugString(fmha_out);
-    VLOG(6) << "gate_out: " << TensorDebugString(gate_out);
+    if (VLOG_IS_ON(6)) {
+      VLOG(6) << "batch_size : " << batch_size_;
+      VLOG(6) << "total_q   : " << total_q_;
+      VLOG(6) << "total_k   : " << total_k_;
+      VLOG(6) << "num_heads : " << num_heads_;
+      VLOG(6) << "head_size : " << head_size_;
+      VLOG(6) << "max_seqlen_q : " << max_seqlen_q_;
+      VLOG(6) << "max_seqlen_k : " << max_seqlen_k_;
+      VLOG(5) << "Allocate softmax_lse: shape=[" << softmax_lse->dims() << "]";
+      VLOG(6) << "cu_seq_q: " << TensorDebugString(&cu_seq_q);
+      VLOG(6) << "cu_seq_k: " << TensorDebugString(&cu_seq_k);
+      VLOG(6) << "temp_bias: " << TensorDebugString(&temp_bias);
+      VLOG(6) << "temp_mask: " << TensorDebugString(&temp_mask);
+      VLOG(6) << "nonbatched_bias: " << TensorDebugString(nonbatched_bias);
+      VLOG(6) << "src_mask: " << TensorDebugString(src_mask);
+      VLOG(6) << "qkv_transpose_out: " << TensorDebugString(qkv_transpose_out);
+      VLOG(6) << "softmax_lse: " << TensorDebugString(softmax_lse);
+      VLOG(6) << "fmha_out: " << TensorDebugString(fmha_out);
+      VLOG(6) << "gate_out: " << TensorDebugString(gate_out);
+    }
 
     // 7. flas_attn part one, get temp worksapce size.
     float p_dropout = 0.f;
@@ -1053,8 +1059,6 @@ class FlashAttnWithGating {
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
-    WaitWithDebugInfo(dev_ctx_);
-    VLOG(5) << "Get workspace_size=" << workspace_size;
 
     // 8. Run flash-attention kernel.
     phi::DenseTensor workspace = CreateWorkspace(workspace_size);
@@ -1134,7 +1138,6 @@ class FlashAttnWithGating {
 
     int seq_batch_size = static_cast<int>(config->batch_size) *
                          static_cast<int>(config->seq_len_m);
-    WaitWithDebugInfo(dev_ctx_);
 
     // 2. Init with cu_seq_q and cu_seq_k for flash_attn.
     phi::DenseTensor cu_seq_q, cu_seq_k;
@@ -1164,48 +1167,49 @@ class FlashAttnWithGating {
     dims_merge_func(src_mask, &temp_mask, "temp_mask");
     dims_merge_func(nonbatched_bias, &temp_bias, "temp_bias");
 
-    phi::DDim qkv_dims({3,
-                        seq_batch_size * static_cast<int>(config->seq_len_r),
-                        static_cast<int>(config->num_heads),
-                        static_cast<int>(config->head_dim)});
+    // phi::DDim qkv_dims({3,
+    //                     seq_batch_size * static_cast<int>(config->seq_len_r),
+    //                     static_cast<int>(config->num_heads),
+    //                     static_cast<int>(config->head_dim)});
     int batch_size_ = seq_batch_size;
-    int total_q_ = qkv_dims[1];    // q.dims()[0]
-    int total_k_ = qkv_dims[1];    // q.dims()[0]
-    int num_heads_ = qkv_dims[2];  // q.dims()[1]
-    int head_size_ = qkv_dims[3];  // q.dims()[2]
+    int total_q_ = seq_batch_size * static_cast<int>(config->seq_len_r);
+    int total_k_ = total_q_;                               // qkv_dims[1];
+    int num_heads_ = static_cast<int>(config->num_heads);  // qkv_dims[2];
+    int head_size_ = static_cast<int>(config->head_dim);   // qkv_dims[3];
     int max_seqlen_q_ = config->seq_len_r;
     int max_seqlen_k_ = config->m_size;
-    VLOG(6) << "batch_size : " << batch_size_;
-    VLOG(6) << "total_q   : " << total_q_;
-    VLOG(6) << "total_k   : " << total_k_;
-    VLOG(6) << "num_heads : " << num_heads_;
-    VLOG(6) << "head_size : " << head_size_;
-    VLOG(6) << "max_seqlen_q : " << max_seqlen_q_;
-    VLOG(6) << "max_seqlen_k : " << max_seqlen_k_;
 
     // 5. construct softmax_lse
     phi::DenseTensor softmax_d;
     softmax_d.Resize(softmax_lse->dims());
     AllocWithDebugInfo<float>(dev_ctx_, "d_softmax_lse", &softmax_d);
-    WaitWithDebugInfo(dev_ctx_);
 
     phi::DenseTensor bias_d;
     if (nonbatched_bias) {
       bias_d = phi::Empty<T, phi::GPUContext>(
           dev_ctx_, {batch_size_, num_heads_, max_seqlen_q_, max_seqlen_k_});
     }
-    WaitWithDebugInfo(dev_ctx_);
 
     // 6. construct random seed
     auto seed_offset_pair = GenerateSeedOffsetPair(batch_size_, num_heads_);
     uint64_t seed = seed_offset_pair.first;
     uint64_t offset = seed_offset_pair.second;
-    VLOG(6) << "fmha_out: " << TensorDebugString(fmha_out);
-    VLOG(6) << "fmha_out_grad: " << TensorDebugString(fmha_out_grad);
-    VLOG(6) << "softmax_lse: " << TensorDebugString(softmax_lse);
-    VLOG(6) << "softmax_d: " << TensorDebugString(&softmax_d);
-    VLOG(6) << "nonbatched_bias: " << TensorDebugString(nonbatched_bias);
-    VLOG(6) << "bias_d: " << TensorDebugString(&bias_d);
+
+    if (VLOG_IS_ON(6)) {
+      VLOG(6) << "batch_size : " << batch_size_;
+      VLOG(6) << "total_q   : " << total_q_;
+      VLOG(6) << "total_k   : " << total_k_;
+      VLOG(6) << "num_heads : " << num_heads_;
+      VLOG(6) << "head_size : " << head_size_;
+      VLOG(6) << "max_seqlen_q : " << max_seqlen_q_;
+      VLOG(6) << "max_seqlen_k : " << max_seqlen_k_;
+      VLOG(6) << "fmha_out: " << TensorDebugString(fmha_out);
+      VLOG(6) << "fmha_out_grad: " << TensorDebugString(fmha_out_grad);
+      VLOG(6) << "softmax_lse: " << TensorDebugString(softmax_lse);
+      VLOG(6) << "softmax_d: " << TensorDebugString(&softmax_d);
+      VLOG(6) << "nonbatched_bias: " << TensorDebugString(nonbatched_bias);
+      VLOG(6) << "bias_d: " << TensorDebugString(&bias_d);
+    }
 
     // 7. flas_attn part one, get temp worksapce size.
     uint64_t workspace_size;
@@ -1252,8 +1256,6 @@ class FlashAttnWithGating {
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
-    VLOG(5) << "Get workspace_size=" << workspace_size;
-
     phi::DenseTensor workspace = CreateWorkspace(workspace_size);
     succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
         static_cast<const void*>(q_ptr),
@@ -1299,9 +1301,6 @@ class FlashAttnWithGating {
 
     if (nonbatched_bias) {
       // compare block reduce
-      // auto size = attn_bias->sizes();
-      // dbias = ds.reshape({ -1, size[0], size[1], size[2], size[3] }).sum({
-      // 0 }); result.push_back( dbias );
       const auto temp_bias_num = temp_bias.numel();
       const auto bias_d_num = bias_d.numel();
       auto dbias_first_dim = bias_d_num / temp_bias_num;
@@ -1342,7 +1341,6 @@ class FlashAttnWithGating {
     int64_t grid = (seq_size + block - 1) / block;
     FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
         start, step, end, cu_seq_q->data<int32_t>(), cu_seq_k->data<int32_t>());
-    WaitWithDebugInfo(dev_ctx_);
     VLOG(5) << "AllocAndInit cu_seq_q and cu_seq_k: start=" << start
             << ", step=" << step << ", end=" << end;
   }
@@ -1353,7 +1351,6 @@ class FlashAttnWithGating {
       workspace = phi::Empty<float, phi::GPUContext>(
           dev_ctx_, {int64_t(workspace_size / sizeof(float))});
     }
-    WaitWithDebugInfo(dev_ctx_);
     VLOG(5) << "Allocate workspace: workspace_size=" << workspace_size;
     return workspace;
   }
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cu b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
index 03ccc1282fa51..2af6c9088f4f7 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cu
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
@@ -428,7 +428,6 @@ class FusedGateAttentionOpKernel : public framework::OpKernel<T> {
     // 2. FMHA
     if (config.CanUseFlashAttn()) {
       auto *softmax_lse = ctx.Output<phi::DenseTensor>("SoftmaxLse");
-
       auto fmha_compute = FlashAttnWithGating<T>(dev_ctx, merge_qkv);
       fmha_compute.ComputeForward(nonbatched_bias,
                                   src_mask,

From 05499c71355f0f83391d824870da909bba37988a Mon Sep 17 00:00:00 2001
From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com>
Date: Fri, 28 Apr 2023 09:19:06 +0800
Subject: [PATCH 124/405] =?UTF-8?q?=E3=80=90Prim=E3=80=91comp=5Felementwis?=
 =?UTF-8?q?e=5Fdouble=5Fgrad=20(first=20part)=20(#53385)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add mul doubel grad

* add sub_double_grad

* add add sub high test

* add mutiply test

* modify other unsqueeze

* delete api.yaml

* only for make ci run

* midify unsqueeze

* modify unsqueeze

* tmp

* modify operants gen
---
 cmake/external/cinn.cmake                     |   2 +-
 .../generator/eager_gen.py                    |   3 +
 .../elementwise/elementwise_add_op.cc         |  39 +-
 .../elementwise/elementwise_mul_op.cc         |  53 ++-
 .../elementwise/elementwise_sub_op.cc         |  40 ++-
 paddle/fluid/prim/api/api.yaml                |   1 -
 .../composite_backward_api.h                  | 143 +++++++-
 .../fluid/prim/api/manual_prim/utils/utils.h  |  18 +
 paddle/phi/api/yaml/backward.yaml             |  13 -
 paddle/phi/api/yaml/legacy_backward.yaml      |  27 +-
 paddle/phi/api/yaml/op_compat.yaml            |   2 +-
 paddle/phi/api/yaml/tensor_operants.yaml      |   1 -
 python/paddle/fluid/backward.py               |  12 +-
 .../unittests/test_activation_nn_grad.py      |   8 +
 test/prim/prim/vjp/CMakeLists.txt             |   2 +
 test/prim/prim/vjp/test_comp_high_grad.py     | 334 ++++++++++++++++++
 16 files changed, 640 insertions(+), 58 deletions(-)
 create mode 100644 test/prim/prim/vjp/test_comp_high_grad.py

diff --git a/cmake/external/cinn.cmake b/cmake/external/cinn.cmake
index c41e094a25e9e..7d494ef516cae 100644
--- a/cmake/external/cinn.cmake
+++ b/cmake/external/cinn.cmake
@@ -20,7 +20,7 @@ if(NOT CINN_GIT_TAG)
   set(CINN_GIT_TAG develop)
 endif()
 
-message(STATUS "CINN version: " ${CINN_GIT_TAG})
+message(STATUS "CINN  version: " ${CINN_GIT_TAG})
 
 # TODO(zhhsplendid): CINN has lots of warnings during early development.
 # They will be treated as errors under paddle. We set no-error now and we will
diff --git a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
index c5db4aee73de0..f69b5944c5603 100644
--- a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
+++ b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
@@ -67,6 +67,9 @@
 prim_white_list = [
     "matmul_double_grad",
     "tanh_double_grad",
+    "add_double_grad",
+    "multiply_double_grad",
+    "subtract_double_grad",
 ]
 
 # dict of special api that forward api's output will affect bacward api's output
diff --git a/paddle/fluid/operators/elementwise/elementwise_add_op.cc b/paddle/fluid/operators/elementwise/elementwise_add_op.cc
index 872977b616166..0b522c1d6b35b 100644
--- a/paddle/fluid/operators/elementwise/elementwise_add_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_add_op.cc
@@ -99,6 +99,42 @@ class ElementwiseAddDoubleGradMaker : public framework::SingleGradOpMaker<T> {
   }
 };
 
+class ElementwiseAddCompositeDoubleGradOpMaker
+    : public prim::CompositeGradOpMakerBase {
+  using prim::CompositeGradOpMakerBase::CompositeGradOpMakerBase;
+
+ public:
+  void Apply() override {
+    // get input
+    paddle::Tensor y = this->GetSingleForwardInput("Y");
+    paddle::Tensor out_grad = this->GetSingleOutputGrad("Out");
+    paddle::optional<paddle::Tensor> ddx =
+        this->GetOptionalSingleOutputGrad(framework::GradVarName("X"));
+    paddle::optional<paddle::Tensor> ddy =
+        this->GetOptionalSingleOutputGrad(framework::GradVarName("Y"));
+    // get output
+    paddle::Tensor grad_out_grad_t =
+        this->GetSingleInputGrad(framework::GradVarName("Out"));
+
+    // get attr
+    int axis = static_cast<int>(this->Attr<int>("axis"));
+    PADDLE_ENFORCE_EQ(
+        axis,
+        -1,
+        phi::errors::InvalidArgument("We only support axis = -1 in composite "
+                                     "add_doubel_grad but we got: ",
+                                     axis));
+
+    paddle::Tensor* grad_out_grad = this->GetOutputPtr(&grad_out_grad_t);
+    std::string grad_out_grad_name = this->GetOutputName(grad_out_grad_t);
+
+    VLOG(6) << "Runing add_double_grad composite func";
+    prim::add_double_grad<prim::DescTensor>(
+        y, out_grad, ddx, ddy, axis, grad_out_grad);
+    this->RecoverOutputName(grad_out_grad_t, grad_out_grad_name);
+  }
+};
+
 template <typename T>
 class ElementwiseAddTripleGradMaker : public framework::SingleGradOpMaker<T> {
  public:
@@ -139,7 +175,8 @@ REGISTER_OPERATOR(
     ops::ElementwiseGradOpInplaceInferer,
     ops::ElementwiseGradNoBufVarsInferer,
     ops::ElementwiseAddDoubleGradMaker<paddle::framework::OpDesc>,
-    ops::ElementwiseAddDoubleGradMaker<paddle::imperative::OpBase>);
+    ops::ElementwiseAddDoubleGradMaker<paddle::imperative::OpBase>,
+    ops::ElementwiseAddCompositeDoubleGradOpMaker);
 
 REGISTER_OPERATOR(
     elementwise_add_grad_grad,
diff --git a/paddle/fluid/operators/elementwise/elementwise_mul_op.cc b/paddle/fluid/operators/elementwise/elementwise_mul_op.cc
index c4a1060497ea1..0fc1cab739105 100644
--- a/paddle/fluid/operators/elementwise/elementwise_mul_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_mul_op.cc
@@ -118,6 +118,56 @@ class ElementwiseMulDoubleGradMaker : public framework::SingleGradOpMaker<T> {
   }
 };
 
+class ElementwiseMulCompositeDoubleGradOpMaker
+    : public prim::CompositeGradOpMakerBase {
+  using prim::CompositeGradOpMakerBase::CompositeGradOpMakerBase;
+
+ public:
+  void Apply() override {
+    // get input
+    paddle::Tensor x = this->GetSingleForwardInput("X");
+    paddle::Tensor y = this->GetSingleForwardInput("Y");
+    paddle::Tensor out_grad = this->GetSingleOutputGrad("Out");
+    paddle::optional<paddle::Tensor> ddx =
+        this->GetOptionalSingleOutputGrad(framework::GradVarName("X"));
+    paddle::optional<paddle::Tensor> ddy =
+        this->GetOptionalSingleOutputGrad(framework::GradVarName("Y"));
+
+    // get attr
+    int axis = static_cast<int>(this->Attr<int>("axis"));
+    PADDLE_ENFORCE_EQ(
+        axis,
+        -1,
+        phi::errors::InvalidArgument("We only support axis = -1 in composite "
+                                     "add_doubel_grad but we got: ",
+                                     axis));
+
+    // get output
+    paddle::Tensor x_grad_t = this->GetSingleInputGrad("X");
+    paddle::Tensor y_grad_t = this->GetSingleInputGrad("Y");
+    paddle::Tensor grad_out_grad_t =
+        this->GetSingleInputGrad(framework::GradVarName("Out"));
+
+    // get output ptr
+    paddle::Tensor* x_grad = this->GetOutputPtr(&x_grad_t);
+    paddle::Tensor* y_grad = this->GetOutputPtr(&y_grad_t);
+    paddle::Tensor* grad_out_grad = this->GetOutputPtr(&grad_out_grad_t);
+    // get output orginal name
+    std::string x_grad_name = this->GetOutputName(x_grad_t);
+    std::string y_grad_name = this->GetOutputName(y_grad_t);
+    std::string grad_out_grad_name = this->GetOutputName(grad_out_grad_t);
+
+    VLOG(6) << "Runing multiply_double_grad composite func";
+    prim::multiply_double_grad<prim::DescTensor>(
+        x, y, out_grad, ddx, ddy, axis, x_grad, y_grad, grad_out_grad);
+
+    // recover output name
+    this->RecoverOutputName(x_grad_t, x_grad_name);
+    this->RecoverOutputName(y_grad_t, y_grad_name);
+    this->RecoverOutputName(grad_out_grad_t, grad_out_grad_name);
+  }
+};
+
 template <typename T>
 class ElementwiseMulTripleGradMaker : public framework::SingleGradOpMaker<T> {
  public:
@@ -162,7 +212,8 @@ REGISTER_OPERATOR(
     elementwise_mul_grad,
     ops::ElementwiseOpGrad,
     ops::ElementwiseMulDoubleGradMaker<paddle::framework::OpDesc>,
-    ops::ElementwiseMulDoubleGradMaker<paddle::imperative::OpBase>);
+    ops::ElementwiseMulDoubleGradMaker<paddle::imperative::OpBase>,
+    ops::ElementwiseMulCompositeDoubleGradOpMaker);
 
 REGISTER_OPERATOR(
     elementwise_mul_grad_grad,
diff --git a/paddle/fluid/operators/elementwise/elementwise_sub_op.cc b/paddle/fluid/operators/elementwise/elementwise_sub_op.cc
index f1df64420063a..0c8c9bc6e804c 100644
--- a/paddle/fluid/operators/elementwise/elementwise_sub_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_sub_op.cc
@@ -102,6 +102,42 @@ class ElementwiseSubDoubleGradMaker : public framework::SingleGradOpMaker<T> {
   }
 };
 
+class ElementwiseSubCompositeDoubleGradOpMaker
+    : public prim::CompositeGradOpMakerBase {
+  using prim::CompositeGradOpMakerBase::CompositeGradOpMakerBase;
+
+ public:
+  void Apply() override {
+    // get input
+    paddle::Tensor y = this->GetSingleForwardInput("Y");
+    paddle::Tensor out_grad = this->GetSingleOutputGrad("Out");
+    paddle::optional<paddle::Tensor> ddx =
+        this->GetOptionalSingleOutputGrad(framework::GradVarName("X"));
+    paddle::optional<paddle::Tensor> ddy =
+        this->GetOptionalSingleOutputGrad(framework::GradVarName("Y"));
+    // get output
+    paddle::Tensor grad_out_grad_t =
+        this->GetSingleInputGrad(framework::GradVarName("Out"));
+
+    // get attr
+    int axis = static_cast<int>(this->Attr<int>("axis"));
+    PADDLE_ENFORCE_EQ(
+        axis,
+        -1,
+        phi::errors::InvalidArgument("We only support axis = -1 in composite "
+                                     "subtract_doubel_grad but we got: ",
+                                     axis));
+
+    paddle::Tensor* grad_out_grad = this->GetOutputPtr(&grad_out_grad_t);
+    std::string grad_out_grad_name = this->GetOutputName(grad_out_grad_t);
+
+    VLOG(6) << "Runing subtract_double_grad composite func";
+    prim::subtract_double_grad<prim::DescTensor>(
+        y, out_grad, ddx, ddy, axis, grad_out_grad);
+    this->RecoverOutputName(grad_out_grad_t, grad_out_grad_name);
+  }
+};
+
 }  // namespace operators
 }  // namespace paddle
 
@@ -124,7 +160,9 @@ REGISTER_OPERATOR(
     ops::ElementwiseGradOpInplaceInferer,
     ops::ElementwiseGradNoBufVarsInferer,
     ops::ElementwiseSubDoubleGradMaker<paddle::framework::OpDesc>,
-    ops::ElementwiseSubDoubleGradMaker<paddle::imperative::OpBase>);
+    ops::ElementwiseSubDoubleGradMaker<paddle::imperative::OpBase>,
+    ops::ElementwiseSubCompositeDoubleGradOpMaker);
+
 REGISTER_OPERATOR(elementwise_sub_grad_grad,
                   ops::ElementwiseOpDoubleGradWithoutDXDY,
                   ops::ElementwiseDoubleGradOpInplaceInferer,
diff --git a/paddle/fluid/prim/api/api.yaml b/paddle/fluid/prim/api/api.yaml
index c626d39cf7bdb..ec3bd5741371e 100644
--- a/paddle/fluid/prim/api/api.yaml
+++ b/paddle/fluid/prim/api/api.yaml
@@ -12,7 +12,6 @@
 - bitwise_not
 - bitwise_or
 - bitwise_xor
-- unsqueeze
 - exp
 - scale
 - matmul
diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
index 4784f2fb61751..be825f52362ee 100644
--- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
+++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
@@ -249,6 +249,30 @@ void subtract_grad(const Tensor& x,
   }
 }
 
+template <typename T>
+void subtract_double_grad(const Tensor& y,
+                          const Tensor& grad_out,
+                          const paddle::optional<Tensor>& grad_x_grad,
+                          const paddle::optional<Tensor>& grad_y_grad,
+                          int axis,
+                          Tensor* grad_out_grad) {
+  if (grad_out_grad) {
+    // ddout = ddx - ddy
+    if (!grad_x_grad && !grad_y_grad) {
+      grad_out_grad = nullptr;
+    } else {
+      Tensor ddout = full<T>(phi::vectorize(grad_out.dims()), 0.0, y.dtype());
+      if (grad_x_grad) {
+        ddout = ddout + grad_x_grad.get();
+      }
+      if (grad_y_grad) {
+        ddout = ddout - grad_y_grad.get();
+      }
+      set_output<T>(ddout, grad_out_grad);
+    }
+  }
+}
+
 template <typename T>
 void add_grad(const Tensor& x,
               const Tensor& y,
@@ -291,6 +315,30 @@ void add_grad(const Tensor& x,
   }
 }
 
+template <typename T>
+void add_double_grad(const Tensor& y,
+                     const Tensor& grad_out,
+                     const paddle::optional<Tensor>& grad_x_grad,
+                     const paddle::optional<Tensor>& grad_y_grad,
+                     int axis,
+                     Tensor* grad_out_grad) {
+  if (grad_out_grad) {
+    // ddout = ddx + ddy
+    if (!grad_x_grad && !grad_y_grad) {
+      grad_out_grad = nullptr;
+    } else {
+      Tensor ddout = full<T>(phi::vectorize(grad_out.dims()), 0.0, y.dtype());
+      if (grad_x_grad) {
+        ddout = ddout + grad_x_grad.get();
+      }
+      if (grad_y_grad) {
+        ddout = ddout + grad_y_grad.get();
+      }
+      set_output<T>(ddout, grad_out_grad);
+    }
+  }
+}
+
 template <typename T>
 void sum_grad(const Tensor& x,
               const Tensor& out_grad,
@@ -328,7 +376,8 @@ void sum_grad(const Tensor& x,
           }
         }
       }
-      auto out_grad_ = unsqueeze<T>(out_grad, axis_);
+      auto out_grad_shape = get_unsqueeze_dims(out_grad, axis_);
+      auto out_grad_ = reshape<T>(out_grad, out_grad_shape);
       x_grad_tmp = out_grad_.expand(IntArray(x_dim));
     } else {
       x_grad_tmp = out_grad.expand(IntArray(x_dim));
@@ -521,6 +570,75 @@ void multiply_grad(const Tensor& x,
   }
 }
 
+template <typename T>
+void multiply_double_grad(const Tensor& x,
+                          const Tensor& y,
+                          const Tensor& grad_out,
+                          const paddle::optional<Tensor>& grad_x_grad,
+                          const paddle::optional<Tensor>& grad_y_grad,
+                          int axis,
+                          Tensor* x_grad,
+                          Tensor* y_grad,
+                          Tensor* grad_out_grad) {
+  if (x_grad) {
+    if (grad_y_grad) {
+      auto dx = grad_y_grad.get() * grad_out;
+      if (dx.dims() != x.dims()) {
+        auto axes = get_reduce_dims_from_out(dx.dims(), x.dims());
+        if (!axes.size()) {
+          set_output<T>(dx, x_grad);
+        } else {
+          auto dx_reduce = dx.sum(phi::vectorize(axes), dx.dtype(), false);
+          if (dx_reduce.dims().size() != x.dims().size()) {
+            dx_reduce = reshape<T>(dx_reduce, x.shape());
+          }
+          set_output<T>(dx_reduce, x_grad);
+        }
+      } else {
+        set_output<T>(dx, x_grad);
+      }
+
+    } else {
+      x_grad = nullptr;
+    }
+  }
+  if (y_grad) {
+    if (grad_x_grad) {
+      auto dy = grad_x_grad.get() * grad_out;
+      if (dy.dims() != y.dims()) {
+        auto axes = get_reduce_dims_from_out(dy.dims(), y.dims());
+        if (!axes.size()) {
+          set_output<T>(dy, y_grad);
+        } else {
+          auto dy_reduce = dy.sum(phi::vectorize(axes), dy.dtype(), false);
+          if (dy_reduce.dims().size() != y.dims().size()) {
+            dy_reduce = reshape<T>(dy_reduce, y.shape());
+          }
+          set_output<T>(dy_reduce, y_grad);
+        }
+      } else {
+        set_output<T>(dy, y_grad);
+      }
+    } else {
+      y_grad = nullptr;
+    }
+  }
+  if (grad_out_grad) {
+    if (grad_x_grad && grad_y_grad) {
+      auto ddout = grad_x_grad.get() * y + grad_y_grad.get() * x;
+      set_output<T>(ddout, grad_out_grad);
+    } else if (grad_x_grad) {
+      auto ddout = grad_x_grad.get() * y;
+      set_output<T>(ddout, grad_out_grad);
+    } else if (grad_y_grad) {
+      auto ddout = grad_y_grad.get() * x;
+      set_output<T>(ddout, grad_out_grad);
+    } else {
+      grad_out_grad = nullptr;
+    }
+  }
+}
+
 template <typename T>
 void expand_grad(const Tensor& x,
                  const Tensor& out_grad,
@@ -1063,9 +1181,11 @@ void group_norm_grad(const Tensor& x,
 
     auto p2 = (d2 * mean - d1) * (inv_std_mul_s * inv_std * inv_std);
     auto p3 = -p2 * mean - d2 * inv_std_mul_s;
-    p1 = unsqueeze<T>(p1, std::vector<int64_t>({3}));
-    p2 = unsqueeze<T>(p2, std::vector<int64_t>({2, 3}));
-    p3 = unsqueeze<T>(p3, std::vector<int64_t>({2, 3}));
+    auto first_shape = get_unsqueeze_dims(p1, std::vector<int64_t>({3}));
+    auto second_shape = get_unsqueeze_dims(p2, std::vector<int64_t>({2, 3}));
+    p1 = reshape<T>(p1, first_shape);
+    p2 = reshape<T>(p2, second_shape);
+    p3 = reshape<T>(p3, second_shape);
     auto tmp_1 = reshape<T>(out_grad_data, whole_group_shape) * p1;
     auto tmp_2 = reshape<T>(x_data, whole_group_shape) * p2 + p3;
     auto x_grad_data = tmp_1 + tmp_2;
@@ -1078,10 +1198,11 @@ void group_norm_grad(const Tensor& x,
   }
   if (scale_grad) {
     if (scale_ptr) {
+      auto third_shape = get_unsqueeze_dims(mean, std::vector<int64_t>({2}));
       auto tmp1 = (reshape<T>(sum_y_grad_mul_x, shape_group) -
                    reshape<T>(sum_y_grad, shape_group) *
-                       unsqueeze<T>(mean, std::vector<int64_t>({2}))) *
-                  unsqueeze<T>(inv_std, std::vector<int64_t>({2}));
+                       reshape<T>(mean, third_shape)) *
+                  reshape<T>(inv_std, third_shape);
       auto scale_grad_tmp =
           reshape<T>(tmp1.sum(std::vector<int64_t>({0}), dtype, false),
                      IntArray(std::vector<int64_t>({C})));
@@ -1291,9 +1412,10 @@ void prod_grad(const Tensor& x,
             }
           }
         }
-        auto out_grad_ = unsqueeze<T>(out_grad, axis_);
+        auto out_grad_shape = get_unsqueeze_dims(out_grad, axis_);
+        auto out_grad_ = reshape<T>(out_grad, out_grad_shape);
         x_grad_tmp = out_grad_.expand(IntArray(x_dim));
-        auto out_ = unsqueeze<T>(out, axis_);
+        auto out_ = reshape<T>(out, out_grad_shape);
         out_tmp = out_.expand(IntArray(x_dim));
       } else {
         x_grad_tmp = out_grad.expand(IntArray(x_dim));
@@ -1346,8 +1468,9 @@ void max_grad(const Tensor& x,
         }
       }
     }
-    auto out_grad_ = unsqueeze<T>(out_grad, axis_);
-    auto out_ = unsqueeze<T>(out, axis_);
+    auto out_grad_shape = get_unsqueeze_dims(out_grad, axis_);
+    auto out_grad_ = reshape<T>(out_grad, out_grad_shape);
+    auto out_ = reshape<T>(out, out_grad_shape);
     auto out_grad_tmp = out_grad_.expand(IntArray(x_dim));
     auto out_tmp = out_.expand(IntArray(x_dim));
     auto mask = equal<T>(x, out_tmp);
diff --git a/paddle/fluid/prim/api/manual_prim/utils/utils.h b/paddle/fluid/prim/api/manual_prim/utils/utils.h
index 338960d44849f..d72da6461da75 100644
--- a/paddle/fluid/prim/api/manual_prim/utils/utils.h
+++ b/paddle/fluid/prim/api/manual_prim/utils/utils.h
@@ -114,5 +114,23 @@ static std::vector<DST_T> unsafe_vector_cast(const std::vector<SRC_T>& src) {
   return dst;
 }
 
+// This fucction compute unsqueeze dims for reshape to replace unsqueeze.
+static std::vector<int> get_unsqueeze_dims(const Tensor& origin,
+                                           const IntArray& axis) {
+  auto origin_dims = origin.shape();
+  auto total_shape_size = origin_dims.size() + axis.size();
+  std::vector<int> result;
+  int j = 0, k = 0;
+  for (size_t i = 0; i < total_shape_size; ++i) {
+    if (axis[j] == int64_t(i)) {
+      result.push_back(1);
+      j++;
+    } else {
+      result.push_back(origin_dims[k]);
+      k++;
+    }
+  }
+  return result;
+}
 }  // namespace prim
 }  // namespace paddle
diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index a2189b7084bd2..97a0968f17198 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -1865,7 +1865,6 @@
   kernel :
     func : tanh_double_grad
   composite : tanh_double_grad(out, grad_out, grad_x_grad, out_grad, grad_out_grad)
-  backward : tanh_triple_grad
   inplace : (grad_x_grad -> grad_out_grad)
 
 - backward_op : tanh_grad
@@ -1892,18 +1891,6 @@
     func : tanh_shrink_grad
   inplace : (out_grad -> x_grad)
 
-- backward_op : tanh_triple_grad
-  forward : tanh_double_grad (Tensor out, Tensor grad_out_forward, Tensor grad_x_grad_forward) -> Tensor(grad_out_new), Tensor(grad_out_grad)
-  args : (Tensor out, Tensor grad_out_forward, Tensor grad_x_grad_forward, Tensor grad_out_new_grad, Tensor grad_out_grad_grad)
-  output : Tensor(out_grad), Tensor(grad_out_forward_grad), Tensor(grad_x_grad_forward_grad)
-  infer_meta :
-    func : GeneralTernaryGradInferMeta
-    param : [out, out, grad_x_grad_forward]
-  kernel :
-    func : tanh_triple_grad
-  inplace : (grad_x_grad_forward -> grad_out_forward_grad)
-  optional : grad_out_new_grad, grad_out_grad_grad
-
 - backward_op : temporal_shift_grad
   forward : temporal_shift(Tensor x, int seg_num, float shift_ratio = 0.25f, str data_format = "NCHW") -> Tensor(out)
   args : (Tensor out_grad, int seg_num, float shift_ratio, str data_format)
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index 30bfa1f384758..5f971d988538e 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -30,8 +30,8 @@
   kernel :
     func : add_double_grad
   optional : grad_x_grad, grad_y_grad
-  backward : add_triple_grad
   inplace : (grad_x_grad -> grad_out_grad)
+  composite : add_double_grad(y, grad_out, grad_x_grad, grad_y_grad, axis, grad_out_grad)
 
 - backward_op : add_grad
   forward : add (Tensor x, Tensor y) -> Tensor(out)
@@ -47,17 +47,6 @@
   backward : add_double_grad
   inplace : (out_grad -> x_grad)
 
-- backward_op : add_triple_grad
-  forward : add_double_grad (Tensor y, Tensor grad_out, Tensor grad_grad_x, Tensor grad_grad_y, int axis = -1) -> Tensor(grad_grad_out)
-  args : (Tensor grad_grad_x, Tensor grad_grad_y, Tensor grad_grad_out_grad, int axis = -1)
-  output : Tensor(grad_grad_x_grad), Tensor(grad_grad_y_grad)
-  infer_meta :
-    func : GeneralBinaryGradInferMeta
-    param : [grad_grad_x, grad_grad_y]
-  kernel :
-    func : add_triple_grad
-  inplace : (grad_grad_out_grad -> grad_grad_x_grad)
-
 - backward_op : amax_grad
   forward: amax (Tensor x,  int64_t[] axis={},  bool keepdim=false) -> Tensor(out)
   args : (Tensor x, Tensor out, Tensor out_grad, int64_t[] axis={},  bool keepdim=false, bool reduce_all=false)
@@ -627,8 +616,8 @@
   kernel :
     func : multiply_double_grad
   optional : grad_x_grad, grad_y_grad
-  backward : multiply_triple_grad
   inplace : (grad_x_grad -> grad_out_grad)
+  composite : multiply_double_grad(x, y, grad_out, grad_x_grad, grad_y_grad, axis, x_grad, y_grad, grad_out_grad)
 
 - backward_op : multiply_grad
   forward : multiply (Tensor x, Tensor y) -> Tensor(out)
@@ -642,17 +631,6 @@
   composite: multiply_grad(x, y, out_grad, axis, x_grad, y_grad)
   backward : multiply_double_grad
 
-- backward_op : multiply_triple_grad
-  forward : multiply_double_grad (Tensor x, Tensor y, Tensor fwd_grad_out, Tensor fwd_grad_grad_x, Tensor fwd_grad_grad_y, int aixs = -1) -> Tensor(grad_x), Tensor(grad_y), Tensor(grad_grad_out)
-  args : (Tensor x, Tensor y, Tensor fwd_grad_out, Tensor fwd_grad_grad_x, Tensor fwd_grad_grad_y, Tensor grad_x_grad, Tensor grad_y_grad, Tensor grad_grad_out_grad, int axis = -1)
-  output : Tensor(x_grad), Tensor(y_grad), Tensor(fwd_grad_out_grad), Tensor(fwd_grad_grad_x_grad), Tensor(fwd_grad_grad_y_grad)
-  infer_meta :
-    func : GeneralQuinaryGradInferMeta
-    param : [x, y, fwd_grad_out, fwd_grad_grad_x, fwd_grad_grad_y]
-  kernel :
-    func : multiply_triple_grad
-  optional : fwd_grad_grad_x, fwd_grad_grad_y, grad_x_grad, grad_y_grad, grad_grad_out_grad
-
 - backward_op : norm_grad
   forward : norm (Tensor x, int axis, float epsilon, bool is_test) -> Tensor(out), Tensor(norm)
   args : (Tensor x, Tensor norm, Tensor out_grad, int axis, float epsilon, bool is_test)
@@ -940,6 +918,7 @@
   optional : grad_x_grad, grad_y_grad
   no_need_buffer : y, grad_out
   inplace : (grad_x_grad -> grad_out_grad)
+  composite : subtract_double_grad(y, grad_out, grad_x_grad, grad_y_grad, axis, grad_out_grad)
 
 - backward_op : subtract_grad
   forward : subtract (Tensor x, Tensor y) -> Tensor(out)
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index 46245aa4c928e..c3d326eafbfae 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -2274,7 +2274,7 @@
     attrs : [bool use_mkldnn = false, bool use_cudnn = false]
 
 - op : tanh
-  backward : tanh_grad, tanh_double_grad (tanh_grad_grad), tanh_triple_grad
+  backward : tanh_grad, tanh_double_grad (tanh_grad_grad)
   inputs :
     x : X
   outputs :
diff --git a/paddle/phi/api/yaml/tensor_operants.yaml b/paddle/phi/api/yaml/tensor_operants.yaml
index 8c0b59fcdc4e4..c29cf1a933f0c 100644
--- a/paddle/phi/api/yaml/tensor_operants.yaml
+++ b/paddle/phi/api/yaml/tensor_operants.yaml
@@ -14,7 +14,6 @@
 - bitwise_not
 - bitwise_or
 - bitwise_xor
-- unsqueeze
 - exp
 - scale
 - matmul
diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py
index fa7ae57ff9d97..e6c01f836851a 100755
--- a/python/paddle/fluid/backward.py
+++ b/python/paddle/fluid/backward.py
@@ -1492,17 +1492,21 @@ def find_op_index(block_desc, cur_op_desc):
                 or name in input_grad_names_set
             )
             is_append_grad = False
+            input_grad_names = []
             for op_desc in grad_op_desc:
-                input_grad_names = [
+                input_grad_names += [
                     name
                     for name in op_desc.input_arg_names()
                     if is_grad_name(name)
                 ]
+            if len(input_grad_names) == 0:
+                is_append_grad = True
+                break
+
+            for op_desc in grad_op_desc:
+
                 # some code of gradient ops, like increment, are not very
                 # standard, there is no @GRAD in these ops' inputs.
-                if len(input_grad_names) == 0:
-                    is_append_grad = True
-                    break
 
                 if _some_in_set_(input_grad_names, input_grad_names_set):
                     grad_op_descs.append(op_desc)
diff --git a/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py b/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py
index 882d5446540d3..cddeb3daeedd7 100644
--- a/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py
+++ b/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py
@@ -92,12 +92,16 @@ def func(self, place):
         y = paddle.tanh(x)
         x_arr = np.random.random(shape).astype(dtype)
         x_arr[np.abs(x_arr) < 0.005] = 0.002
+        from paddle.fluid import core
+
+        core._set_prim_backward_enabled(True)
         gradient_checker.triple_grad_check(
             [x], y, x_init=x_arr, place=place, eps=eps
         )
         gradient_checker.triple_grad_check_for_dygraph(
             self.tanh_wrapper, [x], y, x_init=x_arr, place=place
         )
+        core._set_prim_backward_enabled(False)
 
     def test_grad(self):
         paddle.enable_static()
@@ -122,12 +126,16 @@ def func(self, place):
         y = paddle.tanh(x)
         x_arr = np.random.uniform(-1, 1, shape).astype(dtype)
         x_arr[np.abs(x_arr) < 0.005] = 0.002
+        from paddle.fluid import core
+
+        core._set_prim_backward_enabled(True)
         gradient_checker.double_grad_check(
             [x], y, x_init=x_arr, place=place, eps=eps
         )
         gradient_checker.double_grad_check_for_dygraph(
             self.tanh_wrapper, [x], y, x_init=x_arr, place=place
         )
+        core._set_prim_backward_enabled(False)
 
     def test_grad(self):
         paddle.enable_static()
diff --git a/test/prim/prim/vjp/CMakeLists.txt b/test/prim/prim/vjp/CMakeLists.txt
index d71096db0a142..c7cae170629dd 100644
--- a/test/prim/prim/vjp/CMakeLists.txt
+++ b/test/prim/prim/vjp/CMakeLists.txt
@@ -8,5 +8,7 @@ foreach(TEST_OP ${TEST_OPS})
   py_test_modules(${TEST_OP} MODULES ${TEST_OP} ENVS ${GC_ENVS})
 endforeach()
 
+set_tests_properties(test_comp_high_grad PROPERTIES TIMEOUT 50)
+
 add_subdirectory(eager)
 add_subdirectory(static)
diff --git a/test/prim/prim/vjp/test_comp_high_grad.py b/test/prim/prim/vjp/test_comp_high_grad.py
new file mode 100644
index 0000000000000..77fa38684d906
--- /dev/null
+++ b/test/prim/prim/vjp/test_comp_high_grad.py
@@ -0,0 +1,334 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+import unittest
+
+sys.path.append("../../../../python/paddle/fluid/tests/unittests")
+
+import gradient_checker
+import numpy as np
+import parameterized as param
+from decorator_helper import prog_scope
+
+import paddle
+from paddle import fluid
+from paddle.fluid import core
+
+
+@param.parameterized_class(
+    ('shape1', 'shape2'),
+    [
+        (
+            [2, 3, 4],
+            [2, 3, 4],
+        ),
+        (
+            [2, 3, 3, 4],
+            [3, 1, 4],
+        ),
+        (
+            [2, 3, 3, 4],
+            [3, 1, 1],
+        ),
+        (
+            [2, 3, 3, 4],
+            [2, 3, 1, 4],
+        ),
+        (
+            [2, 3, 3, 4],
+            [2, 3, 1, 1],
+        ),
+    ],
+)
+class TestAddHighGradCheck(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.shape1 = cls.shape1
+        cls.shape2 = cls.shape2
+
+    def add_wrapper(self, x):
+        return paddle.add(x[0], x[1])
+
+    @prog_scope()
+    def func_double(self, place):
+        shape1 = self.shape1
+        shape2 = self.shape2
+        eps = 0.0005
+        dtype = np.float64
+        x = paddle.static.data('x', shape1, dtype=dtype)
+        y = paddle.static.data('y', shape2, dtype=dtype)
+        x.persistable = True
+        y.persistable = True
+        out = paddle.add(x, y)
+        x_arr = np.random.uniform(-1, 1, shape1).astype(dtype)
+        y_arr = np.random.uniform(-2, 2, shape2).astype(dtype)
+        x_arr[np.abs(x_arr) < 0.005] = 0.002
+        y_arr[np.abs(y_arr) < 0.005] = 0.002
+        from paddle.fluid import core
+
+        core._set_prim_backward_enabled(True)
+        gradient_checker.double_grad_check(
+            [x, y], y=out, x_init=[x_arr, y_arr], place=place, eps=eps
+        )
+        gradient_checker.double_grad_check_for_dygraph(
+            self.add_wrapper, [x, y], y=out, x_init=[x_arr, y_arr], place=place
+        )
+        core._set_prim_backward_enabled(False)
+
+    @prog_scope()
+    def func_triple(self, place):
+        shape1 = self.shape1
+        shape2 = self.shape2
+        eps = 0.0005
+        dtype = np.float64
+        x = paddle.static.data('x', shape1, dtype=dtype)
+        y = paddle.static.data('y', shape2, dtype=dtype)
+        x.persistable = True
+        y.persistable = True
+        out = paddle.add(x, y)
+        x_arr = np.random.uniform(-1, 1, shape1).astype(dtype)
+        y_arr = np.random.uniform(-1, 1, shape2).astype(dtype)
+        x_arr[np.abs(x_arr) < 0.005] = 0.002
+        y_arr[np.abs(y_arr) < 0.005] = 0.002
+        from paddle.fluid import core
+
+        core._set_prim_backward_enabled(True)
+        gradient_checker.triple_grad_check(
+            [x, y], y=out, x_init=[x_arr, y_arr], place=place, eps=eps
+        )
+        gradient_checker.triple_grad_check_for_dygraph(
+            self.add_wrapper, [x, y], y=out, x_init=[x_arr, y_arr], place=place
+        )
+        core._set_prim_backward_enabled(False)
+
+    def test_high_grad(self):
+        paddle.enable_static()
+        places = [fluid.CPUPlace()]
+        if core.is_compiled_with_cuda():
+            places.append(fluid.CUDAPlace(0))
+        for p in places:
+            self.func_double(p)
+            self.func_triple(p)
+
+
+@param.parameterized_class(
+    ('shape1', 'shape2'),
+    [
+        (
+            [2, 3, 4],
+            [2, 3, 4],
+        ),
+        (
+            [2, 3, 3, 4],
+            [3, 1, 4],
+        ),
+        (
+            [2, 3, 3, 4],
+            [3, 1, 1],
+        ),
+        (
+            [2, 3, 3, 4],
+            [2, 3, 1, 4],
+        ),
+        (
+            [2, 3, 3, 4],
+            [2, 3, 1, 1],
+        ),
+    ],
+)
+class TestSubtractHighGradCheck(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.shape1 = cls.shape1
+        cls.shape2 = cls.shape2
+
+    def subtract_wrapper(self, x):
+        return paddle.subtract(x[0], x[1])
+
+    @prog_scope()
+    def func_double(self, place):
+        shape1 = self.shape1
+        shape2 = self.shape2
+        eps = 0.0005
+        dtype = np.float64
+        x = paddle.static.data('x', shape1, dtype=dtype)
+        y = paddle.static.data('y', shape2, dtype=dtype)
+        x.persistable = True
+        y.persistable = True
+        out = paddle.subtract(x, y)
+        x_arr = np.random.uniform(-1, 1, shape1).astype(dtype)
+        y_arr = np.random.uniform(-2, 2, shape2).astype(dtype)
+        x_arr[np.abs(x_arr) < 0.005] = 0.002
+        y_arr[np.abs(y_arr) < 0.005] = 0.002
+        from paddle.fluid import core
+
+        core._set_prim_backward_enabled(True)
+        gradient_checker.double_grad_check(
+            [x, y], y=out, x_init=[x_arr, y_arr], place=place, eps=eps
+        )
+        gradient_checker.double_grad_check_for_dygraph(
+            self.subtract_wrapper,
+            [x, y],
+            y=out,
+            x_init=[x_arr, y_arr],
+            place=place,
+        )
+        core._set_prim_backward_enabled(False)
+
+    @prog_scope()
+    def func_triple(self, place):
+        shape1 = self.shape1
+        shape2 = self.shape2
+        eps = 0.0005
+        dtype = np.float64
+        x = paddle.static.data('x', shape1, dtype=dtype)
+        y = paddle.static.data('y', shape2, dtype=dtype)
+        x.persistable = True
+        y.persistable = True
+        out = paddle.subtract(x, y)
+        x_arr = np.random.uniform(-1, 1, shape1).astype(dtype)
+        y_arr = np.random.uniform(-2, 2, shape2).astype(dtype)
+        x_arr[np.abs(x_arr) < 0.005] = 0.002
+        y_arr[np.abs(y_arr) < 0.005] = 0.002
+        from paddle.fluid import core
+
+        core._set_prim_backward_enabled(True)
+        gradient_checker.triple_grad_check(
+            [x, y], y=out, x_init=[x_arr, y_arr], place=place, eps=eps
+        )
+        gradient_checker.triple_grad_check_for_dygraph(
+            self.subtract_wrapper,
+            [x, y],
+            y=out,
+            x_init=[x_arr, y_arr],
+            place=place,
+        )
+        core._set_prim_backward_enabled(False)
+
+    def test_high_grad(self):
+        paddle.enable_static()
+        places = [fluid.CPUPlace()]
+        if core.is_compiled_with_cuda():
+            places.append(fluid.CUDAPlace(0))
+        for p in places:
+            self.func_double(p)
+            self.func_triple(p)
+
+
+@param.parameterized_class(
+    ('shape1', 'shape2'),
+    [
+        (
+            [2, 3, 4],
+            [2, 3, 4],
+        ),
+        (
+            [2, 3, 3, 4],
+            [3, 1, 4],
+        ),
+        (
+            [2, 3, 3, 4],
+            [3, 1, 1],
+        ),
+        (
+            [2, 3, 3, 4],
+            [2, 3, 1, 4],
+        ),
+        (
+            [2, 3, 3, 4],
+            [2, 3, 1, 1],
+        ),
+    ],
+)
+class TestMultiplyHighGradCheck(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.shape1 = cls.shape1
+        cls.shape2 = cls.shape2
+
+    def multiply_wrapper(self, x):
+        return paddle.multiply(x[0], x[1])
+
+    @prog_scope()
+    def func_double(self, place):
+        shape1 = self.shape1
+        shape2 = self.shape2
+        eps = 0.0005
+        dtype = np.float64
+        x = paddle.static.data('x', shape1, dtype=dtype)
+        y = paddle.static.data('y', shape2, dtype=dtype)
+        x.persistable = True
+        y.persistable = True
+        out = paddle.multiply(x, y)
+        x_arr = np.random.uniform(-1, 1, shape1).astype(dtype)
+        y_arr = np.random.uniform(-2, 2, shape2).astype(dtype)
+        x_arr[np.abs(x_arr) < 0.005] = 0.002
+        y_arr[np.abs(y_arr) < 0.005] = 0.002
+        from paddle.fluid import core
+
+        core._set_prim_backward_enabled(True)
+        gradient_checker.double_grad_check(
+            [x, y], y=out, x_init=[x_arr, y_arr], place=place, eps=eps
+        )
+        gradient_checker.double_grad_check_for_dygraph(
+            self.multiply_wrapper,
+            [x, y],
+            y=out,
+            x_init=[x_arr, y_arr],
+            place=place,
+        )
+        core._set_prim_backward_enabled(False)
+
+    @prog_scope()
+    def func_triple(self, place):
+        shape1 = self.shape1
+        shape2 = self.shape2
+        eps = 0.0005
+        dtype = np.float64
+        x = paddle.static.data('x', shape1, dtype=dtype)
+        y = paddle.static.data('y', shape2, dtype=dtype)
+        x.persistable = True
+        y.persistable = True
+        out = paddle.multiply(x, y)
+        x_arr = np.random.uniform(-1, 1, shape1).astype(dtype)
+        y_arr = np.random.uniform(-1, 1, shape2).astype(dtype)
+        x_arr[np.abs(x_arr) < 0.005] = 0.002
+        y_arr[np.abs(y_arr) < 0.005] = 0.002
+        from paddle.fluid import core
+
+        core._set_prim_backward_enabled(True)
+        gradient_checker.triple_grad_check(
+            [x, y], y=out, x_init=[x_arr, y_arr], place=place, eps=eps
+        )
+        gradient_checker.triple_grad_check_for_dygraph(
+            self.multiply_wrapper,
+            [x, y],
+            y=out,
+            x_init=[x_arr, y_arr],
+            place=place,
+        )
+        core._set_prim_backward_enabled(False)
+
+    def test_high_grad(self):
+        paddle.enable_static()
+        places = [fluid.CPUPlace()]
+        if core.is_compiled_with_cuda():
+            places.append(fluid.CUDAPlace(0))
+        for p in places:
+            self.func_double(p)
+            self.func_triple(p)
+
+
+if __name__ == '__main__':
+    unittest.main()

From f1e3575e15c9ff20b71cf7545945ba82993fe4d3 Mon Sep 17 00:00:00 2001
From: sneaxiy <32832641+sneaxiy@users.noreply.github.com>
Date: Fri, 28 Apr 2023 10:49:17 +0800
Subject: [PATCH 125/405] fix c_softmax deterministic (#53419)

---
 .../c_softmax_with_cross_entropy_op.cu        | 21 +++++++------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu
index 6a2dab9005a63..114c52f608f34 100644
--- a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu
+++ b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.h"
+#include "paddle/phi/kernels/reduce_sum_kernel.h"
 
 #include "paddle/fluid/framework/eigen.h"
 #include "paddle/fluid/platform/collective_helper.h"
@@ -337,9 +338,7 @@ struct CSoftmaxWithCrossEntropyProcessGroupFunctor<phi::GPUContext, T> {
     eigen_logits_max.device(*dev_ctx.eigen_device()) =
         eigen_logits.maximum(along_axis);
 
-    std::vector<phi::DenseTensor> in_out;
-    in_out.push_back(logits_max);
-    pg->AllReduce(in_out, in_out, opts)->Synchronize();
+    pg->AllReduce(&logits_max, logits_max, opts, true, true);
 
     // step 2, obtain logit - logit_max
     Eigen::DSizes<int, 2> batch_by_one(N, 1);
@@ -390,10 +389,8 @@ struct CSoftmaxWithCrossEntropyProcessGroupFunctor<phi::GPUContext, T> {
           nranks);
     }
 
-    in_out.clear();
-    in_out.push_back(predicted_logits);
     opts.reduce_op = distributed::ReduceOp::SUM;
-    pg->AllReduce(in_out, in_out, opts)->Synchronize();
+    pg->AllReduce(&predicted_logits, predicted_logits, opts, true, true);
 
     // step 4, obtain exp(logit)
     eigen_softmax.device(*dev_ctx.eigen_device()) = eigen_softmax.exp();
@@ -403,15 +400,11 @@ struct CSoftmaxWithCrossEntropyProcessGroupFunctor<phi::GPUContext, T> {
     sum_exp_logits = ctx.AllocateTmpTensor<T, phi::GPUContext>({N, 1}, dev_ctx);
     void* sum_exp_logits_buff = sum_exp_logits.mutable_data<T>(place);
 
-    auto eigen_sum_exp_logits =
-        phi::funcs::EigenMatrix<T>::From(sum_exp_logits);
-    eigen_sum_exp_logits.device(*dev_ctx.eigen_device()) =
-        eigen_softmax.sum(along_axis);
+    phi::SumKernel<T, phi::GPUContext>(
+        dev_ctx, softmax_2d, {-1}, softmax_2d.dtype(), true, &sum_exp_logits);
 
-    in_out.clear();
-    in_out.push_back(sum_exp_logits);
     opts.reduce_op = distributed::ReduceOp::SUM;
-    pg->AllReduce(in_out, in_out, opts)->Synchronize();
+    pg->AllReduce(&sum_exp_logits, sum_exp_logits, opts, true, true);
 
     if (label_type == framework::proto::VarType::INT32) {
       CaculateLoss<T, int32_t>
@@ -431,6 +424,8 @@ struct CSoftmaxWithCrossEntropyProcessGroupFunctor<phi::GPUContext, T> {
                                                      N);
     }
 
+    auto eigen_sum_exp_logits =
+        phi::funcs::EigenMatrix<T>::From(sum_exp_logits);
     eigen_softmax.device(*dev_ctx.eigen_device()) =
         (eigen_softmax *
          eigen_sum_exp_logits.inverse().broadcast(one_by_class));

From 4001f7ae4b98a68048b87079a3109de69f6e5857 Mon Sep 17 00:00:00 2001
From: tianshuo78520a <707759223@qq.com>
Date: Fri, 28 Apr 2023 10:51:34 +0800
Subject: [PATCH 126/405] Change Py3 use Ubuntu20 Docker (#52523)

* test py3.8

* fix

* test gcc12

* test gcc12

* test gcc12

* test py3.8

* test py3

* fix

* fix

* add ubutnu20

* add ubutnu20

* add ubutnu20

* test py3

* fix error

* fix error

* fix error

* fix error

* fix error

* update py version

* fix

* test docker

* test docker

* test docker

* fix pip version,use pip3.9

* log error

* log add

* test ci

* add test log

* del

* fix

* test ci

* test ci

* update dockerfile

* update ci_dockerfile

* update ci_dockerfile

* update ci_dockerfile

* fix

* add git

* fix
---
 tools/dockerfile/build_scripts/install_gcc.sh |  2 --
 tools/dockerfile/ci_dockerfile.sh             | 32 ++++++++++---------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/tools/dockerfile/build_scripts/install_gcc.sh b/tools/dockerfile/build_scripts/install_gcc.sh
index 5b12a63c84508..f0889bb5a919f 100644
--- a/tools/dockerfile/build_scripts/install_gcc.sh
+++ b/tools/dockerfile/build_scripts/install_gcc.sh
@@ -68,7 +68,6 @@ elif [ "$1" == "gcc122" ]; then
   make -j8 && make install
   cd .. && rm -rf temp_gcc122 gcc-12.2.0 gcc-12.2.0.tar.gz
   cp ${lib_so_6} ${lib_so_6}.bak  && rm -f ${lib_so_6} &&
-  ln -s /usr/local/gcc-12.2/lib64/libgfortran.so.5 ${lib_so_5} && \
   ln -s /usr/local/gcc-12.2/lib64/libstdc++.so.6 ${lib_so_6} && \
   cp /usr/local/gcc-12.2/lib64/libstdc++.so.6.0.30 ${lib_path}
 elif [ "$1" == "gcc121" ]; then
@@ -82,7 +81,6 @@ elif [ "$1" == "gcc121" ]; then
   make -j8 && make install
   cd .. && rm -rf temp_gcc122 gcc-12.1.0 gcc-12.1.0.tar.gz
   cp ${lib_so_6} ${lib_so_6}.bak  && rm -f ${lib_so_6} &&
-  ln -s /usr/local/gcc-12.1/lib64/libgfortran.so.5 ${lib_so_5} && \
   ln -s /usr/local/gcc-12.1/lib64/libstdc++.so.6 ${lib_so_6} && \
   cp /usr/local/gcc-12.1/lib64/libstdc++.so.6.0.30 ${lib_path}
 fi
diff --git a/tools/dockerfile/ci_dockerfile.sh b/tools/dockerfile/ci_dockerfile.sh
index 8378ed26d2360..83651c781c760 100644
--- a/tools/dockerfile/ci_dockerfile.sh
+++ b/tools/dockerfile/ci_dockerfile.sh
@@ -78,22 +78,24 @@ function make_ubuntu_trt7_dockerfile(){
 
 function make_centos_dockerfile(){
   dockerfile_name="Dockerfile.cuda9_cudnn7_gcc48_py35_centos6"
-  sed "s/<baseimg>/12.0.1-cudnn8-devel-centos7/g" Dockerfile.centos >${dockerfile_name}
-  sed -i "s#COPY build_scripts /build_scripts#COPY tools/dockerfile/build_scripts  ./build_scripts#g" ${dockerfile_name}
+  sed "s#<baseimg>#ubuntu:20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name}
+  sed -i "s#<setcuda>##g" ${dockerfile_name}
+  sed -i "s#WITH_GPU:-ON#WITH_GPU:-OFF#g" ${dockerfile_name}
+  sed -i "s#RUN apt-key del 7fa2af80##g" ${dockerfile_name}
+  sed -i 's#RUN rm /etc/apt/sources.list.d/\*##g' ${dockerfile_name}
+  sed -i "s#RUN apt-key adv --fetch-keys https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub##g" ${dockerfile_name}
   dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
-  sed -i "${dockerfile_line}i RUN yum install -y pigz graphviz zstd libsndfile ninja-build" ${dockerfile_name}
-  sed -i "${dockerfile_line}i RUN pip3.8 install distro" ${dockerfile_name}
-  sed -i "${dockerfile_line}i ENV LD_LIBRARY_PATH /opt/_internal/cpython-3.8.0/lib:/usr/local/ssl/lib:/opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64 " ${dockerfile_name}
-  sed -i "${dockerfile_line}i ENV PATH /opt/_internal/cpython-3.8.0/bin:/usr/local/ssl:/usr/local/gcc-12.2/bin:/usr/local/go/bin:/root/gopath/bin:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/java/jdk1.8.0_192/bin " ${dockerfile_name}
-  sed -i "${dockerfile_line}i RUN rm -f /usr/bin/cc && ln -s /usr/local/gcc-12.2/bin/gcc /usr/bin/cc" ${dockerfile_name}
-  sed -i "${dockerfile_line}i RUN rm -f /usr/bin/g++ && ln -s /usr/local/gcc-12.2/bin/g++ /usr/bin/g++" ${dockerfile_name}
-  sed -i "${dockerfile_line}i RUN rm -f /usr/bin/c++ && ln -s /usr/local/gcc-12.2/bin/c++ /usr/bin/c++" ${dockerfile_name}
-  sed -i "${dockerfile_line}i RUN rm -f /usr/bin/gcc && ln -s /usr/local/gcc-12.2/bin/gcc /usr/bin/gcc" ${dockerfile_name}
-  sed -i "${dockerfile_line}i RUN ln -s /usr/lib64/libz.so /usr/local/lib/libz.so \\
-    RUN ln -s /usr/local/lib/libnccl.so /usr/local/cuda/lib64/" ${dockerfile_name}
-  sed -i $"${dockerfile_line}i RUN wget --no-check-certificate -q  https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \\
-    RUN tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
-  sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc122 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-12.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-12.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" ${dockerfile_name}
+  sed -i 's#RUN bash /build_scripts/install_trt.sh##g' ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
+     tar -xzf     hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y libsndfile1 zstd pigz libcurl4-openssl-dev gettext zstd ninja-build" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.8 install PyGithub distro \&\& pip3.9 install PyGithub wheel \&\& pip3.10 install PyGithub distro" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y libcurl4-openssl-dev gettext pigz zstd ninja-build \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
+    tar -xvf git-2.17.1.tar.gz \&\& \
+    cd git-2.17.1 \&\& \
+    ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \
+    make -j8 \&\& make install " ${dockerfile_name}
+  sed -i 's#<install_cpu_package>#RUN apt-get install -y gcc g++ make#g' ${dockerfile_name}
 }
 
 

From 8163faaa4013d7c52424556d79b5b97fd5f0121f Mon Sep 17 00:00:00 2001
From: superwinner1 <82640284+superwinner1@users.noreply.github.com>
Date: Fri, 28 Apr 2023 11:17:08 +0800
Subject: [PATCH 127/405] =?UTF-8?q?=E3=80=90Hackathon=20No.55=E3=80=91add?=
 =?UTF-8?q?=20fmin=20BF16=20test=20(#53100)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* 'fmin'

* 'fix'

* 'fix'
---
 .../phi/kernels/funcs/elementwise_functor.h   | 12 +++++++
 .../kernels/gpu/elementwise_grad_kernel.cu    |  1 +
 paddle/phi/kernels/kps/elementwise_kernel.cu  |  1 +
 .../fluid/tests/unittests/test_fmin_op.py     | 31 ++++++++++++++++++-
 4 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/paddle/phi/kernels/funcs/elementwise_functor.h b/paddle/phi/kernels/funcs/elementwise_functor.h
index af0ae3b6447be..a291c1542d3e2 100644
--- a/paddle/phi/kernels/funcs/elementwise_functor.h
+++ b/paddle/phi/kernels/funcs/elementwise_functor.h
@@ -14,6 +14,7 @@ limitations under the License. */
 
 #pragma once
 
+#include "paddle/phi/common/bfloat16.h"
 #include "paddle/phi/common/complex.h"
 #include "paddle/phi/common/float16.h"
 #include "paddle/phi/core/enforce.h"
@@ -190,6 +191,17 @@ struct FMinFunctor<dtype::float16> {
   }
 };
 
+template <>
+struct FMinFunctor<dtype::bfloat16> {
+  inline HOSTDEVICE dtype::bfloat16 operator()(const dtype::bfloat16 a,
+                                               const dtype::bfloat16 b) const {
+    float float_a = static_cast<float>(a);
+    float float_b = static_cast<float>(b);
+    auto result = std::fmin(float_a, float_b);
+    return static_cast<dtype::bfloat16>(result);
+  }
+};
+
 template <>
 struct FMinFunctor<int> {
   inline HOSTDEVICE int operator()(const int a, const int b) const {
diff --git a/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu b/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu
index 30e222663da10..b69434c82da8c 100644
--- a/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu
@@ -108,6 +108,7 @@ PD_REGISTER_KERNEL(fmin_grad,
                    double,
                    int,
                    phi::dtype::float16,
+                   phi::dtype::bfloat16,
                    int64_t) {}
 
 PD_REGISTER_KERNEL(maximum_grad,
diff --git a/paddle/phi/kernels/kps/elementwise_kernel.cu b/paddle/phi/kernels/kps/elementwise_kernel.cu
index cbf811a9830b0..550e579ac1159 100644
--- a/paddle/phi/kernels/kps/elementwise_kernel.cu
+++ b/paddle/phi/kernels/kps/elementwise_kernel.cu
@@ -166,6 +166,7 @@ PD_REGISTER_KERNEL(fmin,
                    double,
                    int,
                    float16,
+                   bfloat16,
                    int64_t) {}
 
 PD_REGISTER_KERNEL(heaviside,
diff --git a/python/paddle/fluid/tests/unittests/test_fmin_op.py b/python/paddle/fluid/tests/unittests/test_fmin_op.py
index 19e43fa3d1d6c..1956d5b4fc433 100644
--- a/python/paddle/fluid/tests/unittests/test_fmin_op.py
+++ b/python/paddle/fluid/tests/unittests/test_fmin_op.py
@@ -15,7 +15,7 @@
 import unittest
 
 import numpy as np
-from eager_op_test import OpTest
+from eager_op_test import OpTest, convert_float_to_uint16
 
 import paddle
 from paddle.fluid import core
@@ -243,6 +243,35 @@ def test_check_grad_normal(self):
         self.check_grad(['X', 'Y'], 'Out')
 
 
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA and not support the bfloat16",
+)
+class TestFminBF16OP(OpTest):
+    def setUp(self):
+        self.op_type = "elementwise_fmin"
+        self.python_api = paddle.fmin
+        self.dtype = np.uint16
+        x = np.random.uniform(1, 1, [13, 17]).astype("float32")
+        sgn = np.random.choice([-1, 1], [13, 17]).astype("float32")
+        y = x + sgn * np.random.uniform(1, 1, [13, 17]).astype("float32")
+        out = np.fmin(x, y)
+        self.inputs = {
+            'X': convert_float_to_uint16(x),
+            'Y': convert_float_to_uint16(y),
+        }
+        self.outputs = {'Out': convert_float_to_uint16(out)}
+
+    def test_check_output(self):
+        place = core.CUDAPlace(0)
+        self.check_output_with_place(place)
+
+    def test_check_grad(self):
+        place = core.CUDAPlace(0)
+        self.check_grad_with_place(place, ['X', 'Y'], 'Out')
+
+
 if __name__ == "__main__":
     paddle.enable_static()
     unittest.main()

From 98fc42775528aab63e25b5ebd5481317ed1d734b Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Fri, 28 Apr 2023 11:17:36 +0800
Subject: [PATCH 128/405] Add broadcast_tensors tests (#52961)

---
 .../gpu/broadcast_tensors_grad_kernel.cu      |   6 +-
 .../kernels/gpu/broadcast_tensors_kernel.cu   |   5 +-
 .../fluid/tests/unittests/eager_op_test.py    |  53 ++++++---
 .../unittests/test_broadcast_tensors_op.py    | 105 +++++++++++++++---
 python/paddle/tensor/manipulation.py          |  10 +-
 5 files changed, 142 insertions(+), 37 deletions(-)

diff --git a/paddle/phi/kernels/gpu/broadcast_tensors_grad_kernel.cu b/paddle/phi/kernels/gpu/broadcast_tensors_grad_kernel.cu
index 7acfd33e94a9a..a202c428394d7 100644
--- a/paddle/phi/kernels/gpu/broadcast_tensors_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/broadcast_tensors_grad_kernel.cu
@@ -16,7 +16,7 @@
 
 #include <vector>
 
-#include "paddle/phi/common/float16.h"
+#include "paddle/phi/common/amp_type_traits.h"
 #include "paddle/phi/core/dense_tensor.h"
 #include "paddle/phi/core/enforce.h"
 #include "paddle/phi/core/kernel_registry.h"
@@ -104,8 +104,10 @@ PD_REGISTER_KERNEL(broadcast_tensors_grad,
                    GPU,
                    ALL_LAYOUT,
                    phi::BroadcastTensorsGradKernel,
+                   bool,
                    int,
                    int64_t,
                    float,
                    double,
-                   phi::dtype::float16) {}
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/paddle/phi/kernels/gpu/broadcast_tensors_kernel.cu b/paddle/phi/kernels/gpu/broadcast_tensors_kernel.cu
index 82dac4552a42f..3d16797cb66c0 100644
--- a/paddle/phi/kernels/gpu/broadcast_tensors_kernel.cu
+++ b/paddle/phi/kernels/gpu/broadcast_tensors_kernel.cu
@@ -14,7 +14,7 @@
 
 #include "paddle/phi/kernels/broadcast_tensors_kernel.h"
 
-#include "paddle/phi/common/float16.h"
+#include "paddle/phi/common/amp_type_traits.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/impl/broadcast_tensors_kernel_impl.h"
 
@@ -27,4 +27,5 @@ PD_REGISTER_KERNEL(broadcast_tensors,
                    int64_t,
                    float,
                    double,
-                   phi::dtype::float16) {}
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/python/paddle/fluid/tests/unittests/eager_op_test.py b/python/paddle/fluid/tests/unittests/eager_op_test.py
index 0bcb34f956b23..1be451d65e83b 100644
--- a/python/paddle/fluid/tests/unittests/eager_op_test.py
+++ b/python/paddle/fluid/tests/unittests/eager_op_test.py
@@ -2689,6 +2689,26 @@ def fluid_dtype_to_np_dtype(self, dtype):
     def np_value_to_fluid_value(input):
         return input
 
+    def cast_bf16_output(self, block, cast_inputs):
+        output_names = []
+        for i in range(0, len(cast_inputs)):
+            cast_output = block.create_var(
+                dtype="float32", shape=cast_inputs[i].shape
+            )
+            cast_op = block.append_op(
+                inputs={"X": cast_inputs[i]},
+                outputs={"Out": cast_output},
+                type="cast",
+                attrs={
+                    "in_dtype": core.VarDesc.VarType.BF16,
+                    "out_dtype": core.VarDesc.VarType.FP32,
+                },
+            )
+            cast_op.desc.infer_var_type(block.desc)
+            cast_op.desc.infer_shape(block.desc)
+            output_names.append(cast_output.name)
+        return output_names
+
     def _get_gradient(
         self,
         input_to_check,
@@ -2712,21 +2732,24 @@ def _get_gradient(
             if user_defined_grad_outputs is None:
                 if self.dtype == np.uint16:
                     cast_inputs = list(map(block.var, output_names))
-                    cast_outputs = block.create_var(
-                        dtype="float32", shape=cast_inputs[0].shape
-                    )
-                    cast_op = block.append_op(
-                        inputs={"X": cast_inputs},
-                        outputs={"Out": cast_outputs},
-                        type="cast",
-                        attrs={
-                            "in_dtype": core.VarDesc.VarType.BF16,
-                            "out_dtype": core.VarDesc.VarType.FP32,
-                        },
-                    )
-                    cast_op.desc.infer_var_type(block.desc)
-                    cast_op.desc.infer_shape(block.desc)
-                    output_names = [cast_outputs.name]
+                    if self.op_type == "broadcast_tensors":
+                        output_names = self.cast_bf16_output(block, cast_inputs)
+                    else:
+                        cast_outputs = block.create_var(
+                            dtype="float32", shape=cast_inputs[0].shape
+                        )
+                        cast_op = block.append_op(
+                            inputs={"X": cast_inputs},
+                            outputs={"Out": cast_outputs},
+                            type="cast",
+                            attrs={
+                                "in_dtype": core.VarDesc.VarType.BF16,
+                                "out_dtype": core.VarDesc.VarType.FP32,
+                            },
+                        )
+                        cast_op.desc.infer_var_type(block.desc)
+                        cast_op.desc.infer_shape(block.desc)
+                        output_names = [cast_outputs.name]
                 loss = append_loss_ops(block, output_names)
                 param_grad_list = append_backward(
                     loss=loss,
diff --git a/python/paddle/fluid/tests/unittests/test_broadcast_tensors_op.py b/python/paddle/fluid/tests/unittests/test_broadcast_tensors_op.py
index d2af1933469aa..be37ff0157898 100644
--- a/python/paddle/fluid/tests/unittests/test_broadcast_tensors_op.py
+++ b/python/paddle/fluid/tests/unittests/test_broadcast_tensors_op.py
@@ -16,7 +16,7 @@
 import unittest
 
 import numpy as np
-from eager_op_test import OpTest
+from eager_op_test import OpTest, convert_float_to_uint16
 
 import paddle
 from paddle.fluid import core
@@ -43,7 +43,7 @@ def find_output_shape(input_list):
     return list(reversed(output_shape))
 
 
-def make_inputs_outputs(input_shapes, dtype):
+def make_inputs_outputs(input_shapes, dtype, is_bfloat16=False):
     """Automatically generate formatted inputs and outputs from input_shapes"""
     input_list = [
         np.random.random(shape).astype(dtype) for shape in input_shapes
@@ -53,6 +53,16 @@ def make_inputs_outputs(input_shapes, dtype):
         x + np.zeros(output_shape).astype(x.dtype) for x in input_list
     ]
 
+    if is_bfloat16:
+        input_list = [
+            convert_float_to_uint16(input_list[i])
+            for i in range(len(input_list))
+        ]
+        output_list = [
+            convert_float_to_uint16(output_list[i])
+            for i in range(len(output_list))
+        ]
+
     output_formatted = {
         "Out": [(f"out{i}", output_list[i]) for i in range(len(output_list))]
     }
@@ -63,24 +73,24 @@ def make_inputs_outputs(input_shapes, dtype):
     return input_formatted, output_formatted
 
 
-def gen_rank_diff_test(dtype):
+def gen_rank_diff_test(dtype, is_bfloat16=False):
     input_shapes = [(2, 60, 1), (6, 2, 1, 10)]
-    return make_inputs_outputs(input_shapes, dtype)
+    return make_inputs_outputs(input_shapes, dtype, is_bfloat16)
 
 
-def gen_no_broadcast_test(dtype):
+def gen_no_broadcast_test(dtype, is_bfloat16=False):
     input_shapes = [(12, 1, 10, 1), (12, 1, 10, 1)]
-    return make_inputs_outputs(input_shapes, dtype)
+    return make_inputs_outputs(input_shapes, dtype, is_bfloat16)
 
 
-def gen_mixed_tensors_test(dtype):
+def gen_mixed_tensors_test(dtype, is_bfloat16=False):
     input_shapes = [(2, 60, 1), (2, 2, 1, 30), (1, 2, 60, 1)]
-    return make_inputs_outputs(input_shapes, dtype)
+    return make_inputs_outputs(input_shapes, dtype, is_bfloat16)
 
 
-def gen_empty_tensors_test(dtype):
+def gen_empty_tensors_test(dtype, is_bfloat16=False):
     input_shapes = [(0), (0), (0)]
-    return make_inputs_outputs(input_shapes, dtype)
+    return make_inputs_outputs(input_shapes, dtype, is_bfloat16)
 
 
 class TestCPUBroadcastTensorsOp(OpTest):
@@ -125,7 +135,7 @@ def run_triple_in_test(self, test_func, args):
     def test_check_output(self):
         self.run_dual_test(
             self.check_output_with_place,
-            {"place": self.place, "atol": 1e-1},
+            {"place": self.place},
         )
 
     def test_check_grad_normal(self):
@@ -135,7 +145,6 @@ def test_check_grad_normal(self):
                 "place": self.place,
                 "inputs_to_check": ['x0', 'x1'],
                 "output_names": ['out0', 'out1'],
-                "max_relative_error": 0.05,
             },
         )
         self.run_triple_in_test(
@@ -144,7 +153,6 @@ def test_check_grad_normal(self):
                 "place": self.place,
                 "inputs_to_check": ['x0', 'x1', 'x2'],
                 "output_names": ['out0', 'out1', "out2"],
-                "max_relative_error": 0.05,
             },
         )
 
@@ -152,14 +160,77 @@ def test_check_grad_normal(self):
 @unittest.skipIf(
     not core.is_compiled_with_cuda(), "core is not compiled with CUDA"
 )
-class TestCUDABroadcastTensorsOp(TestCPUBroadcastTensorsOp):
+class TestBroadcastTensorsFP16Op(TestCPUBroadcastTensorsOp):
     def set_place(self):
         self.place = core.CUDAPlace(0)
 
     def set_dtypes(self):
-        self.dtypes = ['float64']
-        if core.is_float16_supported(self.place):
-            self.dtypes.append('float16')
+        self.dtypes = ['float16']
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support bfloat16",
+)
+class TestBroadcastTensorsBF16Op(OpTest):
+    def setUp(self):
+        self.op_type = "broadcast_tensors"
+        self.dtype = np.uint16
+        self.np_dtype = "float32"
+        self.use_mkldnn = False
+        self.attrs = {'use_mkldnn': self.use_mkldnn}
+        self.test_gen_func_list = [
+            gen_rank_diff_test,
+            gen_no_broadcast_test,
+            gen_mixed_tensors_test,
+        ]
+        self.python_api = paddle.broadcast_tensors
+        self.place = core.CUDAPlace(0)
+
+    def run_dual_test(self, test_func, args):
+        for gen_func in self.test_gen_func_list:
+            self.inputs, self.outputs = gen_func(self.np_dtype, True)
+            if len(self.outputs["Out"]) < 3:
+                self.python_out_sig = [
+                    f"out{i}" for i in range(len(self.outputs["Out"]))
+                ]
+                test_func(**args)
+
+    def run_triple_in_test(self, test_func, args):
+        self.inputs, self.outputs = self.test_gen_func_list[2](
+            self.np_dtype, True
+        )
+        self.python_out_sig = [
+            f"out{i}" for i in range(len(self.outputs["Out"]))
+        ]
+        test_func(**args)
+
+    def test_check_output(self):
+        self.run_dual_test(
+            self.check_output_with_place,
+            {"place": self.place},
+        )
+
+    def test_check_grad_normal(self):
+        self.run_dual_test(
+            self.check_grad_with_place,
+            {
+                "place": self.place,
+                "inputs_to_check": ['x0', 'x1'],
+                "output_names": ['out0', 'out1'],
+                "check_dygraph": False,
+            },
+        )
+        self.run_triple_in_test(
+            self.check_grad_with_place,
+            {
+                "place": self.place,
+                "inputs_to_check": ['x0', 'x1', 'x2'],
+                "output_names": ['out0', 'out1', 'out2'],
+                "check_dygraph": False,
+            },
+        )
 
 
 class TestBroadcastTensorsAPI(unittest.TestCase):
diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py
index 27e2a4b812de5..5c4a029711b0f 100644
--- a/python/paddle/tensor/manipulation.py
+++ b/python/paddle/tensor/manipulation.py
@@ -1240,7 +1240,15 @@ def broadcast_tensors(input, name=None):
             check_variable_and_dtype(
                 x,
                 'input[' + str(id) + ']',
-                ['bool', 'float32', 'float64', 'int32', 'int64'],
+                [
+                    'bool',
+                    'float16',
+                    'float32',
+                    'float64',
+                    'int32',
+                    'int64',
+                    'uint16',
+                ],
                 'broadcast_tensors',
             )
             if x.dtype != input[0].dtype:

From 1c1b487c19b27d55d9ae166f7fe1dc1545a8a130 Mon Sep 17 00:00:00 2001
From: lj970926 <41257772+lj970926@users.noreply.github.com>
Date: Fri, 28 Apr 2023 11:23:16 +0800
Subject: [PATCH 129/405] [XPU][BUG] Add cumsum grad kernel to xpu2 op list
 (#53386)

* clang format

* add cumsum_grad op to xpu2_op_list
---
 paddle/phi/backends/xpu/xpu2_op_list.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc
index 303963450b197..07ad62e442d76 100644
--- a/paddle/phi/backends/xpu/xpu2_op_list.cc
+++ b/paddle/phi/backends/xpu/xpu2_op_list.cc
@@ -169,6 +169,10 @@ XPUOpMap& get_kl2_ops() {
                      phi::DataType::FLOAT16,
                      phi::DataType::INT32,
                      phi::DataType::INT64})},
+      {"cumsum_grad",
+       XPUKernelSet({phi::DataType::FLOAT32,
+                     phi::DataType::INT32,
+                     phi::DataType::INT64})},
       {"cumprod",
        XPUKernelSet({phi::DataType::FLOAT32,
                      phi::DataType::INT32,

From 207e0f33c6ca36718c7e06317087d64e4ce9a302 Mon Sep 17 00:00:00 2001
From: wangshengxiang <121413869+shengxiangwang@users.noreply.github.com>
Date: Fri, 28 Apr 2023 11:23:50 +0800
Subject: [PATCH 130/405] fix: fix deformable_conv_grad op test (#53415)

---
 cmake/external/xpu.cmake                | 2 +-
 test/xpu/test_deformable_conv_op_xpu.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake
index 65c02e1e520a8..6202d4edf0496 100644
--- a/cmake/external/xpu.cmake
+++ b/cmake/external/xpu.cmake
@@ -8,7 +8,7 @@ set(XPU_API_LIB_NAME "libxpuapi.so")
 set(XPU_RT_LIB_NAME "libxpurt.so")
 set(XPU_XFT_LIB_NAME "libxft.so")
 
-set(XPU_BASE_DATE "20230408")
+set(XPU_BASE_DATE "20230427")
 set(XPU_XCCL_BASE_VERSION "1.0.13")
 set(XPU_XFT_BASE_VERSION "latest")
 
diff --git a/test/xpu/test_deformable_conv_op_xpu.py b/test/xpu/test_deformable_conv_op_xpu.py
index 8577cb2497704..62d274fbc74f0 100644
--- a/test/xpu/test_deformable_conv_op_xpu.py
+++ b/test/xpu/test_deformable_conv_op_xpu.py
@@ -138,6 +138,8 @@ def __init__(self):
     class TestModulatedDeformableConvOp(XPUOpTest):
         def setUp(self):
             self.op_type = "deformable_conv"
+            # set to e-6 because of atomic add in XPU
+            self.epsilon_xpu2xpu = 0.000001
             self.dtype = self.in_type
             self.place = paddle.XPUPlace(0)
             self.init_group()

From 6d9bbee37966d1ef5cfb1d907542c9911e86ce18 Mon Sep 17 00:00:00 2001
From: Roc <30228238+sljlp@users.noreply.github.com>
Date: Fri, 28 Apr 2023 11:25:00 +0800
Subject: [PATCH 131/405] [KUNLUN] fix pp send /recv on xpu (#53427)

To make it synchronized at the first recv operator.
If warping all send and recv operators with group start and end, the received tensor will be not complete.
---
 .../pp_utils/p2p_communication.py             | 33 ++++++++++++++++---
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
index 9d3cb5bd17098..2c30dc8bedfe0 100644
--- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
+++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
@@ -24,6 +24,26 @@
 _use_cache = False
 _enable_partial_send_recv = True
 
+_xpu_comm_group_started = False
+
+
+def _xpu_comm_group_start():
+    if not paddle.is_compiled_with_xpu():
+        return
+    global _xpu_comm_group_started
+    assert not _xpu_comm_group_started
+    framework.core.ProcessGroupBKCL.group_start()
+    _xpu_comm_group_started = True
+
+
+def _xpu_comm_group_end():
+    if not paddle.is_compiled_with_xpu():
+        return
+    global _xpu_comm_group_started
+    if _xpu_comm_group_started:
+        framework.core.ProcessGroupBKCL.group_end()
+        _xpu_comm_group_started = False
+
 
 def initialize_p2p_groups(hcg, use_cache=True, enable_partial_send_recv=True):
     global _hcg, _use_cache, _enable_partial_send_recv
@@ -350,9 +370,8 @@ def _p2p_helper(
 
     # TODO(Yuang Liu): use batch_isend_irecv replace all these comm ops
     tasks = []
-    if paddle.is_compiled_with_xpu():
-        framework.core.ProcessGroupBKCL.group_start()
     # start to p2p communicate
+    _xpu_comm_group_start()
     if tensor_send_prev is not None:
         if isinstance(tensor_send_prev, tuple):
             for d in tensor_send_prev:
@@ -388,6 +407,7 @@ def _p2p_helper(
                     use_calc_stream=sync_recv,
                 )
                 if sync_recv:
+                    _xpu_comm_group_end()
                     allgather_partial(
                         d,
                         nranks=mp_degree,
@@ -406,7 +426,9 @@ def _p2p_helper(
                 group=_hcg.recv_prev_group,
                 use_calc_stream=sync_recv,
             )
+
             if sync_recv:
+                _xpu_comm_group_end()
                 allgather_partial(
                     tensor_recv_prev,
                     nranks=mp_degree,
@@ -451,7 +473,9 @@ def _p2p_helper(
                     group=_hcg.recv_next_group,
                     use_calc_stream=sync_recv,
                 )
+
                 if sync_recv:
+                    _xpu_comm_group_end()
                     allgather_partial(
                         d,
                         nranks=mp_degree,
@@ -472,6 +496,7 @@ def _p2p_helper(
                 use_calc_stream=sync_recv,
             )
             if sync_recv:
+                _xpu_comm_group_end()
                 allgather_partial(
                     tensor_recv_next,
                     nranks=mp_degree,
@@ -481,9 +506,7 @@ def _p2p_helper(
                 )
             else:
                 tasks.append(task)
-    if paddle.is_compiled_with_xpu():
-        framework.core.ProcessGroupBKCL.group_end()
-
+    _xpu_comm_group_end()
     if not sync_recv:
         if framework.in_dygraph_mode():
             # wait irecv tasks in eager dygraph mode with new comm library

From 64adfe7a16929c92536be5c0e0699a7bc8db053d Mon Sep 17 00:00:00 2001
From: Zhang Jun <ewalker@live.cn>
Date: Fri, 28 Apr 2023 12:26:17 +0800
Subject: [PATCH 132/405] [inference][trt]trt support 0 dims (#53383)

* trt support 0 dim

* trt support 0 dim

* update activation ut
---
 .../inference/tensorrt/convert/op_converter.h |  16 +-
 paddle/fluid/inference/tensorrt/op_teller.cc  |   5 +-
 .../operators/tensorrt/tensorrt_engine_op.h   |  36 +----
 .../inference/test_trt_convert_activation.py  |  20 ++-
 test/ir/inference/test_trt_convert_celu.py    | 142 ------------------
 .../inference/test_trt_convert_logsigmoid.py  | 139 -----------------
 test/ir/inference/test_trt_convert_silu.py    | 139 -----------------
 .../inference/test_trt_convert_tanhshrink.py  | 139 -----------------
 8 files changed, 34 insertions(+), 602 deletions(-)
 delete mode 100644 test/ir/inference/test_trt_convert_celu.py
 delete mode 100755 test/ir/inference/test_trt_convert_logsigmoid.py
 delete mode 100755 test/ir/inference/test_trt_convert_silu.py
 delete mode 100755 test/ir/inference/test_trt_convert_tanhshrink.py

diff --git a/paddle/fluid/inference/tensorrt/convert/op_converter.h b/paddle/fluid/inference/tensorrt/convert/op_converter.h
index 87ad887cef383..fcd6146a5f400 100644
--- a/paddle/fluid/inference/tensorrt/convert/op_converter.h
+++ b/paddle/fluid/inference/tensorrt/convert/op_converter.h
@@ -306,13 +306,15 @@ class OpConverter {
         auto max_input_shape = engine->max_input_shape()[input];
         auto optim_input_shape = engine->optim_input_shape()[input];
         size_t ranks = min_input_shape.size();
-        if (ranks == 0) {
-          all_dynamic_shape_set = false;
-          LOG(INFO) << "trt input [" << input.c_str()
-                    << "] dynamic shape info not set, please check and retry.";
-          // check other input
-          continue;
-        }
+        // allow 0 dim for dynamic shape input
+        // if (ranks == 0) {
+        //   all_dynamic_shape_set = false;
+        //   LOG(INFO) << "trt input [" << input.c_str()
+        //             << "] dynamic shape info not set, please check and
+        //             retry.";
+        //   // check other input
+        //   continue;
+        // }
         std::vector<int64_t> input_shape;
         // input_shape.push_back(-1);
         for (size_t i = 0; i < ranks; i++) {
diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index 6a94e14d7e68b..6aff2fb3bf6ca 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -116,9 +116,10 @@ struct SimpleOpTypeSetTeller : public Teller {
       auto x_var_name = desc.Input("X")[0];
       auto* x_var_desc = block->FindVar(x_var_name);
       const auto x_shape = x_var_desc->GetShape();
-      if (x_shape.size() == 1) {
+      if (!with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
         VLOG(3) << op_type
-                << " op does not support input's dim is 1 in tensorrt.";
+                << " op does not support input's dim is 1 or 0 in tensorrt "
+                   "static shape mode.";
         return false;
       }
 #if !IS_TRT_VERSION_GE(7000)
diff --git a/paddle/fluid/operators/tensorrt/tensorrt_engine_op.h b/paddle/fluid/operators/tensorrt/tensorrt_engine_op.h
index 2f6dee8b52d14..d942bb0bf5d20 100644
--- a/paddle/fluid/operators/tensorrt/tensorrt_engine_op.h
+++ b/paddle/fluid/operators/tensorrt/tensorrt_engine_op.h
@@ -39,6 +39,7 @@
 #include "paddle/fluid/inference/tensorrt/engine.h"
 #include "paddle/fluid/inference/tensorrt/helper.h"
 #include "paddle/fluid/inference/utils/io_utils.h"
+#include "paddle/utils/string/string_helper.h"
 
 namespace paddle {
 namespace inference {
@@ -64,19 +65,10 @@ using inference::tensorrt::TRTInt8Calibrator;
 
 static void RuntimeStaticShapeCheck(std::vector<int64_t> runtime_input_shape,
                                     std::vector<int64_t> model_input_shape) {
-  auto comma_fold = [](std::string a, int b) {
-    return std::move(a) + ", " + std::to_string(b);
-  };
   std::string model_input_shape_str =
-      std::accumulate(std::next(model_input_shape.begin()),
-                      model_input_shape.end(),
-                      std::to_string(model_input_shape[0]),
-                      comma_fold);
+      string::join_strings(model_input_shape, ',');
   std::string runtime_input_shape_str =
-      std::accumulate(std::next(runtime_input_shape.begin()),
-                      runtime_input_shape.end(),
-                      std::to_string(runtime_input_shape[0]),
-                      comma_fold);
+      string::join_strings(runtime_input_shape, ',');
   PADDLE_ENFORCE_EQ(
       model_input_shape == runtime_input_shape,
       true,
@@ -137,24 +129,10 @@ static void RuntimeDynamicShapeCheck(
     }
     return true;
   };
-  auto comma_fold = [](std::string a, int b) {
-    return std::move(a) + ", " + std::to_string(b);
-  };
   std::string runtime_input_shape_str =
-      std::accumulate(std::next(runtime_input_shape.begin()),
-                      runtime_input_shape.end(),
-                      std::to_string(runtime_input_shape[0]),
-                      comma_fold);
-  std::string min_input_shape_str =
-      std::accumulate(std::next(min_input_shape.begin()),
-                      min_input_shape.end(),
-                      std::to_string(min_input_shape[0]),
-                      comma_fold);
-  std::string max_input_shape_str =
-      std::accumulate(std::next(max_input_shape.begin()),
-                      max_input_shape.end(),
-                      std::to_string(max_input_shape[0]),
-                      comma_fold);
+      string::join_strings(runtime_input_shape, ',');
+  std::string min_input_shape_str = string::join_strings(min_input_shape, ',');
+  std::string max_input_shape_str = string::join_strings(max_input_shape, ',');
   PADDLE_ENFORCE_EQ(is_input_shape_valid(
                         runtime_input_shape, min_input_shape, max_input_shape),
                     true,
@@ -551,7 +529,6 @@ class TensorRTEngineOp : public framework::OperatorBase {
     platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance();
     auto &dev_ctx = *pool.Get(dev_place);
     auto stream = reinterpret_cast<const phi::GPUContext &>(dev_ctx).stream();
-
     std::vector<std::string> output_maps =
         Attr<std::vector<std::string>>("output_name_mapping");
 
@@ -566,7 +543,6 @@ class TensorRTEngineOp : public framework::OperatorBase {
       trt_context = engine->context();
       binding_offset = engine->GetBindingsOffset();
     }
-
     // Bind input tensor to TRT.
     for (const auto &x : runtime_input_names_) {
 #if IS_TRT_VERSION_LT(8000)
diff --git a/test/ir/inference/test_trt_convert_activation.py b/test/ir/inference/test_trt_convert_activation.py
index 85583c37f6443..bba6beae14223 100644
--- a/test/ir/inference/test_trt_convert_activation.py
+++ b/test/ir/inference/test_trt_convert_activation.py
@@ -33,7 +33,9 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool:
 
     def sample_program_configs(self):
         def generate_input1(dims, batch, attrs: List[Dict[str, Any]]):
-            if dims == 1:
+            if dims == 0:
+                return np.random.random([]).astype(np.float32)
+            elif dims == 1:
                 return np.random.random([32]).astype(np.float32)
             elif dims == 2:
                 return np.random.random([3, 32]).astype(np.float32)
@@ -42,7 +44,7 @@ def generate_input1(dims, batch, attrs: List[Dict[str, Any]]):
             else:
                 return np.random.random([batch, 3, 32, 32]).astype(np.float32)
 
-        for dims in [1, 2, 3, 4]:
+        for dims in [0, 1, 2, 3, 4]:
             for batch in [1, 4]:
                 for op_type in [
                     "relu",
@@ -51,9 +53,13 @@ def generate_input1(dims, batch, attrs: List[Dict[str, Any]]):
                     "relu6",
                     "elu",
                     "selu",
+                    "silu",
                     "softsign",
                     "stanh",
                     "thresholded_relu",
+                    "celu",
+                    "logsigmoid",
+                    "tanh_shrink",
                     "softplus",
                 ]:
                     # few samples to reduce time
@@ -63,6 +69,8 @@ def generate_input1(dims, batch, attrs: List[Dict[str, Any]]):
                         for alpha in [0.67]:
                             self.dims = dims
                             dics = [{}]
+                            if op_type == "celu":
+                                dics = [{"alpha": 1.0}]
                             if op_type == "elu":
                                 dics = [{"alpha": alpha}]
                             if op_type == "selu":
@@ -103,7 +111,11 @@ def sample_predictor_configs(
         self, program_config
     ) -> (paddle_infer.Config, List[int], float):
         def generate_dynamic_shape(attrs):
-            if self.dims == 1:
+            if self.dims == 0:
+                self.dynamic_shape.min_input_shape = {"input_data": []}
+                self.dynamic_shape.max_input_shape = {"input_data": []}
+                self.dynamic_shape.opt_input_shape = {"input_data": []}
+            elif self.dims == 1:
                 self.dynamic_shape.min_input_shape = {"input_data": [1]}
                 self.dynamic_shape.max_input_shape = {"input_data": [64]}
                 self.dynamic_shape.opt_input_shape = {"input_data": [32]}
@@ -132,7 +144,7 @@ def clear_dynamic_shape():
             self.dynamic_shape.opt_input_shape = {}
 
         def generate_trt_nodes_num(attrs, dynamic_shape):
-            if self.dims == 1:
+            if not dynamic_shape and (self.dims == 1 or self.dims == 0):
                 return 0, 3
             return 1, 2
 
diff --git a/test/ir/inference/test_trt_convert_celu.py b/test/ir/inference/test_trt_convert_celu.py
deleted file mode 100644
index e8bba5bf0edae..0000000000000
--- a/test/ir/inference/test_trt_convert_celu.py
+++ /dev/null
@@ -1,142 +0,0 @@
-# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-from functools import partial
-from typing import Any, Dict, List
-
-import numpy as np
-from program_config import ProgramConfig, TensorConfig
-from trt_layer_auto_scan_test import TrtLayerAutoScanTest
-
-import paddle.inference as paddle_infer
-
-
-class TrtConvertCeluTest(TrtLayerAutoScanTest):
-    def is_program_valid(self, program_config: ProgramConfig) -> bool:
-        return True
-
-    def sample_program_configs(self):
-        def generate_input1(dims, attrs: List[Dict[str, Any]]):
-            if dims == 1:
-                return np.ones([3]).astype(np.float32)
-            elif dims == 2:
-                return np.ones([3, 64]).astype(np.float32)
-            elif dims == 3:
-                return np.ones([3, 64, 64]).astype(np.float32)
-            else:
-                return np.ones([1, 3, 64, 64]).astype(np.float32)
-
-        for dims in [1, 2, 3, 4]:
-            for alpha in [1.0, 2.0, 3.0]:
-                self.dims = dims
-
-                dics = [{"alpha": alpha}]
-
-                ops_config = [
-                    {
-                        "op_type": "celu",
-                        "op_inputs": {
-                            "X": ["input_data"],
-                        },
-                        "op_outputs": {"Out": ["output_data"]},
-                        "op_attrs": dics[0],
-                    }
-                ]
-                ops = self.generate_op_config(ops_config)
-
-                program_config = ProgramConfig(
-                    ops=ops,
-                    weights={},
-                    inputs={
-                        "input_data": TensorConfig(
-                            data_gen=partial(generate_input1, dims, dics)
-                        )
-                    },
-                    outputs=["output_data"],
-                )
-
-                yield program_config
-
-    def sample_predictor_configs(
-        self, program_config
-    ) -> (paddle_infer.Config, List[int], float):
-        def generate_dynamic_shape(attrs):
-            if self.dims == 1:
-                self.dynamic_shape.min_input_shape = {"input_data": [1]}
-                self.dynamic_shape.max_input_shape = {"input_data": [128]}
-                self.dynamic_shape.opt_input_shape = {"input_data": [64]}
-            elif self.dims == 2:
-                self.dynamic_shape.min_input_shape = {"input_data": [1, 32]}
-                self.dynamic_shape.max_input_shape = {"input_data": [4, 64]}
-                self.dynamic_shape.opt_input_shape = {"input_data": [3, 64]}
-            elif self.dims == 3:
-                self.dynamic_shape.min_input_shape = {"input_data": [1, 32, 32]}
-                self.dynamic_shape.max_input_shape = {
-                    "input_data": [10, 64, 64]
-                }
-                self.dynamic_shape.opt_input_shape = {"input_data": [3, 64, 64]}
-            else:
-                self.dynamic_shape.min_input_shape = {
-                    "input_data": [1, 3, 32, 32]
-                }
-                self.dynamic_shape.max_input_shape = {
-                    "input_data": [4, 3, 64, 64]
-                }
-                self.dynamic_shape.opt_input_shape = {
-                    "input_data": [1, 3, 64, 64]
-                }
-
-        def clear_dynamic_shape():
-            self.dynamic_shape.min_input_shape = {}
-            self.dynamic_shape.max_input_shape = {}
-            self.dynamic_shape.opt_input_shape = {}
-
-        def generate_trt_nodes_num(attrs, dynamic_shape):
-            if self.dims == 1:
-                return 0, 3
-            return 1, 2
-
-        attrs = [
-            program_config.ops[i].attrs for i in range(len(program_config.ops))
-        ]
-
-        # for static_shape
-        clear_dynamic_shape()
-        self.trt_param.precision = paddle_infer.PrecisionType.Float32
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, False
-        ), 1e-5
-        self.trt_param.precision = paddle_infer.PrecisionType.Half
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, False
-        ), (1e-3, 1e-3)
-
-        # for dynamic_shape
-        generate_dynamic_shape(attrs)
-        self.trt_param.precision = paddle_infer.PrecisionType.Float32
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, True
-        ), 1e-5
-        self.trt_param.precision = paddle_infer.PrecisionType.Half
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, True
-        ), (1e-3, 1e-3)
-
-    def test(self):
-        self.run_test()
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/test/ir/inference/test_trt_convert_logsigmoid.py b/test/ir/inference/test_trt_convert_logsigmoid.py
deleted file mode 100755
index b28a02a94f72b..0000000000000
--- a/test/ir/inference/test_trt_convert_logsigmoid.py
+++ /dev/null
@@ -1,139 +0,0 @@
-# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-from functools import partial
-from typing import Any, Dict, List
-
-import numpy as np
-from program_config import ProgramConfig, TensorConfig
-from trt_layer_auto_scan_test import TrtLayerAutoScanTest
-
-import paddle.inference as paddle_infer
-
-
-class TrtConvertLogSigmoidTest(TrtLayerAutoScanTest):
-    def is_program_valid(self, program_config: ProgramConfig) -> bool:
-        return True
-
-    def sample_program_configs(self):
-        def generate_input1(dims, attrs: List[Dict[str, Any]]):
-            if dims == 1:
-                return np.ones([3]).astype(np.float32)
-            elif dims == 2:
-                return np.ones([3, 64]).astype(np.float32)
-            elif dims == 3:
-                return np.ones([3, 64, 64]).astype(np.float32)
-            else:
-                return np.ones([1, 3, 64, 64]).astype(np.float32)
-
-        for dims in [1, 2, 3, 4]:
-            self.dims = dims
-
-            ops_config = [
-                {
-                    "op_type": "logsigmoid",
-                    "op_inputs": {
-                        "X": ["input_data"],
-                    },
-                    "op_outputs": {"Out": ["output_data"]},
-                    "op_attrs": {},
-                }
-            ]
-            ops = self.generate_op_config(ops_config)
-
-            program_config = ProgramConfig(
-                ops=ops,
-                weights={},
-                inputs={
-                    "input_data": TensorConfig(
-                        data_gen=partial(generate_input1, dims, {})
-                    )
-                },
-                outputs=["output_data"],
-            )
-
-            yield program_config
-
-    def sample_predictor_configs(
-        self, program_config
-    ) -> (paddle_infer.Config, List[int], float):
-        def generate_dynamic_shape(attrs):
-            if self.dims == 1:
-                self.dynamic_shape.min_input_shape = {"input_data": [1]}
-                self.dynamic_shape.max_input_shape = {"input_data": [128]}
-                self.dynamic_shape.opt_input_shape = {"input_data": [64]}
-            elif self.dims == 2:
-                self.dynamic_shape.min_input_shape = {"input_data": [1, 32]}
-                self.dynamic_shape.max_input_shape = {"input_data": [4, 64]}
-                self.dynamic_shape.opt_input_shape = {"input_data": [3, 64]}
-            elif self.dims == 3:
-                self.dynamic_shape.min_input_shape = {"input_data": [1, 32, 32]}
-                self.dynamic_shape.max_input_shape = {
-                    "input_data": [10, 64, 64]
-                }
-                self.dynamic_shape.opt_input_shape = {"input_data": [3, 64, 64]}
-            else:
-                self.dynamic_shape.min_input_shape = {
-                    "input_data": [1, 3, 32, 32]
-                }
-                self.dynamic_shape.max_input_shape = {
-                    "input_data": [4, 3, 64, 64]
-                }
-                self.dynamic_shape.opt_input_shape = {
-                    "input_data": [1, 3, 64, 64]
-                }
-
-        def clear_dynamic_shape():
-            self.dynamic_shape.min_input_shape = {}
-            self.dynamic_shape.max_input_shape = {}
-            self.dynamic_shape.opt_input_shape = {}
-
-        def generate_trt_nodes_num(attrs, dynamic_shape):
-            if self.dims == 1:
-                return 0, 3
-            return 1, 2
-
-        attrs = [
-            program_config.ops[i].attrs for i in range(len(program_config.ops))
-        ]
-
-        # for static_shape
-        clear_dynamic_shape()
-        self.trt_param.precision = paddle_infer.PrecisionType.Float32
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, False
-        ), 1e-5
-        self.trt_param.precision = paddle_infer.PrecisionType.Half
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, False
-        ), (1e-3, 1e-3)
-
-        # for dynamic_shape
-        generate_dynamic_shape(attrs)
-        self.trt_param.precision = paddle_infer.PrecisionType.Float32
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, True
-        ), 1e-5
-        self.trt_param.precision = paddle_infer.PrecisionType.Half
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, True
-        ), (1e-3, 1e-3)
-
-    def test(self):
-        self.run_test()
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/test/ir/inference/test_trt_convert_silu.py b/test/ir/inference/test_trt_convert_silu.py
deleted file mode 100755
index 052d0bfccf7ec..0000000000000
--- a/test/ir/inference/test_trt_convert_silu.py
+++ /dev/null
@@ -1,139 +0,0 @@
-# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-from functools import partial
-from typing import Any, Dict, List
-
-import numpy as np
-from program_config import ProgramConfig, TensorConfig
-from trt_layer_auto_scan_test import TrtLayerAutoScanTest
-
-import paddle.inference as paddle_infer
-
-
-class TrtConvertSiluTest(TrtLayerAutoScanTest):
-    def is_program_valid(self, program_config: ProgramConfig) -> bool:
-        return True
-
-    def sample_program_configs(self):
-        def generate_input1(dims, attrs: List[Dict[str, Any]]):
-            if dims == 1:
-                return np.ones([3]).astype(np.float32)
-            elif dims == 2:
-                return np.ones([3, 64]).astype(np.float32)
-            elif dims == 3:
-                return np.ones([3, 64, 64]).astype(np.float32)
-            else:
-                return np.ones([1, 3, 64, 64]).astype(np.float32)
-
-        for dims in [1, 2, 3, 4]:
-            self.dims = dims
-
-            ops_config = [
-                {
-                    "op_type": "silu",
-                    "op_inputs": {
-                        "X": ["input_data"],
-                    },
-                    "op_outputs": {"Out": ["output_data"]},
-                    "op_attrs": {},
-                }
-            ]
-            ops = self.generate_op_config(ops_config)
-
-            program_config = ProgramConfig(
-                ops=ops,
-                weights={},
-                inputs={
-                    "input_data": TensorConfig(
-                        data_gen=partial(generate_input1, dims, {})
-                    )
-                },
-                outputs=["output_data"],
-            )
-
-            yield program_config
-
-    def sample_predictor_configs(
-        self, program_config
-    ) -> (paddle_infer.Config, List[int], float):
-        def generate_dynamic_shape(attrs):
-            if self.dims == 1:
-                self.dynamic_shape.min_input_shape = {"input_data": [1]}
-                self.dynamic_shape.max_input_shape = {"input_data": [128]}
-                self.dynamic_shape.opt_input_shape = {"input_data": [64]}
-            elif self.dims == 2:
-                self.dynamic_shape.min_input_shape = {"input_data": [1, 32]}
-                self.dynamic_shape.max_input_shape = {"input_data": [4, 64]}
-                self.dynamic_shape.opt_input_shape = {"input_data": [3, 64]}
-            elif self.dims == 3:
-                self.dynamic_shape.min_input_shape = {"input_data": [1, 32, 32]}
-                self.dynamic_shape.max_input_shape = {
-                    "input_data": [10, 64, 64]
-                }
-                self.dynamic_shape.opt_input_shape = {"input_data": [3, 64, 64]}
-            else:
-                self.dynamic_shape.min_input_shape = {
-                    "input_data": [1, 3, 32, 32]
-                }
-                self.dynamic_shape.max_input_shape = {
-                    "input_data": [4, 3, 64, 64]
-                }
-                self.dynamic_shape.opt_input_shape = {
-                    "input_data": [1, 3, 64, 64]
-                }
-
-        def clear_dynamic_shape():
-            self.dynamic_shape.min_input_shape = {}
-            self.dynamic_shape.max_input_shape = {}
-            self.dynamic_shape.opt_input_shape = {}
-
-        def generate_trt_nodes_num(attrs, dynamic_shape):
-            if self.dims == 1:
-                return 0, 3
-            return 1, 2
-
-        attrs = [
-            program_config.ops[i].attrs for i in range(len(program_config.ops))
-        ]
-
-        # for static_shape
-        clear_dynamic_shape()
-        self.trt_param.precision = paddle_infer.PrecisionType.Float32
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, False
-        ), 1e-5
-        self.trt_param.precision = paddle_infer.PrecisionType.Half
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, False
-        ), (1e-3, 1e-3)
-
-        # for dynamic_shape
-        generate_dynamic_shape(attrs)
-        self.trt_param.precision = paddle_infer.PrecisionType.Float32
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, True
-        ), 1e-5
-        self.trt_param.precision = paddle_infer.PrecisionType.Half
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, True
-        ), (1e-3, 1e-3)
-
-    def test(self):
-        self.run_test()
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/test/ir/inference/test_trt_convert_tanhshrink.py b/test/ir/inference/test_trt_convert_tanhshrink.py
deleted file mode 100755
index 71bb4443794e9..0000000000000
--- a/test/ir/inference/test_trt_convert_tanhshrink.py
+++ /dev/null
@@ -1,139 +0,0 @@
-# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-from functools import partial
-from typing import Any, Dict, List
-
-import numpy as np
-from program_config import ProgramConfig, TensorConfig
-from trt_layer_auto_scan_test import TrtLayerAutoScanTest
-
-import paddle.inference as paddle_infer
-
-
-class TrtConvertTanhshrinkTest(TrtLayerAutoScanTest):
-    def is_program_valid(self, program_config: ProgramConfig) -> bool:
-        return True
-
-    def sample_program_configs(self):
-        def generate_input1(dims, attrs: List[Dict[str, Any]]):
-            if dims == 1:
-                return np.ones([3]).astype(np.float32)
-            elif dims == 2:
-                return np.ones([3, 64]).astype(np.float32)
-            elif dims == 3:
-                return np.ones([3, 64, 64]).astype(np.float32)
-            else:
-                return np.ones([1, 3, 64, 64]).astype(np.float32)
-
-        for dims in [1, 2, 3, 4]:
-            self.dims = dims
-
-            ops_config = [
-                {
-                    "op_type": "tanh_shrink",
-                    "op_inputs": {
-                        "X": ["input_data"],
-                    },
-                    "op_outputs": {"Out": ["output_data"]},
-                    "op_attrs": {},
-                }
-            ]
-            ops = self.generate_op_config(ops_config)
-
-            program_config = ProgramConfig(
-                ops=ops,
-                weights={},
-                inputs={
-                    "input_data": TensorConfig(
-                        data_gen=partial(generate_input1, dims, {})
-                    )
-                },
-                outputs=["output_data"],
-            )
-
-            yield program_config
-
-    def sample_predictor_configs(
-        self, program_config
-    ) -> (paddle_infer.Config, List[int], float):
-        def generate_dynamic_shape(attrs):
-            if self.dims == 1:
-                self.dynamic_shape.min_input_shape = {"input_data": [1]}
-                self.dynamic_shape.max_input_shape = {"input_data": [128]}
-                self.dynamic_shape.opt_input_shape = {"input_data": [64]}
-            elif self.dims == 2:
-                self.dynamic_shape.min_input_shape = {"input_data": [1, 32]}
-                self.dynamic_shape.max_input_shape = {"input_data": [4, 64]}
-                self.dynamic_shape.opt_input_shape = {"input_data": [3, 64]}
-            elif self.dims == 3:
-                self.dynamic_shape.min_input_shape = {"input_data": [1, 32, 32]}
-                self.dynamic_shape.max_input_shape = {
-                    "input_data": [10, 64, 64]
-                }
-                self.dynamic_shape.opt_input_shape = {"input_data": [3, 64, 64]}
-            else:
-                self.dynamic_shape.min_input_shape = {
-                    "input_data": [1, 3, 32, 32]
-                }
-                self.dynamic_shape.max_input_shape = {
-                    "input_data": [4, 3, 64, 64]
-                }
-                self.dynamic_shape.opt_input_shape = {
-                    "input_data": [1, 3, 64, 64]
-                }
-
-        def clear_dynamic_shape():
-            self.dynamic_shape.min_input_shape = {}
-            self.dynamic_shape.max_input_shape = {}
-            self.dynamic_shape.opt_input_shape = {}
-
-        def generate_trt_nodes_num(attrs, dynamic_shape):
-            if self.dims == 1:
-                return 0, 3
-            return 1, 2
-
-        attrs = [
-            program_config.ops[i].attrs for i in range(len(program_config.ops))
-        ]
-
-        # for static_shape
-        clear_dynamic_shape()
-        self.trt_param.precision = paddle_infer.PrecisionType.Float32
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, False
-        ), 1e-5
-        self.trt_param.precision = paddle_infer.PrecisionType.Half
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, False
-        ), (1e-3, 1e-3)
-
-        # for dynamic_shape
-        generate_dynamic_shape(attrs)
-        self.trt_param.precision = paddle_infer.PrecisionType.Float32
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, True
-        ), 1e-5
-        self.trt_param.precision = paddle_infer.PrecisionType.Half
-        yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, True
-        ), (1e-3, 1e-3)
-
-    def test(self):
-        self.run_test()
-
-
-if __name__ == "__main__":
-    unittest.main()

From 9c4065316d89f3f57a9b58437104a29d1837b84a Mon Sep 17 00:00:00 2001
From: iLeGend <824040212@qq.com>
Date: Fri, 28 Apr 2023 14:30:51 +0800
Subject: [PATCH 133/405] =?UTF-8?q?=E3=80=90Hackathon=20No.52=E3=80=91?=
 =?UTF-8?q?=E4=B8=BA=20Paddle=20dist=20=E7=AE=97=E5=AD=90=E5=AE=9E?=
 =?UTF-8?q?=E7=8E=B0=20float16=20=E6=95=B0=E6=8D=AE=E7=B1=BB=E5=9E=8B?=
 =?UTF-8?q?=E6=94=AF=E6=8C=81=20(#50915)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 paddle/phi/kernels/dist_grad_kernel.cc        |  9 +-
 paddle/phi/kernels/funcs/math_cuda_utils.h    | 27 +++---
 paddle/phi/kernels/gpu/dist_kernel.cu         | 84 +++++++++++--------
 .../fluid/tests/unittests/test_dist_op.py     | 40 +++++++++
 python/paddle/tensor/linalg.py                | 12 ++-
 5 files changed, 116 insertions(+), 56 deletions(-)

diff --git a/paddle/phi/kernels/dist_grad_kernel.cc b/paddle/phi/kernels/dist_grad_kernel.cc
index 17c24fa905b5c..3c7ffc01edf07 100644
--- a/paddle/phi/kernels/dist_grad_kernel.cc
+++ b/paddle/phi/kernels/dist_grad_kernel.cc
@@ -98,6 +98,11 @@ PD_REGISTER_KERNEL(
     dist_grad, CPU, ALL_LAYOUT, phi::DistGradKernel, float, double) {}
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-PD_REGISTER_KERNEL(
-    dist_grad, GPU, ALL_LAYOUT, phi::DistGradKernel, float, double) {}
+PD_REGISTER_KERNEL(dist_grad,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::DistGradKernel,
+                   phi::dtype::float16,
+                   float,
+                   double) {}
 #endif
diff --git a/paddle/phi/kernels/funcs/math_cuda_utils.h b/paddle/phi/kernels/funcs/math_cuda_utils.h
index b493e2ac41bfd..26ead61e4e5f9 100644
--- a/paddle/phi/kernels/funcs/math_cuda_utils.h
+++ b/paddle/phi/kernels/funcs/math_cuda_utils.h
@@ -23,6 +23,9 @@ limitations under the License. */
 
 #include <algorithm>
 
+#include "paddle/phi/backends/gpu/gpu_device_function.h"
+#include "paddle/phi/common/data_type.h"
+
 namespace phi {
 namespace funcs {
 
@@ -170,11 +173,7 @@ struct KeyValuePair<half> {
 template <typename T>
 __inline__ __device__ T WarpReduceSum(T val, unsigned lane_mask) {
   for (int mask = HALF_WARP; mask > 0; mask >>= 1)
-#if defined(PADDLE_WITH_CUDA) && (__CUDA_ARCH__ >= 350 && CUDA_VERSION >= 9000)
-    val += __shfl_xor_sync(lane_mask, val, mask, warpSize);
-#else
-    val += __shfl_xor(val, mask, warpSize);
-#endif
+    val += phi::backends::gpu::CudaShuffleXorSync(lane_mask, val, mask);
   return val;
 }
 
@@ -243,11 +242,8 @@ __inline__ __device__ T BlockReduceSumV2(T *val) {
 template <typename T>
 __inline__ __device__ T WarpReduceMax(T val, unsigned lane_mask) {
   for (int mask = HALF_WARP; mask > 0; mask >>= 1)
-#if defined(PADDLE_WITH_CUDA) && (__CUDA_ARCH__ >= 350 && CUDA_VERSION >= 9000)
-    val = max(val, __shfl_xor_sync(lane_mask, val, mask, warpSize));
-#else
-    val = max(val, __shfl_xor(val, mask, warpSize));
-#endif
+    val = std::max(
+        val, phi::backends::gpu::CudaShuffleXorSync(lane_mask, val, mask));
   return val;
 }
 
@@ -265,11 +261,8 @@ __inline__ __device__ T WarpReduceMaxV2(T *val) {
 template <typename T>
 __inline__ __device__ T WarpReduceMin(T val, unsigned lane_mask) {
   for (int mask = HALF_WARP; mask > 0; mask >>= 1)
-#if defined(PADDLE_WITH_CUDA) && (__CUDA_ARCH__ >= 350 && CUDA_VERSION >= 9000)
-    val = min(val, __shfl_xor_sync(lane_mask, val, mask, warpSize));
-#else
-    val = min(val, __shfl_xor(val, mask, warpSize));
-#endif
+    val = std::min(
+        val, phi::backends::gpu::CudaShuffleXorSync(lane_mask, val, mask));
   return val;
 }
 
@@ -310,7 +303,7 @@ __inline__ __device__ T BlockReduceMax(T val, unsigned mask) {
 
   // align block_span to warpSize
   int block_span = (blockDim.x + warpSize - 1) >> 5;
-  val = (lane < block_span) ? shared[lane] : -1e10f;
+  val = (lane < block_span) ? shared[lane] : std::numeric_limits<T>::min();
   val = WarpReduceMax(val, mask);
 
   return val;
@@ -358,7 +351,7 @@ __inline__ __device__ T BlockReduceMin(T val, unsigned mask) {
 
   // align block_span to warpSize
   int block_span = (blockDim.x + warpSize - 1) >> 5;
-  val = (lane < block_span) ? shared[lane] : 1e10f;
+  val = (lane < block_span) ? shared[lane] : std::numeric_limits<T>::max();
   val = WarpReduceMin(val, mask);
 
   return val;
diff --git a/paddle/phi/kernels/gpu/dist_kernel.cu b/paddle/phi/kernels/gpu/dist_kernel.cu
index 5040be8eaaca7..0a197053e1016 100644
--- a/paddle/phi/kernels/gpu/dist_kernel.cu
+++ b/paddle/phi/kernels/gpu/dist_kernel.cu
@@ -12,9 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/phi/kernels/dist_kernel.h"
+#include <algorithm>
+
 #include "paddle/phi/backends/gpu/gpu_launch_config.h"
+#include "paddle/phi/common/float16.h"
 #include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/dist_kernel.h"
 #include "paddle/phi/kernels/elementwise_subtract_kernel.h"
 #include "paddle/phi/kernels/funcs/math_cuda_utils.h"
 #include "paddle/phi/kernels/gpu/reduce.h"
@@ -24,47 +27,56 @@ namespace phi {
 
 #define FULL_MASK 0xffffffff
 
-template <typename T>
+template <typename Tx, typename Ty = Tx>
 struct ZeroOrderFunctor {
- public:
-  __device__ T operator()(const T& x, const T& y) const {
-    return static_cast<T>((x - y) != 0);
+  HOSTDEVICE explicit inline ZeroOrderFunctor() {}
+
+  HOSTDEVICE inline Ty operator()(const Tx& x, const Tx& y) const {
+    return static_cast<Ty>(x != y);
   }
 };
 
-template <typename T>
+template <typename Tx, typename Ty = Tx>
 struct OtherOrderFunctor {
-  explicit OtherOrderFunctor(const T& p_order) : p_order_(p_order) {}
-  __device__ T operator()(const T& x, const T& y) const {
-    return static_cast<T>(pow(abs(x - y), p_order_));
+  HOSTDEVICE explicit inline OtherOrderFunctor(const Ty& _p_order)
+      : p_order(_p_order) {}
+
+  HOSTDEVICE inline Ty operator()(const Tx& x, const Tx& y) const {
+    return static_cast<Ty>(
+        pow(abs(static_cast<Ty>(x) - static_cast<Ty>(y)), p_order));
   }
 
  private:
-  T p_order_;
+  Ty p_order;
 };
 
-template <typename T>
+template <typename Tx, typename Ty = Tx>
 struct PowFunctor {
-  explicit PowFunctor(const T& p_order) : p_order_(p_order) {}
-  HOSTDEVICE inline T operator()(const T x) const {
-    return static_cast<T>(pow(x, p_order_));
+  HOSTDEVICE explicit inline PowFunctor(const Ty& _p_order)
+      : p_order(_p_order) {}
+
+  HOSTDEVICE inline Tx operator()(const Tx x) const {
+    return static_cast<Tx>(pow(static_cast<Ty>(x), p_order));
   }
-  T p_order_;
+
+ private:
+  Ty p_order;
 };
 
 template <typename T, typename Functor>
 __global__ void ReduceSumWithSubtract(
     const T* x, const T* y, T* out, int64_t N, Functor func) {
-  T sum_val = 0;
+  using MT = typename phi::dtype::MPTypeTrait<T>::Type;
+  MT sum_val(0.0);
   for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N;
        i += blockDim.x * gridDim.x) {
-    sum_val += func(x[i], y[i]);
+    sum_val += static_cast<MT>(func(x[i], y[i]));
   }
 
   __syncthreads();
-  sum_val = phi::funcs::BlockReduceSum<T>(sum_val, FULL_MASK);
+  sum_val = phi::funcs::BlockReduceSum<MT>(sum_val, FULL_MASK);
   if (threadIdx.x == 0) {
-    out[blockIdx.x] = sum_val;
+    out[blockIdx.x] = static_cast<T>(sum_val);
   }
 }
 
@@ -73,10 +85,10 @@ __global__ void ReduceMaxWithSubtract(const T* x,
                                       const T* y,
                                       T* out,
                                       int64_t N) {
-  T max_val = -1e10f;
+  T max_val = std::numeric_limits<T>::min();
   for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N;
        i += blockDim.x * gridDim.x) {
-    max_val = max(max_val, abs(x[i] - y[i]));
+    max_val = std::max(max_val, abs(x[i] - y[i]));
   }
 
   __syncthreads();
@@ -91,10 +103,10 @@ __global__ void ReduceMinWithSubtract(const T* x,
                                       const T* y,
                                       T* out,
                                       int64_t N) {
-  T min_val = 1e10f;
+  T min_val = std::numeric_limits<T>::max();
   for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N;
        i += blockDim.x * gridDim.x) {
-    min_val = min(min_val, abs(x[i] - y[i]));
+    min_val = std::min(min_val, abs(x[i] - y[i]));
   }
 
   __syncthreads();
@@ -110,6 +122,7 @@ void DistKernel(const Context& dev_ctx,
                 const DenseTensor& y,
                 float p,
                 DenseTensor* out) {
+  using MT = typename phi::dtype::MPTypeTrait<T>::Type;
   DenseTensor intermediate;
   const T* x_ptr = x.data<T>();
   const T* y_ptr = y.data<T>();
@@ -131,9 +144,8 @@ void DistKernel(const Context& dev_ctx,
       ReduceSumWithSubtract<T>
           <<<config.block_per_grid.x, config.thread_per_block.x, 0, stream>>>(
               x_ptr, y_ptr, i_ptr, n, ZeroOrderFunctor<T>());
-      phi::funcs::ReduceKernel<T, T, kps::AddFunctor, kps::IdentityFunctor<T>>(
-          dev_ctx, intermediate, out, kps::IdentityFunctor<T>(), reduce_axis);
-
+      phi::funcs::ReduceKernel<T, T, kps::AddFunctor, kps::IdentityFunctor<MT>>(
+          dev_ctx, intermediate, out, kps::IdentityFunctor<MT>(), reduce_axis);
     } else if (p == INFINITY) {
       ReduceMaxWithSubtract<T>
           <<<config.block_per_grid.x, config.thread_per_block.x, 0, stream>>>(
@@ -150,19 +162,19 @@ void DistKernel(const Context& dev_ctx,
           dev_ctx, intermediate, out, kps::IdentityFunctor<T>(), reduce_axis);
 
     } else {
-      T p_order = static_cast<T>(p);
+      MT p_order = static_cast<MT>(p);
       ReduceSumWithSubtract<T>
           <<<config.block_per_grid.x, config.thread_per_block.x, 0, stream>>>(
-              x_ptr, y_ptr, i_ptr, n, OtherOrderFunctor<T>(p_order));
-      phi::funcs::ReduceKernel<T, T, kps::AddFunctor, kps::IdentityFunctor<T>>(
-          dev_ctx, intermediate, out, kps::IdentityFunctor<T>(), reduce_axis);
+              x_ptr, y_ptr, i_ptr, n, OtherOrderFunctor<T, MT>(p_order));
+      phi::funcs::ReduceKernel<T, T, kps::AddFunctor, kps::IdentityFunctor<MT>>(
+          dev_ctx, intermediate, out, kps::IdentityFunctor<MT>(), reduce_axis);
 
       const DenseTensor* tmp_norm = out;
       std::vector<const DenseTensor*> ins = {tmp_norm};
       std::vector<DenseTensor*> outs = {out};
-      T p_order_ = static_cast<T>(1. / p_order);
+      MT p_order_ = static_cast<MT>(static_cast<MT>(1.) / p_order);
       phi::funcs::ElementwiseKernel<T>(
-          dev_ctx, ins, &outs, PowFunctor<T>(p_order_));
+          dev_ctx, ins, &outs, PowFunctor<T, MT>(p_order_));
     }
 
   } else {
@@ -173,4 +185,10 @@ void DistKernel(const Context& dev_ctx,
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(dist, GPU, ALL_LAYOUT, phi::DistKernel, float, double) {}
+PD_REGISTER_KERNEL(dist,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::DistKernel,
+                   phi::dtype::float16,
+                   float,
+                   double) {}
diff --git a/python/paddle/fluid/tests/unittests/test_dist_op.py b/python/paddle/fluid/tests/unittests/test_dist_op.py
index 96c0de915cff2..3654da4e00fae 100644
--- a/python/paddle/fluid/tests/unittests/test_dist_op.py
+++ b/python/paddle/fluid/tests/unittests/test_dist_op.py
@@ -158,6 +158,46 @@ def init_case(self):
         self.p = 1.5
 
 
+class TestDistFP16Op(OpTest):
+    def init_data_type(self):
+        self.data_type = 'float16'
+
+
+class TestDistFP16OpCase1(TestDistFP16Op):
+    def init_case(self):
+        self.x_shape = (3, 5, 5, 6)
+        self.y_shape = (5, 5, 6)
+        self.p = 1.0
+
+
+class TestDistFP16OpCase2(TestDistFP16Op):
+    def init_case(self):
+        self.x_shape = (10, 10)
+        self.y_shape = (4, 10, 10)
+        self.p = 2.0
+
+
+class TestDistFP16OpCase3(TestDistFP16Op):
+    def init_case(self):
+        self.x_shape = (15, 10)
+        self.y_shape = (15, 10)
+        self.p = float("inf")
+
+
+class TestDistFP16OpCase4(TestDistFP16Op):
+    def init_case(self):
+        self.x_shape = (2, 3, 4, 5, 8)
+        self.y_shape = (3, 1, 5, 8)
+        self.p = float("-inf")
+
+
+class TestDistFP16OpCase5(TestDistFP16Op):
+    def init_case(self):
+        self.x_shape = (4, 1, 4, 8)
+        self.y_shape = (2, 2, 1, 4, 4, 8)
+        self.p = 1.5
+
+
 class TestDistAPI(unittest.TestCase):
     def init_data_type(self):
         self.data_type = (
diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py
index 2235cf93cfb60..57550c86368a2 100644
--- a/python/paddle/tensor/linalg.py
+++ b/python/paddle/tensor/linalg.py
@@ -675,8 +675,8 @@ def dist(x, y, p=2, name=None):
         ||z||_{p}=(\sum_{i=1}^{m}|z_i|^p)^{\\frac{1}{p}}
 
     Args:
-        x (Tensor): 1-D to 6-D Tensor, its data type is float32 or float64.
-        y (Tensor): 1-D to 6-D Tensor, its data type is float32 or float64.
+        x (Tensor): 1-D to 6-D Tensor, its data type is float16, float32 or float64.
+        y (Tensor): 1-D to 6-D Tensor, its data type is float16, float32 or float64.
         p (float, optional): The norm to be computed, its data type is float32 or float64. Default: 2.
         name (str, optional): The default value is `None`. Normally there is no need for
             user to set this property. For more information, please refer to :ref:`api_guide_Name`.
@@ -706,8 +706,12 @@ def dist(x, y, p=2, name=None):
     if in_dygraph_mode():
         return _C_ops.dist(x, y, p)
 
-    check_variable_and_dtype(x, 'dtype', ['float32', 'float64'], 'dist')
-    check_variable_and_dtype(y, 'dtype', ['float32', 'float64'], 'dist')
+    check_variable_and_dtype(
+        x, 'dtype', ['float16', 'float32', 'float64'], 'dist'
+    )
+    check_variable_and_dtype(
+        y, 'dtype', ['float16', 'float32', 'float64'], 'dist'
+    )
     check_type(p, 'p', (float, int), 'dist')
     helper = LayerHelper("dist", **locals())
     out = helper.create_variable_for_type_inference(x.dtype)

From 4ccbcce576241d836b98ce138be006b437c1d108 Mon Sep 17 00:00:00 2001
From: jjyaoao <jjyaoao@126.com>
Date: Fri, 28 Apr 2023 14:34:08 +0800
Subject: [PATCH 134/405] remove is_npu_pinned_place (#53391)

---
 paddle/fluid/eager/amp_auto_cast.h       | 1 -
 paddle/fluid/eager/amp_utils.h           | 1 -
 paddle/fluid/eager/eager_amp_auto_cast.h | 1 -
 paddle/fluid/imperative/amp_auto_cast.cc | 3 +--
 paddle/fluid/platform/place.cc           | 7 +------
 paddle/fluid/platform/place.h            | 1 -
 6 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/paddle/fluid/eager/amp_auto_cast.h b/paddle/fluid/eager/amp_auto_cast.h
index ab06e3d33e542..8200f41f568c5 100644
--- a/paddle/fluid/eager/amp_auto_cast.h
+++ b/paddle/fluid/eager/amp_auto_cast.h
@@ -27,7 +27,6 @@ static inline bool NeedCast(const paddle::Tensor& tensor,
   if (paddle::platform::is_gpu_place(place) ||
       paddle::platform::is_cuda_pinned_place(place) ||
       paddle::platform::is_xpu_place(place) ||
-      paddle::platform::is_npu_pinned_place(place) ||
       paddle::platform::is_custom_place(place)) {
     // CudaPinndePlace is added for varbase created by dataloader
     if ((data_type == phi::DataType::FLOAT32 ||
diff --git a/paddle/fluid/eager/amp_utils.h b/paddle/fluid/eager/amp_utils.h
index 2e06eaab8acae..4bfb2fdb4f64f 100644
--- a/paddle/fluid/eager/amp_utils.h
+++ b/paddle/fluid/eager/amp_utils.h
@@ -96,7 +96,6 @@ inline phi::DataType GetDtypeWithPlace(
       is_right_place = (paddle::platform::is_gpu_place(place) ||
                         paddle::platform::is_cuda_pinned_place(place) ||
                         paddle::platform::is_xpu_place(place) ||
-                        paddle::platform::is_npu_pinned_place(place) ||
                         paddle::platform::is_custom_place(place));
       if (is_right_place) {
         break;
diff --git a/paddle/fluid/eager/eager_amp_auto_cast.h b/paddle/fluid/eager/eager_amp_auto_cast.h
index b4311cddc7e3e..96143c0cd7a0c 100644
--- a/paddle/fluid/eager/eager_amp_auto_cast.h
+++ b/paddle/fluid/eager/eager_amp_auto_cast.h
@@ -27,7 +27,6 @@ static inline bool NeedCast(const paddle::Tensor& tensor,
   if (paddle::platform::is_gpu_place(place) ||
       paddle::platform::is_cuda_pinned_place(place) ||
       paddle::platform::is_xpu_place(place) ||
-      paddle::platform::is_npu_pinned_place(place) ||
       paddle::platform::is_custom_place(place) ||
       paddle::platform::is_cpu_place(place)) {
     // CudaPinndePlace is added for varbase created by dataloader
diff --git a/paddle/fluid/imperative/amp_auto_cast.cc b/paddle/fluid/imperative/amp_auto_cast.cc
index 335bd4b132533..16ad89d99ac15 100644
--- a/paddle/fluid/imperative/amp_auto_cast.cc
+++ b/paddle/fluid/imperative/amp_auto_cast.cc
@@ -243,8 +243,7 @@ inline bool NeedCast(const std::shared_ptr<VarType>& var) {
   if (paddle::platform::is_gpu_place(place) ||
       paddle::platform::is_cuda_pinned_place(place) ||
       paddle::platform::is_xpu_place(place) ||
-      paddle::platform::is_custom_place(place) ||
-      paddle::platform::is_npu_pinned_place(place)) {
+      paddle::platform::is_custom_place(place)) {
     // CudaPinndePlace is added for varbase created by dataloader
     if (data_type == paddle::framework::proto::VarType::FP32 ||
         data_type == paddle::framework::proto::VarType::FP16 ||
diff --git a/paddle/fluid/platform/place.cc b/paddle/fluid/platform/place.cc
index fdaa67e3dd17d..ba5a25aa21a73 100644
--- a/paddle/fluid/platform/place.cc
+++ b/paddle/fluid/platform/place.cc
@@ -45,10 +45,6 @@ bool is_cuda_pinned_place(const Place &p) {
   return p.GetType() == phi::AllocationType::GPUPINNED;
 }
 
-bool is_npu_pinned_place(const Place &p) {
-  return p.GetType() == phi::AllocationType::NPUPINNED;
-}
-
 bool is_custom_place(const Place &p) {
   return p.GetType() == phi::AllocationType::CUSTOM;
 }
@@ -64,8 +60,7 @@ bool places_are_same_class(const Place &p1, const Place &p2) {
 
 bool is_same_place(const Place &p1, const Place &p2) {
   if (places_are_same_class(p1, p2)) {
-    if (is_cpu_place(p1) || is_cuda_pinned_place(p1) ||
-        is_npu_pinned_place(p1)) {
+    if (is_cpu_place(p1) || is_cuda_pinned_place(p1)) {
       return true;
     } else if (is_xpu_place(p1)) {
       return p1 == p2;
diff --git a/paddle/fluid/platform/place.h b/paddle/fluid/platform/place.h
index f787a1b28783d..3ea1dcded2c62 100644
--- a/paddle/fluid/platform/place.h
+++ b/paddle/fluid/platform/place.h
@@ -50,7 +50,6 @@ bool is_xpu_place(const Place &);
 bool is_ipu_place(const Place &);
 bool is_cpu_place(const Place &);
 bool is_cuda_pinned_place(const Place &);
-bool is_npu_pinned_place(const Place &);
 bool is_custom_place(const Place &p);
 bool places_are_same_class(const Place &, const Place &);
 bool is_same_place(const Place &, const Place &);

From 8f5eae47cd13e5615cc5abafe95560370fb4f489 Mon Sep 17 00:00:00 2001
From: Zhan Rongrui <46243324+zrr1999@users.noreply.github.com>
Date: Fri, 28 Apr 2023 14:35:03 +0800
Subject: [PATCH 135/405] [Bug fixes] Fix bugs in some sparse test (#53428)

---
 python/paddle/fluid/tests/unittests/test_sparse_reshape_op.py   | 2 ++
 python/paddle/fluid/tests/unittests/test_sparse_transpose_op.py | 2 ++
 python/paddle/fluid/tests/unittests/test_sparse_unary_op.py     | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/python/paddle/fluid/tests/unittests/test_sparse_reshape_op.py b/python/paddle/fluid/tests/unittests/test_sparse_reshape_op.py
index 34b7a95299f3e..84315ede556ac 100644
--- a/python/paddle/fluid/tests/unittests/test_sparse_reshape_op.py
+++ b/python/paddle/fluid/tests/unittests/test_sparse_reshape_op.py
@@ -36,6 +36,8 @@ def check_result(self, x_shape, new_shape, format):
         paddle.sparse.reshape.
         """
         mask = np.random.randint(0, 2, x_shape)
+        while np.sum(mask) == 0:
+            mask = paddle.randint(0, 2, x_shape)
         np_x = np.random.randint(-100, 100, x_shape) * mask
 
         # check cpu kernel
diff --git a/python/paddle/fluid/tests/unittests/test_sparse_transpose_op.py b/python/paddle/fluid/tests/unittests/test_sparse_transpose_op.py
index 32772d3891383..74a9d7e3308b3 100644
--- a/python/paddle/fluid/tests/unittests/test_sparse_transpose_op.py
+++ b/python/paddle/fluid/tests/unittests/test_sparse_transpose_op.py
@@ -23,6 +23,8 @@ class TestTranspose(unittest.TestCase):
     # x: sparse, out: sparse
     def check_result(self, x_shape, dims, format):
         mask = paddle.randint(0, 2, x_shape).astype("float32")
+        while paddle.sum(mask) == 0:
+            mask = paddle.randint(0, 2, x_shape).astype("float32")
         # "+ 1" to make sure that all zero elements in "origin_x" is caused by multiplying by "mask",
         # or the backward checks may fail.
         origin_x = (paddle.rand(x_shape, dtype='float32') + 1) * mask
diff --git a/python/paddle/fluid/tests/unittests/test_sparse_unary_op.py b/python/paddle/fluid/tests/unittests/test_sparse_unary_op.py
index edb7393bf3056..908121ace0118 100644
--- a/python/paddle/fluid/tests/unittests/test_sparse_unary_op.py
+++ b/python/paddle/fluid/tests/unittests/test_sparse_unary_op.py
@@ -30,6 +30,8 @@ def to_sparse(self, x, format):
     def check_result(self, dense_func, sparse_func, format, *args):
         origin_x = paddle.rand([8, 16, 32], dtype='float32')
         mask = paddle.randint(0, 2, [8, 16, 32]).astype('float32')
+        while paddle.sum(mask) == 0:
+            mask = paddle.randint(0, 2, [8, 16, 32]).astype("float32")
 
         # --- check sparse coo with dense --- #
         dense_x = origin_x * mask

From bca1b0c6a947bcae1e289e4f0eaa6b36fcc63ec7 Mon Sep 17 00:00:00 2001
From: dasen <100328659+dasenCoding@users.noreply.github.com>
Date: Fri, 28 Apr 2023 15:24:20 +0800
Subject: [PATCH 136/405] =?UTF-8?q?=E3=80=90Hackathon=204th=20No.11?=
 =?UTF-8?q?=E3=80=91=20=E4=B8=BA=20paddle=20=E6=B7=BB=E5=8A=A0=20Geometric?=
 =?UTF-8?q?=20Distribution=20API=20(#51224)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 python/paddle/distribution/__init__.py        |   2 +
 python/paddle/distribution/geometric.py       | 343 +++++++++++++++++
 python/paddle/distribution/kl.py              |   6 +
 .../test_distribution_geometric.py            | 350 ++++++++++++++++++
 .../test_distribution_geometric_static.py     | 341 +++++++++++++++++
 5 files changed, 1042 insertions(+)
 create mode 100644 python/paddle/distribution/geometric.py
 create mode 100644 test/distribution/test_distribution_geometric.py
 create mode 100644 test/distribution/test_distribution_geometric_static.py

diff --git a/python/paddle/distribution/__init__.py b/python/paddle/distribution/__init__.py
index 418ef478aaf13..05e4b8e11256c 100644
--- a/python/paddle/distribution/__init__.py
+++ b/python/paddle/distribution/__init__.py
@@ -29,6 +29,7 @@
 from paddle.distribution.transformed_distribution import TransformedDistribution
 from paddle.distribution.uniform import Uniform
 from paddle.distribution.laplace import Laplace
+from paddle.distribution.geometric import Geometric
 
 __all__ = [  # noqa
     'Bernoulli',
@@ -47,6 +48,7 @@
     'Laplace',
     'LogNormal',
     'Gumbel',
+    'Geometric',
 ]
 
 __all__.extend(transform.__all__)
diff --git a/python/paddle/distribution/geometric.py b/python/paddle/distribution/geometric.py
new file mode 100644
index 0000000000000..599b78c1f9e42
--- /dev/null
+++ b/python/paddle/distribution/geometric.py
@@ -0,0 +1,343 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numbers
+
+import numpy as np
+
+import paddle
+from paddle.distribution import distribution, uniform
+from paddle.fluid import framework
+
+
+class Geometric(distribution.Distribution):
+    r"""
+    Geometric distribution parameterized by probs.
+
+    In probability theory and statistics, the geometric distribution is one of
+    discrete probability distributions, parameterized by one positive shape parameter, denoted by probs.
+    In n Bernoulli trials, it takes k trials to get the probability of success for the first time.
+    In detail, it is: the probability that the first k-1 times failed and the kth time succeeded.
+    The geometric distribution is a special case of the Pascal distribution when r=1.
+
+    The probability mass function (pmf) is
+
+    .. math::
+            Pr(Y=k)=(1-p)^kp
+
+    where k is number of trials performed and p is probability of success for each trial and k=0,1,2,3,4..., p belong to (0,1].
+
+    Args:
+        probs (Real|Tensor): Probability parameter.
+            The value of probs must be positive. When the parameter is a tensor, probs is probability of success for each trial.
+
+    Returns:
+        Geometric distribution for instantiation of probs.
+
+    Examples:
+
+        .. code-block:: python
+
+            import paddle
+            from paddle.distribution import Geometric
+
+            geom = Geometric(0.5)
+
+            geom.mean
+            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        [2.])
+
+            geom.variance
+            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        [2.])
+
+            geom.stddev
+            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        [1.41421354])
+    """
+
+    def __init__(self, probs):
+        if isinstance(probs, (numbers.Real, paddle.Tensor, framework.Variable)):
+            if isinstance(probs, numbers.Real):
+                probs = paddle.full(
+                    shape=(), fill_value=probs, dtype=paddle.float32
+                )
+
+            all_ones = paddle.full(
+                shape=probs.shape, fill_value=1, dtype=probs.dtype
+            )
+            all_zeros = paddle.full(
+                shape=probs.shape, fill_value=0, dtype=probs.dtype
+            )
+            all_false = paddle.full(
+                shape=probs.shape, fill_value=False, dtype=bool
+            )
+
+            lessthen_0 = probs <= all_zeros
+            morethen_1 = probs > all_ones
+
+        else:
+            raise TypeError(
+                f"Expected type of probs is Number.Real|Tensor|framework.Variable, but got {type(probs)}"
+            )
+
+        if paddle.equal_all(lessthen_0, all_false) and paddle.equal_all(
+            morethen_1, all_false
+        ):
+            batch_shape = tuple(probs.shape)
+        else:
+            raise ValueError(
+                "Expected parameter probs of distribution Geometric to satisfy the"
+                "constraint Interval(lower_bound=0.0, upper_bound=1.0)"
+            )
+
+        self.probs = probs
+        super().__init__(batch_shape)
+
+    @property
+    def mean(self):
+        """Mean of geometric distribution."""
+        return 1.0 / self.probs
+
+    @property
+    def variance(self):
+        """Variance of geometric distribution."""
+        return paddle.to_tensor(
+            (1.0 / self.probs - 1.0) / self.probs,
+            dtype=self.probs.dtype,
+        )
+
+    @property
+    def stddev(self):
+        """Standard deviation of Geometric distribution."""
+        return paddle.sqrt(self.variance)
+
+    def pmf(self, k):
+        r"""Probability mass funciotn evaluated at k.
+
+        .. math::
+
+            P(X=k) = (1-p)^{k-1} p, \quad k=1,2,3,\ldots
+
+        Args:
+            k (int): Value to be evaluated.
+
+        Returns:
+            Tensor: Probability.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Geometric
+
+                geom = Geometric(0.5)
+                geom.pmf(2)
+                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [0.25000000])
+        """
+        if isinstance(k, (numbers.Integral, framework.Variable)):
+            return paddle.pow((1.0 - self.probs), k - 1.0) * self.probs
+        else:
+            raise TypeError(
+                f"Expected type of k is number.Real|framework.Variable, but got {type(k)}"
+            )
+
+    def log_pmf(self, k):
+        r"""Log probability mass function evaluated at k.
+
+        .. math::
+            \log P(X = k) = \log(1-p)^k p
+
+        Args:
+            k (int): Value to be evaluated.
+
+        Returns:
+            Tensor: Log probability.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Geometric
+
+                geom = Geometric(0.5)
+                geom.log_pmf(2)
+                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [-1.38629436])
+        """
+        if isinstance(k, (numbers.Integral, framework.Variable)):
+            return paddle.log(self.pmf(k))
+        else:
+            raise TypeError(
+                f"Expected type of k is number.Real|framework.Variable, but got {type(k)}"
+            )
+
+    def sample(self, shape=()):
+        """Sample from Geometric distribution with sample shape.
+
+        Args:
+            shape (tuple(int)): Sample shape.
+
+        Returns:
+            Sampled data with shape `sample_shape` + `batch_shape` + `event_shape`.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Geometric
+
+                geom = Geometric(0.5)
+                geom.sample((2,2))
+                # Tensor(shape=[2, 2, 1], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [[[4.28128004],
+                #         [0.53546447]],
+                #         [[0.88012987],
+                #         [0.54070371]]])
+        """
+        with paddle.no_grad():
+            return self.rsample(shape)
+
+    def rsample(self, shape=()):
+        """Generate samples of the specified shape.
+
+        Args:
+            shape(tuple(int)): The shape of generated samples.
+
+        Returns:
+            Tensor: A sample tensor that fits the Geometric distribution.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Geometric
+                geom = Geometric(0.5)
+                geom.rsample((2,2))
+                # Tensor(shape=[2, 2, 1], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [[[2.90974379],
+                #         [1.28049409]],
+                #         [[4.60141420],
+                #         [2.98836184]]])
+
+        """
+        shape = distribution.Distribution._extend_shape(
+            self, sample_shape=shape
+        )
+        tiny = np.finfo(dtype='float32').tiny
+
+        sample_uniform = uniform.Uniform(low=float(tiny), high=float(1))
+
+        new_t = sample_uniform.sample(list(shape))
+        return paddle.log(new_t) / paddle.log1p(-(self.probs))
+
+    def entropy(self):
+        r"""Entropy of dirichlet distribution.
+
+        .. math::
+
+            H(X) = -\left[\frac{1}{p} \log p + \frac{1-p}{p^2} \log (1-p) \right]
+
+        Returns:
+            Tensor: Entropy.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Geometric
+
+                geom = Geometric(0.5)
+                geom.entropy()
+                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [1.38629436])
+        """
+        x = (1.0 - self.probs) * paddle.log(1.0 - self.probs)
+        y = self.probs * paddle.log(self.probs)
+
+        return -(x + y) / self.probs
+
+    def cdf(self, k):
+        r"""Cdf of geometric distribution.
+
+        .. math::
+
+            F(X \leq k) = 1 - (1-p)^k, \quad k=0,1,2,\ldots
+
+        Args:
+            k: The number of trials performed.
+
+        Returns:
+            Tensor: Entropy.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Geometric
+
+                geom = Geometric(0.5)
+                geom.cdf(4)
+                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [0.93750000])
+        """
+        if isinstance(k, (numbers.Integral, framework.Variable)):
+            return 1.0 - paddle.pow((1.0 - self.probs), k)
+        else:
+            raise TypeError(
+                f"Expected type of k is number.Real|framework.Variable, but got {type(k)}"
+            )
+
+    def kl_divergence(self, other):
+        r"""Calculate the KL divergence KL(self || other) with two Geometric instances.
+
+        .. math::
+
+            KL(P \| Q) = \frac{p}{q} \log \frac{p}{q} + \log (1-p) - \log (1-q)
+
+        Args:
+            other (Geometric): An instance of Geometric.
+
+        Returns:
+            Tensor: The kl-divergence between two geometric distributions.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Geometric
+
+                geom_p = Geometric(0.5)
+                geom_q = Geometric(0.1)
+                geom_p.kl_divergence(geom_q)
+                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [0.51082563])
+        """
+        if isinstance(other, Geometric):
+            p, q = self.probs, other.probs
+            return p * paddle.log(p / q) + (1.0 - p) * paddle.log(
+                (1.0 - p) / (1.0 - q)
+            )
+        else:
+            raise TypeError(
+                f"Exected type of other is geometric.Geometric, but got {type(other)}"
+            )
diff --git a/python/paddle/distribution/kl.py b/python/paddle/distribution/kl.py
index 3d630b5802b72..cf004300d9407 100644
--- a/python/paddle/distribution/kl.py
+++ b/python/paddle/distribution/kl.py
@@ -21,6 +21,7 @@
 from paddle.distribution.dirichlet import Dirichlet
 from paddle.distribution.distribution import Distribution
 from paddle.distribution.exponential_family import ExponentialFamily
+from paddle.distribution.geometric import Geometric
 from paddle.distribution.laplace import Laplace
 from paddle.distribution.lognormal import LogNormal
 from paddle.distribution.normal import Normal
@@ -200,6 +201,11 @@ def _kl_laplace_laplace(p, q):
     return p.kl_divergence(q)
 
 
+@register_kl(Geometric, Geometric)
+def _kl_geometric_geometric(p, q):
+    return p.kl_divergence(q)
+
+
 @register_kl(ExponentialFamily, ExponentialFamily)
 def _kl_expfamily_expfamily(p, q):
     """Compute kl-divergence using `Bregman divergences <https://www.lix.polytechnique.fr/~nielsen/EntropyEF-ICIP2010.pdf>`_"""
diff --git a/test/distribution/test_distribution_geometric.py b/test/distribution/test_distribution_geometric.py
new file mode 100644
index 0000000000000..bc93b04a1e480
--- /dev/null
+++ b/test/distribution/test_distribution_geometric.py
@@ -0,0 +1,350 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import numbers
+import unittest
+
+import numpy as np
+import scipy.stats
+from config import ATOL, DEVICES, RTOL
+from parameterize import TEST_CASE_NAME, parameterize_cls, place, xrand
+
+import paddle
+from paddle.distribution import geometric, kl
+from paddle.nn.functional import log_softmax
+
+np.random.seed(2023)
+
+
+@place(DEVICES)
+@parameterize_cls(
+    (TEST_CASE_NAME, 'probs'),
+    [
+        (
+            'one-dim',
+            xrand(
+                (2,),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+        ),
+        (
+            'multi-dim',
+            xrand(
+                (2, 3),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+        ),
+    ],
+)
+class TestGeometric(unittest.TestCase):
+    def setUp(self):
+        probs = self.probs
+        if not isinstance(self.probs, numbers.Real):
+            probs = paddle.to_tensor(self.probs, dtype=paddle.float32)
+
+        self._paddle_geom = geometric.Geometric(probs)
+
+    def test_mean(self):
+        with paddle.fluid.dygraph.guard(self.place):
+            np.testing.assert_allclose(
+                self._paddle_geom.mean,
+                scipy.stats.geom.mean(self.probs),
+                rtol=RTOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+                atol=ATOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+            )
+
+    def test_variance(self):
+        with paddle.fluid.dygraph.guard(self.place):
+            np.testing.assert_allclose(
+                self._paddle_geom.variance,
+                scipy.stats.geom.var(self.probs),
+                rtol=RTOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+                atol=ATOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+            )
+
+    def test_stddev(self):
+        with paddle.fluid.dygraph.guard(self.place):
+            np.testing.assert_allclose(
+                self._paddle_geom.stddev,
+                scipy.stats.geom.std(self.probs),
+                rtol=RTOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+                atol=ATOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+            )
+
+    def test_entropy(self):
+        with paddle.fluid.dygraph.guard(self.place):
+            np.testing.assert_allclose(
+                self._paddle_geom.entropy(),
+                scipy.stats.geom.entropy(self.probs),
+                rtol=RTOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+                atol=ATOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+            )
+
+    def test_init_prob_value_error(self):
+        with self.assertRaises(ValueError):
+            paddle.distribution.geometric.Geometric(2)
+
+    def test_init_prob_type_error(self):
+        with self.assertRaises(TypeError):
+            paddle.distribution.geometric.Geometric([2])
+
+    def test_sample_shape(self):
+        cases = [
+            {
+                'input': (),
+                'expect': ()
+                + tuple(paddle.squeeze(self._paddle_geom.probs).shape),
+            },
+            {
+                'input': (4, 2),
+                'expect': (4, 2)
+                + tuple(paddle.squeeze(self._paddle_geom.probs).shape),
+            },
+        ]
+        for case in cases:
+            self.assertTrue(
+                tuple(self._paddle_geom.sample(case.get('input')).shape)
+                == case.get('expect')
+            )
+
+    def test_sample(self):
+        sample_shape = (80000,)
+        samples = self._paddle_geom.sample(sample_shape)
+        sample_values = samples.numpy()
+        self.assertEqual(sample_values.dtype, self.probs.dtype)
+
+        np.testing.assert_allclose(
+            sample_values.mean(axis=0),
+            scipy.stats.geom.mean(self.probs),
+            rtol=0.7,
+            atol=ATOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+        )
+        np.testing.assert_allclose(
+            sample_values.var(axis=0),
+            scipy.stats.geom.var(self.probs),
+            rtol=0.7,
+            atol=ATOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+        )
+
+    def test_rsample_shape(self):
+        cases = [
+            {
+                'input': (),
+                'expect': ()
+                + tuple(paddle.squeeze(self._paddle_geom.probs).shape),
+            },
+            {
+                'input': (2, 5),
+                'expect': (2, 5)
+                + tuple(paddle.squeeze(self._paddle_geom.probs).shape),
+            },
+        ]
+        for case in cases:
+            self.assertTrue(
+                tuple(self._paddle_geom.rsample(case.get('input')).shape)
+                == case.get('expect')
+            )
+
+    def test_rsample(self):
+        sample_shape = (100000,)
+        samples = self._paddle_geom.rsample(sample_shape)
+        sample_values = samples.numpy()
+        self.assertEqual(sample_values.dtype, self.probs.dtype)
+
+        np.testing.assert_allclose(
+            sample_values.mean(axis=0),
+            scipy.stats.geom.mean(self.probs),
+            rtol=0.7,
+            atol=ATOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+        )
+        np.testing.assert_allclose(
+            sample_values.var(axis=0),
+            scipy.stats.geom.var(self.probs),
+            rtol=0.7,
+            atol=ATOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+        )
+
+    def test_back_rsample(self):
+        sample_shape = (100000,)
+
+        with paddle.fluid.dygraph.guard(self.place):
+            self._paddle_geom.probs.stop_gradient = False
+
+            rs_value = self._paddle_geom.rsample(sample_shape)
+            softmax_rs = log_softmax(rs_value)
+            grads = paddle.grad([softmax_rs], [self._paddle_geom.probs])
+
+            self.assertEqual(len(grads), 1)
+            self.assertEqual(grads[0].dtype, self._paddle_geom.probs.dtype)
+            self.assertEqual(grads[0].shape, self._paddle_geom.probs.shape)
+
+
+@place(DEVICES)
+@parameterize_cls(
+    (TEST_CASE_NAME, 'probs', 'value'),
+    [
+        (
+            'one-dim',
+            xrand(
+                (2,),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+            5,
+        ),
+        (
+            'mult-dim',
+            xrand(
+                (2, 2),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+            5,
+        ),
+        (
+            'mult-dim',
+            xrand(
+                (2, 2, 2),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+            5,
+        ),
+    ],
+)
+class TestGeometricPMF(unittest.TestCase):
+    def setUp(self):
+        self._paddle_geom = geometric.Geometric(
+            probs=paddle.to_tensor(self.probs)
+        )
+
+    def test_pmf(self):
+        with paddle.fluid.dygraph.guard(self.place):
+            np.testing.assert_allclose(
+                self._paddle_geom.pmf(self.value),
+                scipy.stats.geom.pmf(self.value, self.probs),
+                rtol=RTOL.get(str(self.probs.dtype)),
+                atol=ATOL.get(str(self.probs.dtype)),
+            )
+
+    def test_log_pmf(self):
+        with paddle.fluid.dygraph.guard(self.place):
+            np.testing.assert_allclose(
+                self._paddle_geom.log_pmf(self.value),
+                scipy.stats.geom.logpmf(self.value, self.probs),
+                rtol=RTOL.get(str(self.probs.dtype)),
+                atol=ATOL.get(str(self.probs.dtype)),
+            )
+
+    def test_cdf(self):
+        with paddle.fluid.dygraph.guard(self.place):
+            np.testing.assert_allclose(
+                self._paddle_geom.cdf(self.value),
+                scipy.stats.geom.cdf(self.value, self.probs),
+                rtol=RTOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+                atol=ATOL.get(str(self._paddle_geom.probs.numpy().dtype)),
+            )
+
+    def test_pmf_error(self):
+        self.assertRaises(TypeError, self._paddle_geom.pmf, [1, 2])
+
+    def test_log_pmf_error(self):
+        self.assertRaises(TypeError, self._paddle_geom.log_pmf, [1, 2])
+
+    def test_cdf_error(self):
+        self.assertRaises(TypeError, self._paddle_geom.cdf, [1, 2])
+
+
+@place(DEVICES)
+@parameterize_cls(
+    (TEST_CASE_NAME, 'probs1', 'probs2'),
+    [
+        (
+            'one-dim',
+            xrand(
+                (2,),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+            xrand(
+                (2,),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+        ),
+        (
+            'multi-dim',
+            xrand(
+                (2, 2),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+            xrand(
+                (2, 2),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+        ),
+    ],
+)
+class TestGeometricKL(unittest.TestCase):
+    def setUp(self):
+        paddle.disable_static()
+        self._geometric1 = geometric.Geometric(
+            probs=paddle.to_tensor(self.probs1)
+        )
+        self._geometric2 = geometric.Geometric(
+            probs=paddle.to_tensor(self.probs2)
+        )
+
+    def test_kl_divergence(self):
+        np.testing.assert_allclose(
+            kl.kl_divergence(self._geometric1, self._geometric2),
+            self._kl(),
+            rtol=RTOL.get(str(self._geometric1.probs.numpy().dtype)),
+            atol=ATOL.get(str(self._geometric1.probs.numpy().dtype)),
+        )
+
+    def test_kl1_error(self):
+        self.assertRaises(
+            TypeError,
+            self._geometric1.kl_divergence,
+            paddle.distribution.beta.Beta,
+        )
+
+    def test_kl2_error(self):
+        self.assertRaises(
+            TypeError,
+            self._geometric2.kl_divergence,
+            paddle.distribution.beta.Beta,
+        )
+
+    def _kl(self):
+        return self.probs1 * np.log(self.probs1 / self.probs2) + (
+            1.0 - self.probs1
+        ) * np.log((1.0 - self.probs1) / (1.0 - self.probs2))
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/distribution/test_distribution_geometric_static.py b/test/distribution/test_distribution_geometric_static.py
new file mode 100644
index 0000000000000..b50af23251907
--- /dev/null
+++ b/test/distribution/test_distribution_geometric_static.py
@@ -0,0 +1,341 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import unittest
+
+import numpy as np
+import scipy.stats
+from config import ATOL, DEVICES, RTOL
+from parameterize import TEST_CASE_NAME, parameterize_cls, place, xrand
+
+import paddle
+from paddle.distribution import geometric
+
+np.random.seed(2023)
+
+paddle.enable_static()
+
+
+@place(DEVICES)
+@parameterize_cls(
+    (TEST_CASE_NAME, 'probs'),
+    [
+        (
+            'one-dim',
+            xrand(
+                (2,),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+        ),
+        (
+            'multi-dim',
+            xrand(
+                (2, 3),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+        ),
+    ],
+)
+class TestGeometric(unittest.TestCase):
+    def setUp(self):
+        self.program = paddle.static.Program()
+        self.executor = paddle.static.Executor(self.place)
+        with paddle.static.program_guard(self.program):
+            # scale no need convert to tensor for scale input unittest
+            probs = paddle.static.data(
+                'probs', self.probs.shape, self.probs.dtype
+            )
+
+            self._paddle_geometric = geometric.Geometric(probs)
+            self.feeds = {'probs': self.probs}
+
+    def test_mean(self):
+        with paddle.static.program_guard(self.program):
+            [mean] = self.executor.run(
+                self.program,
+                feed=self.feeds,
+                fetch_list=[self._paddle_geometric.mean],
+            )
+            np.testing.assert_allclose(
+                mean,
+                scipy.stats.geom.mean(self.probs),
+                rtol=RTOL.get(str(self.probs.dtype)),
+                atol=ATOL.get(str(self.probs.dtype)),
+            )
+
+    def test_variance(self):
+        with paddle.static.program_guard(self.program):
+            [variance] = self.executor.run(
+                self.program,
+                feed=self.feeds,
+                fetch_list=[self._paddle_geometric.variance],
+            )
+            np.testing.assert_allclose(
+                variance,
+                scipy.stats.geom.var(self.probs),
+                rtol=RTOL.get(str(self.probs.dtype)),
+                atol=ATOL.get(str(self.probs.dtype)),
+            )
+
+    def test_stddev(self):
+        with paddle.static.program_guard(self.program):
+            [stddev] = self.executor.run(
+                self.program,
+                feed=self.feeds,
+                fetch_list=[self._paddle_geometric.stddev],
+            )
+            np.testing.assert_allclose(
+                stddev,
+                scipy.stats.geom.std(self.probs),
+                rtol=RTOL.get(str(self.probs.dtype)),
+                atol=ATOL.get(str(self.probs.dtype)),
+            )
+
+    def test_sample(self):
+        with paddle.static.program_guard(self.program):
+            [data] = self.executor.run(
+                self.program,
+                feed=self.feeds,
+                fetch_list=self._paddle_geometric.sample(),
+            )
+            self.assertTrue(
+                data.shape, np.broadcast_arrays(self.probs)[0].shape
+            )
+
+    def test_rsample(self):
+        with paddle.static.program_guard(self.program):
+            [data] = self.executor.run(
+                self.program,
+                feed=self.feeds,
+                fetch_list=self._paddle_geometric.rsample(),
+            )
+            self.assertTrue(
+                data.shape, np.broadcast_arrays(self.probs)[0].shape
+            )
+
+    def test_entropy(self):
+        with paddle.static.program_guard(self.program):
+            [entropy] = self.executor.run(
+                self.program,
+                feed=self.feeds,
+                fetch_list=[self._paddle_geometric.entropy()],
+            )
+            np.testing.assert_allclose(
+                entropy,
+                scipy.stats.geom.entropy(self.probs),
+                rtol=RTOL.get(str(self.probs.dtype)),
+                atol=ATOL.get(str(self.probs.dtype)),
+            )
+
+    def test_init_prob_type_error(self):
+        with self.assertRaises(TypeError):
+            paddle.distribution.geometric.Geometric([0.5])
+
+
+@place(DEVICES)
+@parameterize_cls(
+    (TEST_CASE_NAME, 'probs', 'value'),
+    [
+        (
+            'one-dim',
+            xrand(
+                (2,),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+            5,
+        ),
+        (
+            'mult-dim',
+            xrand(
+                (2, 2),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+            5,
+        ),
+        (
+            'mult-dim',
+            xrand(
+                (2, 2, 2),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+            5,
+        ),
+    ],
+)
+class TestGeometricPMF(unittest.TestCase):
+    def setUp(self):
+        self.program = paddle.static.Program()
+        self.executor = paddle.static.Executor(self.place)
+        with paddle.static.program_guard(self.program):
+            probs = paddle.static.data(
+                'probs', self.probs.shape, self.probs.dtype
+            )
+
+            self._paddle_geometric = geometric.Geometric(probs)
+            self.feeds = {'probs': self.probs, 'value': self.value}
+
+    def test_pmf(self):
+        with paddle.static.program_guard(self.program):
+            [pmf] = self.executor.run(
+                self.program,
+                feed=self.feeds,
+                fetch_list=[self._paddle_geometric.pmf(self.value)],
+            )
+            np.testing.assert_allclose(
+                pmf,
+                scipy.stats.geom.pmf(self.value, self.probs),
+                rtol=RTOL.get(str(self.probs.dtype)),
+                atol=ATOL.get(str(self.probs.dtype)),
+            )
+
+    def test_log_pmf(self):
+        with paddle.static.program_guard(self.program):
+            [log_pmf] = self.executor.run(
+                self.program,
+                feed=self.feeds,
+                fetch_list=[self._paddle_geometric.log_pmf(self.value)],
+            )
+            np.testing.assert_allclose(
+                log_pmf,
+                scipy.stats.geom.logpmf(self.value, self.probs),
+                rtol=RTOL.get(str(self.probs.dtype)),
+                atol=ATOL.get(str(self.probs.dtype)),
+            )
+
+    def test_cdf(self):
+        with paddle.static.program_guard(self.program):
+            [cdf] = self.executor.run(
+                self.program,
+                feed=self.feeds,
+                fetch_list=[self._paddle_geometric.cdf(self.value)],
+            )
+            np.testing.assert_allclose(
+                cdf,
+                scipy.stats.geom.cdf(self.value, self.probs),
+                rtol=RTOL.get(str(self.probs.dtype)),
+                atol=ATOL.get(str(self.probs.dtype)),
+            )
+
+    def test_pmf_error(self):
+        self.assertRaises(TypeError, self._paddle_geometric.pmf, [1, 2])
+
+    def test_log_pmf_error(self):
+        self.assertRaises(TypeError, self._paddle_geometric.log_pmf, [1, 2])
+
+    def test_cdf_error(self):
+        self.assertRaises(TypeError, self._paddle_geometric.cdf, [1, 2])
+
+
+@place(DEVICES)
+@parameterize_cls(
+    (TEST_CASE_NAME, 'probs1', 'probs2'),
+    [
+        (
+            'one-dim',
+            xrand(
+                (2,),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+            xrand(
+                (2,),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+        ),
+        (
+            'multi-dim',
+            xrand(
+                (2, 2),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+            xrand(
+                (2, 2),
+                dtype='float32',
+                min=np.finfo(dtype='float32').tiny,
+                max=1.0,
+            ),
+        ),
+    ],
+)
+class TestGeometricKL(unittest.TestCase):
+    def setUp(self):
+        paddle.enable_static()
+        self.program_p = paddle.static.Program()
+        self.program_q = paddle.static.Program()
+        self.executor = paddle.static.Executor(self.place)
+        with paddle.static.program_guard(self.program_p, self.program_q):
+            probs_p = paddle.static.data(
+                'probs1', self.probs1.shape, self.probs1.dtype
+            )
+            probs_q = paddle.static.data(
+                'probs2', self.probs2.shape, self.probs2.dtype
+            )
+
+            self._paddle_geomP = geometric.Geometric(probs_p)
+            self._paddle_geomQ = geometric.Geometric(probs_q)
+
+            self.feeds = {
+                'probs1': self.probs1,
+                'probs2': self.probs2,
+            }
+
+    def test_kl_divergence(self):
+        with paddle.static.program_guard(self.program_p, self.program_q):
+            self.executor.run(self.program_q)
+            [kl_diver] = self.executor.run(
+                self.program_p,
+                feed=self.feeds,
+                fetch_list=[
+                    self._paddle_geomP.kl_divergence(self._paddle_geomQ)
+                ],
+            )
+            np.testing.assert_allclose(
+                kl_diver,
+                self._kl(),
+                rtol=RTOL.get(str(self.probs1.dtype)),
+                atol=ATOL.get(str(self.probs1.dtype)),
+            )
+
+    def test_kl1_error(self):
+        self.assertRaises(
+            TypeError,
+            self._paddle_geomP.kl_divergence,
+            paddle.distribution.beta.Beta,
+        )
+
+    def test_kl2_error(self):
+        self.assertRaises(
+            TypeError,
+            self._paddle_geomQ.kl_divergence,
+            paddle.distribution.beta.Beta,
+        )
+
+    def _kl(self):
+        return self.probs1 * np.log(self.probs1 / self.probs2) + (
+            1.0 - self.probs1
+        ) * np.log((1.0 - self.probs1) / (1.0 - self.probs2))

From 896c9315ccaf7c8e9e50780966b6759a5d435fcd Mon Sep 17 00:00:00 2001
From: megemini <megemini@outlook.com>
Date: Fri, 28 Apr 2023 15:30:49 +0800
Subject: [PATCH 137/405] =?UTF-8?q?=E3=80=90Hackathon=204th=20No.12?=
 =?UTF-8?q?=E3=80=91=E4=B8=BA=20Paddle=20=E6=96=B0=E5=A2=9E=20Cauchy=20API?=
 =?UTF-8?q?=20(#52999)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* 【Hackathon 4th No.12】为 Paddle 新增 Cauchy API

* [Change]修改初始化方法与类型检查

* [Change]将测试用例移动到新的目录下

* [Change]适配to_tensor的0D
---
 python/paddle/distribution/__init__.py        |   2 +
 python/paddle/distribution/cauchy.py          | 469 +++++++++++
 python/paddle/distribution/kl.py              |   6 +
 test/distribution/test_distribution_cauchy.py | 726 ++++++++++++++++++
 .../test_distribution_cauchy_static.py        | 472 ++++++++++++
 5 files changed, 1675 insertions(+)
 create mode 100644 python/paddle/distribution/cauchy.py
 create mode 100644 test/distribution/test_distribution_cauchy.py
 create mode 100644 test/distribution/test_distribution_cauchy_static.py

diff --git a/python/paddle/distribution/__init__.py b/python/paddle/distribution/__init__.py
index 05e4b8e11256c..82a2e6ce87a05 100644
--- a/python/paddle/distribution/__init__.py
+++ b/python/paddle/distribution/__init__.py
@@ -16,6 +16,7 @@
 from paddle.distribution.bernoulli import Bernoulli
 from paddle.distribution.beta import Beta
 from paddle.distribution.categorical import Categorical
+from paddle.distribution.cauchy import Cauchy
 from paddle.distribution.dirichlet import Dirichlet
 from paddle.distribution.distribution import Distribution
 from paddle.distribution.gumbel import Gumbel
@@ -35,6 +36,7 @@
     'Bernoulli',
     'Beta',
     'Categorical',
+    'Cauchy',
     'Dirichlet',
     'Distribution',
     'ExponentialFamily',
diff --git a/python/paddle/distribution/cauchy.py b/python/paddle/distribution/cauchy.py
new file mode 100644
index 0000000000000..dde50d8a2287e
--- /dev/null
+++ b/python/paddle/distribution/cauchy.py
@@ -0,0 +1,469 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numbers
+
+import numpy as np
+
+import paddle
+from paddle.distribution import distribution
+from paddle.fluid import framework
+
+
+class Cauchy(distribution.Distribution):
+    r"""Cauchy distribution is also called Cauchy–Lorentz distribution. It is a continuous probability distribution named after Augustin-Louis Cauchy and Hendrik Lorentz. It has a very wide range of applications in natural sciences.
+
+    The Cauchy distribution has the probability density function (PDF):
+
+    .. math::
+
+        { f(x; loc, scale) = \frac{1}{\pi scale \left[1 + \left(\frac{x - loc}{ scale}\right)^2\right]} = { 1 \over \pi } \left[ {  scale \over (x - loc)^2 +  scale^2 } \right], }
+
+    Args:
+        loc (float|Tensor): Location of the peak of the distribution. The data type is float32 or float64.
+        scale (float|Tensor): The half-width at half-maximum (HWHM). The data type is float32 or float64. Must be positive values.
+        name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
+
+    Examples:
+
+        .. code-block:: python
+
+            import paddle
+            from paddle.distribution import Cauchy
+
+            # init Cauchy with float
+            rv = Cauchy(loc=0.1, scale=1.2)
+            print(rv.entropy())
+            # Tensor(shape=1, dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        2.71334577)
+
+            # init Cauchy with N-Dim tensor
+            rv = Cauchy(loc=paddle.to_tensor(0.1), scale=paddle.to_tensor([1.0, 2.0]))
+            print(rv.entropy())
+            # Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        [2.53102422, 3.22417140])
+    """
+
+    def __init__(self, loc, scale, name=None):
+        self.name = name if name is not None else 'Cauchy'
+
+        if not isinstance(loc, (numbers.Real, framework.Variable)):
+            raise TypeError(
+                f"Expected type of loc is Real|Variable, but got {type(loc)}"
+            )
+        if not isinstance(scale, (numbers.Real, framework.Variable)):
+            raise TypeError(
+                f"Expected type of scale is Real|Variable, but got {type(scale)}"
+            )
+
+        if isinstance(loc, numbers.Real):
+            loc = paddle.full(shape=(), fill_value=loc)
+
+        if isinstance(scale, numbers.Real):
+            scale = paddle.full(shape=(), fill_value=scale)
+
+        if loc.shape != scale.shape:
+            self.loc, self.scale = paddle.broadcast_tensors([loc, scale])
+        else:
+            self.loc, self.scale = loc, scale
+
+        self.dtype = self.loc.dtype
+
+        super().__init__(batch_shape=self.loc.shape, event_shape=())
+
+    @property
+    def mean(self):
+        """Mean of Cauchy distribution."""
+        raise ValueError("Cauchy distribution has no mean.")
+
+    @property
+    def variance(self):
+        """Variance of Cauchy distribution."""
+        raise ValueError("Cauchy distribution has no variance.")
+
+    @property
+    def stddev(self):
+        """Standard Deviation of Cauchy distribution."""
+        raise ValueError("Cauchy distribution has no stddev.")
+
+    def sample(self, shape, name=None):
+        """Sample from Cauchy distribution.
+
+        Note:
+            `sample` method has no grad, if you want so, please use `rsample` instead.
+
+        Args:
+            shape (Sequence[int]): Sample shape.
+            name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
+
+        Returns:
+            Tensor: Sampled data with shape `sample_shape` + `batch_shape` + `event_shape`.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Cauchy
+
+                # init Cauchy with float
+                rv = Cauchy(loc=0.1, scale=1.2)
+                print(rv.sample([10]).shape)
+                # [10]
+
+                # init Cauchy with 0-Dim tensor
+                rv = Cauchy(loc=paddle.full((), 0.1), scale=paddle.full((), 1.2))
+                print(rv.sample([10]).shape)
+                # [10]
+
+                # init Cauchy with N-Dim tensor
+                rv = Cauchy(loc=paddle.to_tensor(0.1), scale=paddle.to_tensor([1.0, 2.0]))
+                print(rv.sample([10]).shape)
+                # [10, 2]
+
+                # sample 2-Dim data
+                rv = Cauchy(loc=0.1, scale=1.2)
+                print(rv.sample([10, 2]).shape)
+                # [10, 2]
+
+                rv = Cauchy(loc=paddle.to_tensor(0.1), scale=paddle.to_tensor([1.0, 2.0]))
+                print(rv.sample([10, 2]).shape)
+                # [10, 2, 2]
+        """
+        name = name if name is not None else (self.name + '_sample')
+        with paddle.no_grad():
+            return self.rsample(shape, name)
+
+    def rsample(self, shape, name=None):
+        """Sample from Cauchy distribution (reparameterized).
+
+        Args:
+            shape (Sequence[int]): Sample shape.
+            name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
+
+        Returns:
+            Tensor: Sampled data with shape `sample_shape` + `batch_shape` + `event_shape`.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Cauchy
+
+                # init Cauchy with float
+                rv = Cauchy(loc=0.1, scale=1.2)
+                print(rv.rsample([10]).shape)
+                # [10]
+
+                # init Cauchy with 0-Dim tensor
+                rv = Cauchy(loc=paddle.full((), 0.1), scale=paddle.full((), 1.2))
+                print(rv.rsample([10]).shape)
+                # [10]
+
+                # init Cauchy with N-Dim tensor
+                rv = Cauchy(loc=paddle.to_tensor(0.1), scale=paddle.to_tensor([1.0, 2.0]))
+                print(rv.rsample([10]).shape)
+                # [10, 2]
+
+                # sample 2-Dim data
+                rv = Cauchy(loc=0.1, scale=1.2)
+                print(rv.rsample([10, 2]).shape)
+                # [10, 2]
+
+                rv = Cauchy(loc=paddle.to_tensor(0.1), scale=paddle.to_tensor([1.0, 2.0]))
+                print(rv.rsample([10, 2]).shape)
+                # [10, 2, 2]
+        """
+        name = name if name is not None else (self.name + '_rsample')
+
+        if not isinstance(shape, (np.ndarray, framework.Variable, list, tuple)):
+            raise TypeError(
+                f"Expected type of shape is Sequence[int], but got {type(shape)}"
+            )
+
+        shape = shape if isinstance(shape, tuple) else tuple(shape)
+        shape = self._extend_shape(shape)
+
+        loc = self.loc.expand(shape)
+        scale = self.scale.expand(shape)
+        uniforms = paddle.rand(shape, dtype=self.dtype)
+        return paddle.add(
+            loc,
+            paddle.multiply(scale, paddle.tan(np.pi * (uniforms - 0.5))),
+            name=name,
+        )
+
+    def prob(self, value):
+        r"""Probability density function(PDF) evaluated at value.
+
+        .. math::
+
+            { f(x; loc, scale) = \frac{1}{\pi scale \left[1 + \left(\frac{x - loc}{ scale}\right)^2\right]} = { 1 \over \pi } \left[ {  scale \over (x - loc)^2 +  scale^2 } \right], }
+
+        Args:
+            value (Tensor): Value to be evaluated.
+
+        Returns:
+            Tensor: PDF evaluated at value.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Cauchy
+
+                # init Cauchy with float
+                rv = Cauchy(loc=0.1, scale=1.2)
+                print(rv.prob(paddle.to_tensor(1.5)))
+                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [0.11234467])
+
+                # broadcast to value
+                rv = Cauchy(loc=0.1, scale=1.2)
+                print(rv.prob(paddle.to_tensor([1.5, 5.1])))
+                # Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [0.11234467, 0.01444674])
+
+                # init Cauchy with N-Dim tensor
+                rv = Cauchy(loc=paddle.to_tensor([0.1, 0.1]), scale=paddle.to_tensor([1.0, 2.0]))
+                print(rv.prob(paddle.to_tensor([1.5, 5.1])))
+                # Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [0.10753712, 0.02195240])
+
+                # init Cauchy with N-Dim tensor with broadcast
+                rv = Cauchy(loc=paddle.to_tensor(0.1), scale=paddle.to_tensor([1.0, 2.0]))
+                print(rv.prob(paddle.to_tensor([1.5, 5.1])))
+                # Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [0.10753712, 0.02195240])
+        """
+        name = self.name + '_prob'
+
+        if not isinstance(value, framework.Variable):
+            raise TypeError(
+                f"Expected type of value is Variable, but got {type(value)}"
+            )
+
+        return self.log_prob(value).exp(name=name)
+
+    def log_prob(self, value):
+        """Log of probability densitiy function.
+
+        Args:
+            value (Tensor): Value to be evaluated.
+
+        Returns:
+            Tensor: Log of probability densitiy evaluated at value.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Cauchy
+
+                # init Cauchy with float
+                rv = Cauchy(loc=0.1, scale=1.2)
+                print(rv.log_prob(paddle.to_tensor(1.5)))
+                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [-2.18618369])
+
+                # broadcast to value
+                rv = Cauchy(loc=0.1, scale=1.2)
+                print(rv.log_prob(paddle.to_tensor([1.5, 5.1])))
+                # Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [-2.18618369, -4.23728657])
+
+                # init Cauchy with N-Dim tensor
+                rv = Cauchy(loc=paddle.to_tensor([0.1, 0.1]), scale=paddle.to_tensor([1.0, 2.0]))
+                print(rv.log_prob(paddle.to_tensor([1.5, 5.1])))
+                # Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [-2.22991920, -3.81887865])
+
+                # init Cauchy with N-Dim tensor with broadcast
+                rv = Cauchy(loc=paddle.to_tensor(0.1), scale=paddle.to_tensor([1.0, 2.0]))
+                print(rv.log_prob(paddle.to_tensor([1.5, 5.1])))
+                # Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [-2.22991920, -3.81887865])
+        """
+        name = self.name + '_log_prob'
+
+        if not isinstance(value, framework.Variable):
+            raise TypeError(
+                f"Expected type of value is Variable, but got {type(value)}"
+            )
+
+        value = self._check_values_dtype_in_probs(self.loc, value)
+        loc, scale, value = paddle.broadcast_tensors(
+            [self.loc, self.scale, value]
+        )
+
+        return paddle.subtract(
+            -(
+                paddle.square(paddle.divide(paddle.subtract(value, loc), scale))
+            ).log1p(),
+            paddle.add(
+                paddle.full(loc.shape, np.log(np.pi), dtype=self.dtype),
+                scale.log(),
+            ),
+            name=name,
+        )
+
+    def cdf(self, value):
+        r"""Cumulative distribution function(CDF) evaluated at value.
+
+        .. math::
+
+            { \frac{1}{\pi} \arctan\left(\frac{x-loc}{ scale}\right)+\frac{1}{2}\! }
+
+        Args:
+            value (Tensor): Value to be evaluated.
+
+        Returns:
+            Tensor: CDF evaluated at value.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Cauchy
+
+                # init Cauchy with float
+                rv = Cauchy(loc=0.1, scale=1.2)
+                print(rv.cdf(paddle.to_tensor(1.5)))
+                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [0.77443725])
+
+                # broadcast to value
+                rv = Cauchy(loc=0.1, scale=1.2)
+                print(rv.cdf(paddle.to_tensor([1.5, 5.1])))
+                # Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [0.77443725, 0.92502367])
+
+                # init Cauchy with N-Dim tensor
+                rv = Cauchy(loc=paddle.to_tensor([0.1, 0.1]), scale=paddle.to_tensor([1.0, 2.0]))
+                print(rv.cdf(paddle.to_tensor([1.5, 5.1])))
+                # Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [0.80256844, 0.87888104])
+
+                # init Cauchy with N-Dim tensor with broadcast
+                rv = Cauchy(loc=paddle.to_tensor(0.1), scale=paddle.to_tensor([1.0, 2.0]))
+                print(rv.cdf(paddle.to_tensor([1.5, 5.1])))
+                # Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [0.80256844, 0.87888104])
+        """
+        name = self.name + '_cdf'
+
+        if not isinstance(value, framework.Variable):
+            raise TypeError(
+                f"Expected type of value is Variable, but got {type(value)}"
+            )
+
+        value = self._check_values_dtype_in_probs(self.loc, value)
+        loc, scale, value = paddle.broadcast_tensors(
+            [self.loc, self.scale, value]
+        )
+
+        return (
+            paddle.atan(
+                paddle.divide(paddle.subtract(value, loc), scale), name=name
+            )
+            / np.pi
+            + 0.5
+        )
+
+    def entropy(self):
+        r"""Entropy of Cauchy distribution.
+
+        .. math::
+
+            { \log(4\pi scale)\! }
+
+        Returns:
+            Tensor: Entropy of distribution.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Cauchy
+
+                # init Cauchy with float
+                rv = Cauchy(loc=0.1, scale=1.2)
+                print(rv.entropy())
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        2.71334577)
+
+                # init Cauchy with N-Dim tensor
+                rv = Cauchy(loc=paddle.to_tensor(0.1), scale=paddle.to_tensor([1.0, 2.0]))
+                print(rv.entropy())
+                # Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [2.53102422, 3.22417140])
+
+        """
+        name = self.name + '_entropy'
+        return paddle.add(
+            paddle.full(self.loc.shape, np.log(4 * np.pi), dtype=self.dtype),
+            self.scale.log(),
+            name=name,
+        )
+
+    def kl_divergence(self, other):
+        """The KL-divergence between two Cauchy distributions.
+
+        Note:
+            [1] Frédéric Chyzak, Frank Nielsen, A closed-form formula for the Kullback-Leibler divergence between Cauchy distributions, 2019
+
+        Args:
+            other (Cauchy): instance of Cauchy.
+
+        Returns:
+            Tensor: kl-divergence between two Cauchy distributions.
+
+        Examples:
+
+            .. code-block:: python
+
+                import paddle
+                from paddle.distribution import Cauchy
+
+                rv = Cauchy(loc=0.1, scale=1.2)
+                rv_other = Cauchy(loc=paddle.to_tensor(1.2), scale=paddle.to_tensor([2.3, 3.4]))
+                print(rv.kl_divergence(rv_other))
+                # Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        [0.19819736, 0.31532931])
+        """
+        name = self.name + '_kl_divergence'
+
+        if not isinstance(other, Cauchy):
+            raise TypeError(
+                f"Expected type of other is Cauchy, but got {type(other)}"
+            )
+
+        a_loc = self.loc
+        b_loc = other.loc
+
+        a_scale = self.scale
+        b_scale = other.scale
+
+        t1 = paddle.add(
+            paddle.pow(paddle.add(a_scale, b_scale), 2),
+            paddle.pow(paddle.subtract(a_loc, b_loc), 2),
+        ).log()
+        t2 = (4 * paddle.multiply(a_scale, b_scale)).log()
+
+        return paddle.subtract(t1, t2, name=name)
diff --git a/python/paddle/distribution/kl.py b/python/paddle/distribution/kl.py
index cf004300d9407..2e9b5e878536d 100644
--- a/python/paddle/distribution/kl.py
+++ b/python/paddle/distribution/kl.py
@@ -18,6 +18,7 @@
 from paddle.distribution.bernoulli import Bernoulli
 from paddle.distribution.beta import Beta
 from paddle.distribution.categorical import Categorical
+from paddle.distribution.cauchy import Cauchy
 from paddle.distribution.dirichlet import Dirichlet
 from paddle.distribution.distribution import Distribution
 from paddle.distribution.exponential_family import ExponentialFamily
@@ -186,6 +187,11 @@ def _kl_categorical_categorical(p, q):
     return p.kl_divergence(q)
 
 
+@register_kl(Cauchy, Cauchy)
+def _kl_cauchy_cauchy(p, q):
+    return p.kl_divergence(q)
+
+
 @register_kl(Normal, Normal)
 def _kl_normal_normal(p, q):
     return p.kl_divergence(q)
diff --git a/test/distribution/test_distribution_cauchy.py b/test/distribution/test_distribution_cauchy.py
new file mode 100644
index 0000000000000..c62eb8fd636e1
--- /dev/null
+++ b/test/distribution/test_distribution_cauchy.py
@@ -0,0 +1,726 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import scipy.special
+import scipy.stats
+from config import ATOL, DEVICES, RTOL
+from parameterize import (
+    TEST_CASE_NAME,
+    parameterize_cls,
+    parameterize_func,
+    place,
+)
+from test_distribution import DistributionNumpy
+
+import paddle
+from paddle.distribution import Cauchy
+from paddle.distribution.kl import kl_divergence
+from paddle.fluid.data_feeder import convert_dtype
+
+np.random.seed(2023)
+paddle.seed(2023)
+
+
+def _kstest(samples_a, samples_b):
+    """Uses the Kolmogorov-Smirnov test for goodness of fit."""
+    _, p_value = scipy.stats.ks_2samp(samples_a, samples_b)
+    return not p_value < 0.005
+
+
+class CauchyNumpy(DistributionNumpy):
+    def __init__(self, loc, scale):
+        loc = np.array(loc)
+        scale = np.array(scale)
+        if str(loc.dtype) not in ['float32', 'float64']:
+            self.dtype = 'float32'
+        else:
+            self.dtype = loc.dtype
+
+        self.batch_shape = (loc + scale).shape
+
+        self.loc = loc.astype(self.dtype)
+        self.scale = scale.astype(self.dtype)
+        self.rv = scipy.stats.cauchy(loc=loc, scale=scale)
+
+    def sample(self, shape):
+        shape = np.array(shape, dtype='int')
+        if shape.ndim:
+            shape = shape.tolist()
+        else:
+            shape = [shape.tolist()]
+        return self.rv.rvs(size=shape + list(self.batch_shape))
+
+    def log_prob(self, value):
+        return self.rv.logpdf(value)
+
+    def prob(self, value):
+        return self.rv.pdf(value)
+
+    def cdf(self, value):
+        return self.rv.cdf(value)
+
+    def entropy(self):
+        return self.rv.entropy()
+
+    def kl_divergence(self, other):
+        a_loc = self.loc
+        b_loc = other.loc
+
+        a_scale = self.scale
+        b_scale = other.scale
+
+        t1 = np.log(np.power(a_scale + b_scale, 2) + np.power(a_loc - b_loc, 2))
+        t2 = np.log(4 * a_scale * b_scale)
+        return t1 - t2
+
+
+class CauchyTest(unittest.TestCase):
+    def setUp(self):
+        paddle.disable_static(self.place)
+        with paddle.fluid.dygraph.guard(self.place):
+            # just for convenience
+            self.dtype = self.expected_dtype
+
+            # init numpy with `dtype`
+            self.init_numpy_data(self.loc, self.scale, self.dtype)
+
+            # init paddle and check dtype convert.
+            self.init_dynamic_data(
+                self.loc, self.scale, self.default_dtype, self.dtype
+            )
+
+    def init_numpy_data(self, loc, scale, dtype):
+        loc = np.array(loc).astype(dtype)
+        scale = np.array(scale).astype(dtype)
+        self.rv_np = CauchyNumpy(loc=loc, scale=scale)
+
+    def init_dynamic_data(self, loc, scale, default_dtype, dtype):
+        self.rv_paddle = Cauchy(loc=loc, scale=scale)
+        self.assertTrue(
+            dtype == convert_dtype(self.rv_paddle.loc.dtype),
+            (dtype, self.rv_paddle.loc.dtype),
+        )
+        self.assertTrue(
+            dtype == convert_dtype(self.rv_paddle.scale.dtype),
+            (dtype, self.rv_paddle.scale.dtype),
+        )
+
+
+@place(DEVICES)
+@parameterize_cls(
+    (TEST_CASE_NAME, 'loc', 'scale', 'default_dtype', 'expected_dtype'),
+    [
+        # 0-D params
+        (
+            'params_0d_32_1',
+            paddle.full((), 0.1),
+            paddle.full((), 1.2),
+            'float32',
+            'float32',
+        ),
+        (
+            'params_0d_32_2',
+            paddle.full((), -1.2),
+            paddle.full((), 2.3),
+            'float32',
+            'float32',
+        ),
+        (
+            'params_0d_64_1',
+            paddle.full((), 0.1, dtype='float64'),
+            paddle.full((), 1.2, dtype='float64'),
+            'float64',
+            'float64',
+        ),
+        (
+            'params_0d_64_2',
+            paddle.full((), -1.2, dtype='float64'),
+            paddle.full((), 2.3, dtype='float64'),
+            'float64',
+            'float64',
+        ),
+        # 1-D params
+        ('params_float_1', 0.1, 1.2, 'float64', 'float32'),
+        ('params_float_2', -1.2, 2.3, 'float64', 'float32'),
+        (
+            'params_tensor_32_1',
+            paddle.to_tensor(0.1),
+            paddle.to_tensor(1.2),
+            'float32',
+            'float32',
+        ),
+        (
+            'params_tensor_32_2',
+            paddle.to_tensor(-1.2),
+            paddle.to_tensor(2.3),
+            'float32',
+            'float32',
+        ),
+        (
+            'params_tensor_64_1',
+            paddle.to_tensor(0.1, dtype='float64'),
+            paddle.to_tensor(1.2, dtype='float64'),
+            'float64',
+            'float64',
+        ),
+        (
+            'params_tensor_64_2',
+            paddle.to_tensor(-1.2, dtype='float64'),
+            paddle.to_tensor(2.3, dtype='float64'),
+            'float64',
+            'float64',
+        ),
+        (
+            'params_tensor_list',
+            paddle.to_tensor([0.1]),
+            paddle.to_tensor([1.2]),
+            'float32',
+            'float32',
+        ),
+        (
+            'params_tensor_tuple',
+            paddle.to_tensor((0.1,)),
+            paddle.to_tensor((1.2,)),
+            'float32',
+            'float32',
+        ),
+        # N-D params
+        (
+            'params_0d_1d_1',
+            paddle.full((), 0.1),
+            paddle.full((1,), 1.2),
+            'float32',
+            'float32',
+        ),
+        (
+            'params_0d_1d_2',
+            paddle.full((), 0.1),
+            paddle.to_tensor(1.2),
+            'float32',
+            'float32',
+        ),
+        (
+            'params_1d_0d_1',
+            paddle.full((1,), 0.1),
+            paddle.full((), 1.2),
+            'float32',
+            'float32',
+        ),
+        (
+            'params_1d_0d_2',
+            paddle.to_tensor(0.1),
+            paddle.full((), 1.2),
+            'float32',
+            'float32',
+        ),
+        (
+            'params_0d_3d',
+            paddle.full((), 0.1),
+            paddle.to_tensor([1.1, 2.2, 3.3]),
+            'float32',
+            'float32',
+        ),
+        (
+            'params_3d_0d',
+            paddle.to_tensor([0.1, -0.2, 0.3]),
+            paddle.full((), 1.2),
+            'float32',
+            'float32',
+        ),
+        (
+            'params_1d_3d',
+            paddle.full((1,), 0.1),
+            paddle.to_tensor([1.1, 2.2, 3.3]),
+            'float32',
+            'float32',
+        ),
+        (
+            'params_3d_1d',
+            paddle.to_tensor([0.1, -0.2, 0.3]),
+            paddle.full((1,), 1.2),
+            'float32',
+            'float32',
+        ),
+        (
+            'params_3d_3d',
+            paddle.to_tensor([0.1, -0.2, 0.3]),
+            paddle.to_tensor([1.1, 2.2, 3.3]),
+            'float32',
+            'float32',
+        ),
+    ],
+)
+class CauchyTestFeature(CauchyTest):
+    @parameterize_func(
+        [
+            (paddle.to_tensor([-0.3]),),
+            (paddle.to_tensor([0.3]),),
+            (paddle.to_tensor([1.3]),),
+            (paddle.to_tensor([5.3]),),
+            (paddle.to_tensor(0.3, dtype='float64'),),
+        ]
+    )
+    def test_log_prob(self, value):
+        with paddle.fluid.dygraph.guard(self.place):
+            if convert_dtype(value.dtype) == convert_dtype(
+                self.rv_paddle.loc.dtype
+            ):
+                log_prob = self.rv_paddle.log_prob(value)
+                np.testing.assert_allclose(
+                    log_prob,
+                    self.rv_np.log_prob(value),
+                    rtol=RTOL.get(self.dtype),
+                    atol=ATOL.get(self.dtype),
+                )
+                self.assertTrue(self.dtype == convert_dtype(log_prob.dtype))
+
+            else:
+                with self.assertWarns(UserWarning):
+                    self.rv_paddle.log_prob(value)
+
+    @parameterize_func(
+        [
+            (paddle.to_tensor([-0.3]),),
+            (paddle.to_tensor([0.3]),),
+            (paddle.to_tensor([1.3]),),
+            (paddle.to_tensor([5.3]),),
+            (paddle.to_tensor(0.3, dtype='float64'),),
+        ]
+    )
+    def test_prob(self, value):
+        with paddle.fluid.dygraph.guard(self.place):
+            if convert_dtype(value.dtype) == convert_dtype(
+                self.rv_paddle.loc.dtype
+            ):
+                prob = self.rv_paddle.prob(value)
+                np.testing.assert_allclose(
+                    prob,
+                    self.rv_np.prob(value),
+                    rtol=RTOL.get(self.dtype),
+                    atol=ATOL.get(self.dtype),
+                )
+                self.assertTrue(self.dtype == convert_dtype(prob.dtype))
+
+            else:
+                with self.assertWarns(UserWarning):
+                    self.rv_paddle.prob(value)
+
+    @parameterize_func(
+        [
+            (paddle.to_tensor([-0.3]),),
+            (paddle.to_tensor([0.3]),),
+            (paddle.to_tensor([1.3]),),
+            (paddle.to_tensor([5.3]),),
+            (paddle.to_tensor(0.3, dtype='float64'),),
+        ]
+    )
+    def test_cdf(self, value):
+        with paddle.fluid.dygraph.guard(self.place):
+            if convert_dtype(value.dtype) == convert_dtype(
+                self.rv_paddle.loc.dtype
+            ):
+                cdf = self.rv_paddle.cdf(value)
+                np.testing.assert_allclose(
+                    cdf,
+                    self.rv_np.cdf(value),
+                    rtol=RTOL.get(self.dtype),
+                    atol=ATOL.get(self.dtype),
+                )
+                self.assertTrue(self.dtype == convert_dtype(cdf.dtype))
+
+            else:
+                with self.assertWarns(UserWarning):
+                    self.rv_paddle.cdf(value)
+
+    def test_entropy(self):
+        with paddle.fluid.dygraph.guard(self.place):
+            np.testing.assert_allclose(
+                self.rv_paddle.entropy(),
+                self.rv_np.entropy(),
+                rtol=RTOL.get(self.dtype),
+                atol=ATOL.get(self.dtype),
+            )
+
+    @parameterize_func(
+        [
+            (0.6, 5.7),
+            (-0.6, 5.7),
+        ]
+    )
+    def test_kl_divergence(self, loc, scale):
+        with paddle.fluid.dygraph.guard(self.place):
+
+            # convert loc/scale to paddle's dtype(float32/float64)
+            rv_paddle_other = Cauchy(
+                loc=paddle.full((), loc, dtype=self.rv_paddle.loc.dtype),
+                scale=paddle.full((), scale, dtype=self.rv_paddle.scale.dtype),
+            )
+            rv_np_other = CauchyNumpy(loc=loc, scale=scale)
+
+            np.testing.assert_allclose(
+                self.rv_paddle.kl_divergence(rv_paddle_other),
+                self.rv_np.kl_divergence(rv_np_other),
+                rtol=RTOL.get(self.dtype),
+                atol=ATOL.get(self.dtype),
+            )
+
+            np.testing.assert_allclose(
+                kl_divergence(self.rv_paddle, rv_paddle_other),
+                self.rv_np.kl_divergence(rv_np_other),
+                rtol=RTOL.get(self.dtype),
+                atol=ATOL.get(self.dtype),
+            )
+
+
+@place(DEVICES)
+@parameterize_cls(
+    (
+        TEST_CASE_NAME,
+        'loc',
+        'scale',
+        'default_dtype',
+        'expected_dtype',
+        'shape',
+        'expected_shape',
+    ),
+    [
+        # 0-D params
+        (
+            'params_0d_0d_sample_1d',
+            paddle.full((), 0.1),
+            paddle.full((), 1.2),
+            'float32',
+            'float32',
+            [100],
+            [100],
+        ),
+        (
+            'params_0d_0d_sample_2d',
+            paddle.full((), 0.1),
+            paddle.full((), 1.2),
+            'float32',
+            'float32',
+            [100, 1],
+            [100, 1],
+        ),
+        (
+            'params_0d_0d_sample_3d',
+            paddle.full((), 0.1),
+            paddle.full((), 1.2),
+            'float32',
+            'float32',
+            [100, 2, 3],
+            [100, 2, 3],
+        ),
+        # 1-D params
+        (
+            'params_1d_1d_sample_1d_float',
+            0.1,
+            1.2,
+            'float64',
+            'float32',
+            paddle.to_tensor([100]),
+            [100],
+        ),
+        (
+            'params_1d_1d_sample_1d_32',
+            paddle.to_tensor([0.1]),
+            paddle.to_tensor([1.2]),
+            'float32',
+            'float32',
+            paddle.to_tensor([100]),
+            [100, 1],
+        ),
+        (
+            'params_1d_1d_sample_1d_64',
+            paddle.to_tensor([0.1], dtype='float64'),
+            paddle.to_tensor([1.2], dtype='float64'),
+            'float64',
+            'float64',
+            paddle.to_tensor([100]),
+            [100, 1],
+        ),
+        (
+            'params_1d_1d_sample_2d',
+            paddle.to_tensor([0.1]),
+            paddle.to_tensor([1.2]),
+            'float32',
+            'float32',
+            [100, 2],
+            [100, 2, 1],
+        ),
+        (
+            'params_1d_1d_sample_3d',
+            paddle.to_tensor([0.1]),
+            paddle.to_tensor([1.2]),
+            'float32',
+            'float32',
+            [100, 2, 3],
+            [100, 2, 3, 1],
+        ),
+        # N-D params
+        (
+            'params_0d_1d_sample_1d',
+            paddle.full((), 0.3),
+            paddle.to_tensor([1.2]),
+            'float32',
+            'float32',
+            [100],
+            [100, 1],
+        ),
+        (
+            'params_1d_0d_sample_1d',
+            paddle.to_tensor([0.3]),
+            paddle.full((), 1.2),
+            'float32',
+            'float32',
+            [100],
+            [100, 1],
+        ),
+        (
+            'params_0d_1d_sample_2d',
+            paddle.full((), 0.3),
+            paddle.to_tensor([1.2]),
+            'float32',
+            'float32',
+            [100, 2],
+            [100, 2, 1],
+        ),
+        (
+            'params_1d_0d_sample_2d',
+            paddle.to_tensor([0.3]),
+            paddle.full((), 1.2),
+            'float32',
+            'float32',
+            [100, 2],
+            [100, 2, 1],
+        ),
+        (
+            'params_1d_2d_sample_1d',
+            paddle.to_tensor([0.3]),
+            paddle.to_tensor((1.2, 2.3)),
+            'float32',
+            'float32',
+            [100],
+            [100, 2],
+        ),
+        (
+            'params_2d_1d_sample_1d',
+            paddle.to_tensor((0.3, -0.3)),
+            paddle.to_tensor([1.2]),
+            'float32',
+            'float32',
+            [100],
+            [100, 2],
+        ),
+        (
+            'params_2d_2d_sample_1d',
+            paddle.to_tensor((0.3, -0.3)),
+            paddle.to_tensor((1.2, 2.3)),
+            'float32',
+            'float32',
+            [100],
+            [100, 2],
+        ),
+        (
+            'params_2d_2d_sample_2d',
+            paddle.to_tensor((0.3, -0.3)),
+            paddle.to_tensor((1.2, 2.3)),
+            'float32',
+            'float32',
+            [100, 1],
+            [100, 1, 2],
+        ),
+        (
+            'params_1d_2d_sample_3d',
+            paddle.to_tensor([0.3]),
+            paddle.to_tensor((1.2, 2.3)),
+            'float32',
+            'float32',
+            [100, 1, 2],
+            [100, 1, 2, 2],
+        ),
+    ],
+)
+class CauchyTestSample(CauchyTest):
+    def test_sample(self):
+        with paddle.fluid.dygraph.guard(self.place):
+            sample_np = self.rv_np.sample(self.shape)
+            sample_paddle = self.rv_paddle.sample(self.shape)
+
+            self.assertEqual(list(sample_paddle.shape), self.expected_shape)
+            self.assertEqual(sample_paddle.dtype, self.rv_paddle.loc.dtype)
+
+            if len(self.expected_shape) > len(self.shape):
+                for i in range(self.expected_shape[-1]):
+                    self.assertTrue(
+                        _kstest(
+                            sample_np[..., i].reshape(-1),
+                            sample_paddle.numpy()[..., i].reshape(-1),
+                        )
+                    )
+            else:
+                self.assertTrue(
+                    _kstest(
+                        sample_np.reshape(-1),
+                        sample_paddle.numpy().reshape(-1),
+                    )
+                )
+
+    def test_rsample(self):
+        with paddle.fluid.dygraph.guard(self.place):
+            sample_np = self.rv_np.sample(self.shape)
+            rsample_paddle = self.rv_paddle.rsample(self.shape)
+
+            self.assertEqual(list(rsample_paddle.shape), self.expected_shape)
+            self.assertEqual(rsample_paddle.dtype, self.rv_paddle.loc.dtype)
+
+            if len(self.expected_shape) > len(self.shape):
+                for i in range(self.expected_shape[-1]):
+                    self.assertTrue(
+                        _kstest(
+                            sample_np[..., i].reshape(-1),
+                            rsample_paddle.numpy()[..., i].reshape(-1),
+                        )
+                    )
+            else:
+                self.assertTrue(
+                    _kstest(
+                        sample_np.reshape(-1),
+                        rsample_paddle.numpy().reshape(-1),
+                    )
+                )
+
+    def test_rsample_backpropagation(self):
+        with paddle.fluid.dygraph.guard(self.place):
+            self.rv_paddle.loc.stop_gradient = False
+            self.rv_paddle.scale.stop_gradient = False
+            rsample_paddle = self.rv_paddle.rsample(self.shape)
+            grads = paddle.grad(
+                [rsample_paddle], [self.rv_paddle.loc, self.rv_paddle.scale]
+            )
+            self.assertEqual(len(grads), 2)
+            self.assertEqual(grads[0].dtype, self.rv_paddle.loc.dtype)
+            self.assertEqual(grads[0].shape, self.rv_paddle.loc.shape)
+            self.assertEqual(grads[1].dtype, self.rv_paddle.scale.dtype)
+            self.assertEqual(grads[1].shape, self.rv_paddle.scale.shape)
+
+
+@place(DEVICES)
+@parameterize_cls([TEST_CASE_NAME], ['CauchyTestError'])
+class CauchyTestError(unittest.TestCase):
+    def setUp(self):
+        paddle.disable_static(self.place)
+
+    @parameterize_func(
+        [
+            (-1j + 1, 0.0, TypeError),  # complex
+            (np.array(0.0), 0.0, TypeError),  # ndarray
+            ([0.0, 0.0], 0.0, TypeError),  # list
+            ((0.0, 0.0), 0.0, TypeError),  # tuple
+            (0.0, -1j + 1, TypeError),  # complex
+            (0.0, np.array(0.0), TypeError),  # ndarray
+            (0.0, [0.0, 0.0], TypeError),  # list
+            (0.0, (0.0, 0.0), TypeError),  # tuple
+        ]
+    )
+    def test_bad_init(self, loc, scale, error):
+        with paddle.fluid.dygraph.guard(self.place):
+            self.assertRaises(error, Cauchy, loc, scale)
+
+    def test_bad_property(self):
+        """For property like mean/variance/stddev which is undefined in math,
+        we should raise `ValueError` instead of `NotImplementedError`.
+        """
+        with paddle.fluid.dygraph.guard(self.place):
+            rv = Cauchy(loc=0.0, scale=1.0)
+            with self.assertRaises(ValueError):
+                _ = rv.mean
+            with self.assertRaises(ValueError):
+                _ = rv.variance
+            with self.assertRaises(ValueError):
+                _ = rv.stddev
+
+    @parameterize_func(
+        [
+            (100,),  # int
+            (100.0,),  # float
+        ]
+    )
+    def test_bad_sample_shape_type(self, shape):
+        with paddle.fluid.dygraph.guard(self.place):
+            rv = Cauchy(loc=0.0, scale=1.0)
+
+            with self.assertRaises(TypeError):
+                _ = rv.sample(shape)
+
+            with self.assertRaises(TypeError):
+                _ = rv.rsample(shape)
+
+    @parameterize_func(
+        [
+            (1,),  # int
+            (1.0,),  # float
+            ([1.0],),  # list
+            ((1.0),),  # tuple
+            (np.array(1.0),),  # ndarray
+        ]
+    )
+    def test_bad_value_type(self, value):
+        with paddle.fluid.dygraph.guard(self.place):
+            rv = Cauchy(loc=0.0, scale=1.0)
+
+            with self.assertRaises(TypeError):
+                _ = rv.log_prob(value)
+
+            with self.assertRaises(TypeError):
+                _ = rv.prob(value)
+
+            with self.assertRaises(TypeError):
+                _ = rv.cdf(value)
+
+    @parameterize_func(
+        [
+            (np.array(1.0),),  # ndarray or other distribution
+        ]
+    )
+    def test_bad_kl_other_type(self, other):
+        with paddle.fluid.dygraph.guard(self.place):
+            rv = Cauchy(loc=0.0, scale=1.0)
+
+            with self.assertRaises(TypeError):
+                _ = rv.kl_divergence(other)
+
+    @parameterize_func(
+        [
+            (
+                paddle.to_tensor([0.1, 0.2]),
+                paddle.to_tensor([0.3, 0.4]),
+                paddle.to_tensor([0.1, 0.2, 0.3]),
+            ),
+        ]
+    )
+    def test_bad_broadcast(self, loc, scale, value):
+        with paddle.fluid.dygraph.guard(self.place):
+            rv = Cauchy(loc=loc, scale=scale)
+            self.assertRaises(ValueError, rv.cdf, value)
+            self.assertRaises(ValueError, rv.log_prob, value)
+            self.assertRaises(ValueError, rv.prob, value)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/distribution/test_distribution_cauchy_static.py b/test/distribution/test_distribution_cauchy_static.py
new file mode 100644
index 0000000000000..ecd69a8fbbd77
--- /dev/null
+++ b/test/distribution/test_distribution_cauchy_static.py
@@ -0,0 +1,472 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from config import ATOL, DEVICES, RTOL
+from parameterize import (
+    TEST_CASE_NAME,
+    parameterize_cls,
+    parameterize_func,
+    place,
+)
+from test_distribution_cauchy import CauchyNumpy, _kstest
+
+import paddle
+from paddle.distribution import Cauchy
+from paddle.distribution.kl import kl_divergence
+
+np.random.seed(2023)
+paddle.seed(2023)
+paddle.enable_static()
+default_dtype = paddle.get_default_dtype()
+
+
+@place(DEVICES)
+@parameterize_cls(
+    (TEST_CASE_NAME, 'params'),
+    # params: name, loc, scale, loc_other, scale_other, value
+    [
+        (
+            'params',
+            (
+                # 1-D params
+                (
+                    'params_not_iterable',
+                    0.3,
+                    1.2,
+                    -1.2,
+                    2.3,
+                    3.4,
+                ),
+                (
+                    'params_not_iterable_and_broadcast_for_value',
+                    0.3,
+                    1.2,
+                    -1.2,
+                    2.3,
+                    np.array([[0.1, 1.2], [1.2, 3.4]], dtype=default_dtype),
+                ),
+                # N-D params
+                (
+                    'params_tuple_0305',
+                    (0.3, 0.5),
+                    0.7,
+                    -1.2,
+                    2.3,
+                    3.4,
+                ),
+                (
+                    'params_tuple_03050104',
+                    ((0.3, 0.5), (0.1, 0.4)),
+                    0.7,
+                    -1.2,
+                    2.3,
+                    3.4,
+                ),
+            ),
+        )
+    ],
+)
+class CauchyTestFeature(unittest.TestCase):
+    def setUp(self):
+        self.program = paddle.static.Program()
+        self.executor = paddle.static.Executor(self.place)
+
+        self.params_len = len(self.params)
+
+        with paddle.static.program_guard(self.program):
+            self.init_numpy_data(self.params)
+            self.init_static_data(self.params)
+
+    def init_numpy_data(self, params):
+        self.log_prob_np = []
+        self.prob_np = []
+        self.cdf_np = []
+        self.entropy_np = []
+        self.kl_np = []
+        self.shapes = []
+
+        for _, loc, scale, loc_other, scale_other, value in params:
+            rv_np = CauchyNumpy(loc=loc, scale=scale)
+            rv_np_other = CauchyNumpy(loc=loc_other, scale=scale_other)
+
+            self.log_prob_np.append(rv_np.log_prob(value))
+            self.prob_np.append(rv_np.prob(value))
+            self.cdf_np.append(rv_np.cdf(value))
+            self.entropy_np.append(rv_np.entropy())
+            self.kl_np.append(rv_np.kl_divergence(rv_np_other))
+            # paddle return data ndim>0
+            self.shapes.append(
+                (np.array(loc) + np.array(scale) + np.array(value)).shape
+                or (1,)
+            )
+
+    def init_static_data(self, params):
+        with paddle.static.program_guard(self.program):
+            rv_paddles = []
+            rv_paddles_other = []
+            values = []
+            for name, loc, scale, loc_other, scale_other, value in params:
+                if not isinstance(value, np.ndarray):
+                    value = paddle.full([1], value, dtype=default_dtype)
+                else:
+                    value = paddle.to_tensor(value, place=self.place)
+
+                # We should set name in static mode, or the executor confuse rv_paddles[i].
+                rv_paddles.append(
+                    Cauchy(
+                        loc=paddle.to_tensor(loc),
+                        scale=paddle.to_tensor(scale),
+                        name=name,
+                    )
+                )
+                rv_paddles_other.append(
+                    Cauchy(
+                        loc=paddle.to_tensor(loc_other),
+                        scale=paddle.to_tensor(scale_other),
+                        name=name,
+                    )
+                )
+                values.append(value)
+
+            results = self.executor.run(
+                self.program,
+                feed={},
+                fetch_list=[
+                    [
+                        rv_paddles[i].log_prob(values[i]),
+                        rv_paddles[i].prob(values[i]),
+                        rv_paddles[i].cdf(values[i]),
+                        rv_paddles[i].entropy(),
+                        rv_paddles[i].kl_divergence(rv_paddles_other[i]),
+                        kl_divergence(rv_paddles[i], rv_paddles_other[i]),
+                    ]
+                    for i in range(self.params_len)
+                ],
+            )
+
+            self.log_prob_paddle = []
+            self.prob_paddle = []
+            self.cdf_paddle = []
+            self.entropy_paddle = []
+            self.kl_paddle = []
+            self.kl_func_paddle = []
+            for i in range(self.params_len):
+                (
+                    _log_prob,
+                    _prob,
+                    _cdf,
+                    _entropy,
+                    _kl,
+                    _kl_func,
+                ) = results[i * 6 : (i + 1) * 6]
+                self.log_prob_paddle.append(_log_prob)
+                self.prob_paddle.append(_prob)
+                self.cdf_paddle.append(_cdf)
+                self.entropy_paddle.append(_entropy)
+                self.kl_paddle.append(_kl)
+                self.kl_func_paddle.append(_kl_func)
+
+    def test_all(self):
+        for i in range(self.params_len):
+            self._test_log_prob(i)
+            self._test_prob(i)
+            self._test_cdf(i)
+            self._test_entropy(i)
+            self._test_kl_divergence(i)
+
+    def _test_log_prob(self, i):
+        np.testing.assert_allclose(
+            self.log_prob_np[i],
+            self.log_prob_paddle[i],
+            rtol=RTOL.get(default_dtype),
+            atol=ATOL.get(default_dtype),
+        )
+
+        # check shape
+        self.assertTrue(self.log_prob_paddle[i].shape == self.shapes[i])
+
+    def _test_prob(self, i):
+        np.testing.assert_allclose(
+            self.prob_np[i],
+            self.prob_paddle[i],
+            rtol=RTOL.get(default_dtype),
+            atol=ATOL.get(default_dtype),
+        )
+
+        # check shape
+        self.assertTrue(self.prob_paddle[i].shape == self.shapes[i])
+
+    def _test_cdf(self, i):
+        np.testing.assert_allclose(
+            self.cdf_np[i],
+            self.cdf_paddle[i],
+            rtol=RTOL.get(default_dtype),
+            atol=ATOL.get(default_dtype),
+        )
+
+        # check shape
+        self.assertTrue(self.cdf_paddle[i].shape == self.shapes[i])
+
+    def _test_entropy(self, i):
+        np.testing.assert_allclose(
+            self.entropy_np[i],
+            self.entropy_paddle[i],
+            rtol=RTOL.get(default_dtype),
+            atol=ATOL.get(default_dtype),
+        )
+
+    def _test_kl_divergence(self, i):
+        np.testing.assert_allclose(
+            self.kl_np[i],
+            self.kl_paddle[i],
+            rtol=RTOL.get(default_dtype),
+            atol=ATOL.get(default_dtype),
+        )
+
+        np.testing.assert_allclose(
+            self.kl_np[i],
+            self.kl_func_paddle[i],
+            rtol=RTOL.get(default_dtype),
+            atol=ATOL.get(default_dtype),
+        )
+
+
+@place(DEVICES)
+@parameterize_cls(
+    (
+        TEST_CASE_NAME,
+        'loc',
+        'scale',
+        'shape',
+        'expected_shape',
+    ),
+    [
+        # 1-D params
+        (
+            'params_1d',
+            [0.1],
+            [1.2],
+            [100],
+            [100, 1],
+        ),
+        # N-D params
+        (
+            'params_2d',
+            [0.3],
+            [1.2, 2.3],
+            [100],
+            [100, 2],
+        ),
+    ],
+)
+class CauchyTestSample(unittest.TestCase):
+    def setUp(self):
+        self.program = paddle.static.Program()
+        self.executor = paddle.static.Executor(self.place)
+
+        with paddle.static.program_guard(self.program):
+            self.init_numpy_data(self.loc, self.scale, self.shape)
+            self.init_static_data(self.loc, self.scale, self.shape)
+
+    def init_numpy_data(self, loc, scale, shape):
+        self.rv_np = CauchyNumpy(loc=loc, scale=scale)
+        self.sample_np = self.rv_np.sample(shape)
+
+    def init_static_data(self, loc, scale, shape):
+        with paddle.static.program_guard(self.program):
+            self.rv_paddle = Cauchy(
+                loc=paddle.to_tensor(loc),
+                scale=paddle.to_tensor(scale),
+            )
+
+            [self.sample_paddle, self.rsample_paddle] = self.executor.run(
+                self.program,
+                feed={},
+                fetch_list=[
+                    self.rv_paddle.sample(shape),
+                    self.rv_paddle.rsample(shape),
+                ],
+            )
+
+    def test_sample(self):
+        with paddle.static.program_guard(self.program):
+            self.assertEqual(
+                list(self.sample_paddle.shape), self.expected_shape
+            )
+
+            for i in range(self.expected_shape[-1]):
+                self.assertTrue(
+                    _kstest(
+                        self.sample_np[..., i].reshape(-1),
+                        self.sample_paddle[..., i].reshape(-1),
+                    )
+                )
+
+    def test_rsample(self):
+        """Compare two samples from `rsample` method, one from scipy and another from paddle."""
+        with paddle.static.program_guard(self.program):
+            self.assertEqual(
+                list(self.rsample_paddle.shape), self.expected_shape
+            )
+
+            for i in range(self.expected_shape[-1]):
+                self.assertTrue(
+                    _kstest(
+                        self.sample_np[..., i].reshape(-1),
+                        self.rsample_paddle[..., i].reshape(-1),
+                    )
+                )
+
+
+@place(DEVICES)
+@parameterize_cls([TEST_CASE_NAME], ['CauchyTestError'])
+class CauchyTestError(unittest.TestCase):
+    def setUp(self):
+        self.program = paddle.static.Program()
+        self.executor = paddle.static.Executor(self.place)
+
+    @parameterize_func(
+        [
+            ((0.3,),),  # tuple
+            ([0.3],),  # list
+            (np.array([0.3]),),  # ndarray
+            (-1j + 1,),  # complex
+            ('0',),  # str
+        ]
+    )
+    def test_bad_init_type(self, param):
+        """Test bad init for loc/scale"""
+        with paddle.static.program_guard(self.program):
+            with self.assertRaises(TypeError):
+                [_] = self.executor.run(
+                    self.program,
+                    feed={},
+                    fetch_list=[Cauchy(loc=0.0, scale=param).scale],
+                )
+            with self.assertRaises(TypeError):
+                [_] = self.executor.run(
+                    self.program,
+                    feed={},
+                    fetch_list=[Cauchy(loc=param, scale=1.0).loc],
+                )
+
+    def test_bad_property(self):
+        """For property like mean/variance/stddev which is undefined in math,
+        we should raise `ValueError` instead of `NotImplementedError`.
+        """
+        with paddle.static.program_guard(self.program):
+            rv = Cauchy(loc=0.0, scale=1.0)
+            with self.assertRaises(ValueError):
+                _ = rv.mean
+            with self.assertRaises(ValueError):
+                _ = rv.variance
+            with self.assertRaises(ValueError):
+                _ = rv.stddev
+
+    @parameterize_func(
+        [
+            (100,),  # int
+            (100.0,),  # float
+        ]
+    )
+    def test_bad_sample_shape_type(self, shape):
+        with paddle.static.program_guard(self.program):
+            rv = Cauchy(loc=0.0, scale=1.0)
+
+            with self.assertRaises(TypeError):
+                [_] = self.executor.run(
+                    self.program, feed={}, fetch_list=[rv.sample(shape)]
+                )
+
+            with self.assertRaises(TypeError):
+                [_] = self.executor.run(
+                    self.program, feed={}, fetch_list=[rv.rsample(shape)]
+                )
+
+    @parameterize_func(
+        [
+            (1,),  # int
+            (1.0,),  # float
+            ([1.0],),  # list
+            ((1.0),),  # tuple
+            (np.array(1.0),),  # ndarray
+        ]
+    )
+    def test_bad_value_type(self, value):
+        with paddle.static.program_guard(self.program):
+            rv = Cauchy(loc=0.0, scale=1.0)
+
+            with self.assertRaises(TypeError):
+                [_] = self.executor.run(
+                    self.program, feed={}, fetch_list=[rv.log_prob(value)]
+                )
+
+            with self.assertRaises(TypeError):
+                [_] = self.executor.run(
+                    self.program, feed={}, fetch_list=[rv.prob(value)]
+                )
+
+            with self.assertRaises(TypeError):
+                [_] = self.executor.run(
+                    self.program, feed={}, fetch_list=[rv.cdf(value)]
+                )
+
+    @parameterize_func(
+        [
+            (np.array(1.0),),  # ndarray or other distribution
+        ]
+    )
+    def test_bad_kl_other_type(self, other):
+        with paddle.static.program_guard(self.program):
+            rv = Cauchy(loc=0.0, scale=1.0)
+
+            with self.assertRaises(TypeError):
+                [_] = self.executor.run(
+                    self.program, feed={}, fetch_list=[rv.kl_divergence(other)]
+                )
+
+    @parameterize_func(
+        [
+            (paddle.to_tensor([0.1, 0.2, 0.3]),),
+        ]
+    )
+    def test_bad_broadcast(self, value):
+        with paddle.static.program_guard(self.program):
+            rv = Cauchy(
+                loc=paddle.to_tensor(0.0), scale=paddle.to_tensor((1.0, 2.0))
+            )
+
+            # `logits, value = paddle.broadcast_tensors([self.logits, value])`
+            # raise ValueError in dygraph, raise TypeError in static.
+            with self.assertRaises(TypeError):
+                [_] = self.executor.run(
+                    self.program, feed={}, fetch_list=[rv.cdf(value)]
+                )
+
+            with self.assertRaises(TypeError):
+                [_] = self.executor.run(
+                    self.program, feed={}, fetch_list=[rv.log_prob(value)]
+                )
+
+            with self.assertRaises(TypeError):
+                [_] = self.executor.run(
+                    self.program, feed={}, fetch_list=[rv.prob(value)]
+                )
+
+
+if __name__ == '__main__':
+    unittest.main()

From 005fee12c7009d5aa2de75efc224b843ffe18f53 Mon Sep 17 00:00:00 2001
From: Sanbu <96160062+sanbuphy@users.noreply.github.com>
Date: Fri, 28 Apr 2023 15:40:29 +0800
Subject: [PATCH 138/405] Support static graph code-gen for unpool (#52947)

---
 paddle/fluid/operators/unpool_op.cc      | 85 ------------------------
 paddle/phi/api/yaml/backward.yaml        | 11 +++
 paddle/phi/api/yaml/legacy_backward.yaml | 11 ---
 paddle/phi/api/yaml/legacy_ops.yaml      | 10 ---
 paddle/phi/api/yaml/op_compat.yaml       | 10 +++
 paddle/phi/api/yaml/ops.yaml             | 10 +++
 paddle/phi/ops/compat/unpool_sig.cc      | 36 ----------
 7 files changed, 31 insertions(+), 142 deletions(-)
 delete mode 100644 paddle/phi/ops/compat/unpool_sig.cc

diff --git a/paddle/fluid/operators/unpool_op.cc b/paddle/fluid/operators/unpool_op.cc
index 6eb6b81eb4d3d..62441dd6127e3 100644
--- a/paddle/fluid/operators/unpool_op.cc
+++ b/paddle/fluid/operators/unpool_op.cc
@@ -24,64 +24,6 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-class Unpool2dOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput(
-        "X",
-        "(Tensor) The input tensor of unpool operator. "
-        "The format of input tensor is NCHW. Where N is batch size, C is the "
-        "number of channels, H and W is the height and width of feature.");
-    AddInput(
-        "Indices",
-        "(Tensor) The input tensor of the indices given out by MaxPool2d. "
-        "The format of input tensor is NCHW. Where N is batch size, C is the "
-        "number of channels, H and W is the height and width of feature.");
-    AddOutput("Out",
-              "(Tensor) The output tensor of unpool operator."
-              "The format of output tensor is also NCHW."
-              "Where N is batch size, C is "
-              "the number of channels, H and W is the height and "
-              "width of feature.");
-    AddAttr<std::vector<int>>(
-        "ksize",
-        "(vector), the unpooling window size(height, width) "
-        "of unpooling operator.");
-    AddAttr<std::vector<int>>("strides",
-                              "(vector, default:{1, 1}), "
-                              "strides (height, width) of unpooling operator.")
-        .SetDefault({1, 1});
-    AddAttr<std::vector<int>>("paddings",
-                              "(vector default:{0,0}), "
-                              "paddings (height, width) of unpooling operator.")
-        .SetDefault({0, 0});
-    AddAttr<std::string>(
-        "unpooling_type",
-        "(string), unpooling type, can be \"max\" for max-unpooling ")
-        .InEnum({"max"});
-    AddAttr<std::vector<int>>("output_size",
-                              "(vector, optional). The shape of output.")
-        .SetDefault({0, 0})
-        .SupportTensor();
-    AddAttr<std::string>(
-        "data_format",
-        "(string, default NCHW) Only used in "
-        "An optional string from: \"NHWC\", \"NCHW\". "
-        "Defaults to \"NHWC\". Specify the data format of the output data, "
-        "the input will be transformed automatically. ")
-        .SetDefault("NCHW");
-    AddComment(R"DOC(
-Input shape is: $(N, C_{in}, H_{in}, W_{in})$, Output shape is:
-$(N, C_{out}, H_{out}, W_{out})$, where
-$$
-H_{out} = (H_{in}-1) * strides[0] - 2 * paddings[0] + ksize[0] \\
-W_{out} = (W_{in}-1) * strides[1] - 2 * paddings[1] + ksize[1]
-$$
-Paper: http://www.matthewzeiler.com/wp-content/uploads/2017/07/iccv2011.pdf
-)DOC");
-  }
-};
-
 class Unpool3dOpMaker : public framework::OpProtoAndCheckerMaker {
  public:
   void Make() override {
@@ -200,18 +142,6 @@ class Unpool3dOpGradMaker : public framework::SingleGradOpMaker<T> {
   }
 };
 
-class UnpoolOpGrad : public framework::OperatorWithKernel {
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(ctx, "X"),
-                          ctx.GetPlace());
-  }
-
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-};
-
 class Unpool3dOpGrad : public framework::OperatorWithKernel {
  protected:
   phi::KernelKey GetExpectedKernelType(
@@ -228,21 +158,6 @@ class Unpool3dOpGrad : public framework::OperatorWithKernel {
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-DECLARE_INFER_SHAPE_FUNCTOR(unpool,
-                            UnpoolInferShapeFunctor,
-                            PD_INFER_META(phi::UnpoolInferMeta));
-REGISTER_OPERATOR(unpool,
-                  ops::UnpoolOp,
-                  ops::Unpool2dOpMaker,
-                  ops::UnpoolOpGradMaker<paddle::framework::OpDesc>,
-                  ops::UnpoolOpGradMaker<paddle::imperative::OpBase>,
-                  UnpoolInferShapeFunctor);
-
-DECLARE_INFER_SHAPE_FUNCTOR(unpool_grad,
-                            UnpoolGradInferShapeFunctor,
-                            PD_INFER_META(phi::UnchangedInferMeta));
-
-REGISTER_OPERATOR(unpool_grad, ops::UnpoolOpGrad, UnpoolGradInferShapeFunctor);
 
 DECLARE_INFER_SHAPE_FUNCTOR(unpool,
                             Unpool3dInferShapeFunctor,
diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index 97a0968f17198..0e348847554c2 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -2064,3 +2064,14 @@
   kernel :
     func : where_grad
   no_need_buffer : x, y
+
+- backward_op: unpool_grad
+  forward: unpool (Tensor x, Tensor indices, int[] ksize, int[] strides = {1,1}, int[] paddings ={0,0} ,IntArray output_size = {0,0}, str data_format="NCHW") -> Tensor(out)
+  args: (Tensor x, Tensor indices, Tensor out, Tensor out_grad, int[] ksize, int[] strides, int[] paddings, IntArray output_size, str data_format)
+  output: Tensor(x_grad)
+  infer_meta:
+    func: UnchangedInferMeta
+    param : [x]
+  kernel:
+    func: unpool_grad
+    data_type: x
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index 5f971d988538e..8a3f0c15b677f 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -1053,14 +1053,3 @@
   kernel:
     func: unpool3d_grad
     data_type: x
-
-- backward_op: unpool_grad
-  forward: unpool (Tensor x, Tensor indices, int[] ksize, int[] strides, int[] padding,  IntArray output_size, str data_format) -> Tensor(out)
-  args: (Tensor x, Tensor indices, Tensor out, Tensor out_grad, int[] ksize, int[] strides, int[] padding, IntArray output_size, str data_format)
-  output: Tensor(x_grad)
-  infer_meta:
-    func: UnchangedInferMeta
-    param : [x]
-  kernel:
-    func: unpool_grad
-    data_type: x
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index c3610b4ae5ae1..128b078a78662 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -1217,16 +1217,6 @@
     func : unique
     data_type : x
 
-- op : unpool
-  args: (Tensor x, Tensor indices, int[] ksize, int[] strides, int[] padding, IntArray output_size, str data_format)
-  output: Tensor(out)
-  infer_meta:
-    func: UnpoolInferMeta
-  kernel:
-    func: unpool
-    data_type: x
-  backward: unpool_grad
-
 - op : unpool3d
   args: (Tensor x, Tensor indices, int[] ksize, int[] strides, int[] padding, int[] output_size, str data_format)
   output: Tensor(out)
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index c3d326eafbfae..564a7de4e942a 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -2411,6 +2411,16 @@
   outputs :
     {out : Out, index : Index, counts : Counts}
 
+- op : unpool
+  inputs :
+    {x : X, indices: Indices}
+  outputs :
+    out : Out
+  int_array :
+    output_size:
+      data_type : int
+      support_tensor : true
+
 - op : unsqueeze (unsqueeze2)
   backward : unsqueeze_grad (unsqueeze2_grad), unsqueeze_double_grad(unsqueeze2_double_grad)
   inputs :
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index bea41007e5aec..72bf1d67a2830 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -2133,6 +2133,16 @@
     data_type : x
   optional : index, counts
 
+- op : unpool
+  args: (Tensor x, Tensor indices, int[] ksize, int[] strides = {1,1}, int[] paddings ={0,0} ,IntArray output_size = {0,0}, str data_format="NCHW")
+  output: Tensor(out)
+  infer_meta:
+    func: UnpoolInferMeta
+  kernel:
+    func: unpool
+    data_type: x
+  backward: unpool_grad
+
 - op : unsqueeze
   args : (Tensor x, IntArray axis = {})
   output : Tensor(out), Tensor(xshape)
diff --git a/paddle/phi/ops/compat/unpool_sig.cc b/paddle/phi/ops/compat/unpool_sig.cc
deleted file mode 100644
index fb751b4b4b652..0000000000000
--- a/paddle/phi/ops/compat/unpool_sig.cc
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/core/compat/op_utils.h"
-namespace phi {
-
-KernelSignature UnpoolOpArgumentMapping(const ArgumentMappingContext& ctx) {
-  return KernelSignature(
-      "unpool",
-      {"X", "Indices"},
-      {"ksize", "strides", "paddings", "output_size", "data_format"},
-      {"Out"});
-}
-
-KernelSignature UnpoolGradOpArgumentMapping(const ArgumentMappingContext& ctx) {
-  return KernelSignature(
-      "unpool_grad",
-      {"X", "Indices", "Out", "Out@GRAD"},
-      {"ksize", "strides", "paddings", "output_size", "data_format"},
-      {"X@GRAD"});
-}
-
-}  // namespace phi
-PD_REGISTER_ARG_MAPPING_FN(unpool, phi::UnpoolOpArgumentMapping);
-PD_REGISTER_ARG_MAPPING_FN(unpool_grad, phi::UnpoolGradOpArgumentMapping);

From 396fe483c607ac6339e30616c441189fd2cfad04 Mon Sep 17 00:00:00 2001
From: huangjiyi <43315610+huangjiyi@users.noreply.github.com>
Date: Fri, 28 Apr 2023 15:44:48 +0800
Subject: [PATCH 139/405] Support static graph code generation for op
 edit_distance (#53297)

* update

* fix bug

* support parsing fixed kernel data_type

* update op_compat

* update
---
 paddle/fluid/operators/edit_distance_op.cc    | 104 ------------------
 .../generator/templates/operator_utils.c.j2   |   5 +-
 paddle/phi/api/yaml/legacy_ops.yaml           |  10 --
 paddle/phi/api/yaml/op_compat.yaml            |  10 ++
 paddle/phi/api/yaml/ops.yaml                  |  10 ++
 paddle/phi/ops/compat/edit_distance_sig.cc    |  29 -----
 6 files changed, 24 insertions(+), 144 deletions(-)
 delete mode 100644 paddle/fluid/operators/edit_distance_op.cc
 delete mode 100644 paddle/phi/ops/compat/edit_distance_sig.cc

diff --git a/paddle/fluid/operators/edit_distance_op.cc b/paddle/fluid/operators/edit_distance_op.cc
deleted file mode 100644
index 5eef3d72b3908..0000000000000
--- a/paddle/fluid/operators/edit_distance_op.cc
+++ /dev/null
@@ -1,104 +0,0 @@
-/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/phi/infermeta/multiary.h"
-
-namespace paddle {
-namespace operators {
-
-class EditDistanceOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext &ctx) const override {
-    return phi::KernelKey(framework::proto::VarType::FP32,
-                          ctx.device_context().GetPlace());
-  }
-};
-
-class EditDistanceOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput("Hyps",
-             "2-D Tensor<int64_t>, or 2-D phi::DenseTensor<int64_t> with last "
-             "dimension being 1. "
-             "The indices for hypothesis strings.");
-    AddInput("Refs",
-             "2-D Tensor<int64_t>, or 2-D phi::DenseTensor<int64_t> with last "
-             "dimension being 1. "
-             "The indices for reference strings.");
-    AddInput("HypsLength",
-             "1-D Tensor<int64_t>. "
-             "Sequence length for hyps when hyps is a tensor")
-        .AsDispensable();
-    AddInput("RefsLength",
-             "1-D Tensor<int64_t>. "
-             "Sequence length for refs when refs is a tensor")
-        .AsDispensable();
-    AddOutput("SequenceNum", "The sequence count of current batch");
-    AddAttr<bool>("normalized",
-                  "(bool, default false) Indicated whether to normalize "
-                  "the edit distance by the length of reference string.")
-        .SetDefault(false);
-    AddOutput("Out",
-              "(2-D Tensor with shape [`batch_size` x 1]) "
-              "The output edit distances of EditDistance operator.");
-    AddComment(R"DOC(
-
-EditDistance operator computes the edit distances between a batch of hypothesis
-strings and their references.
-
-Edit distance, also called Levenshtein distance, measures how dissimilar two strings
-are by counting the minimum number of operations to transform one string into another.
-The operations include insertion, deletion, and substitution.
-
-For example, given hypothesis string A = "kitten" and reference B = "sitting",
-A will be transformed into B at least after two substitutions and one
-insertion:
-
-   "kitten" -> "sitten" -> "sittin" -> "sitting"
-
-So the edit distance between A and B is 3.
-
-Input(Hyps) is a 2-D Tensor or a 2-D phi::DenseTensor consisting of all the hypothesis strings.
-And the `batch_size` reference strings are arranged in order in the same way in the
-Input(Refs).
-
-Output(Out) contains the `batch_size` results and each stands for the edit distance
-for a pair of strings respectively. If Attr(normalized) is true, the edit distance
-will be divided by the length of reference string.
-)DOC");
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-DECLARE_INFER_SHAPE_FUNCTOR(edit_distance,
-                            EditDistanceShapeFunctor,
-                            PD_INFER_META(phi::EditDistanceInferMeta));
-
-namespace ops = paddle::operators;
-
-REGISTER_OPERATOR(
-    edit_distance,
-    ops::EditDistanceOp,
-    ops::EditDistanceOpMaker,
-    paddle::framework::EmptyGradOpMaker<paddle::framework::OpDesc>,
-    paddle::framework::EmptyGradOpMaker<paddle::imperative::OpBase>,
-    EditDistanceShapeFunctor);
diff --git a/paddle/fluid/operators/generator/templates/operator_utils.c.j2 b/paddle/fluid/operators/generator/templates/operator_utils.c.j2
index 30272c2a30d62..0b0190f4e4e55 100644
--- a/paddle/fluid/operators/generator/templates/operator_utils.c.j2
+++ b/paddle/fluid/operators/generator/templates/operator_utils.c.j2
@@ -318,13 +318,16 @@ phi::KernelKey GetExpectedKernelType(
   {% if kernel["data_type"]["candidates"] | length == 1 %}
     {% set data_type_arg = kernel["data_type"]["candidates"][0] %}
     {% set inputs = op["inputs"] | map(attribute="fluid_name") | list %}
+    {% set attrs = op["attrs"] | map(attribute="fluid_name") | list %}
     {% if data_type_arg in inputs %}
   auto data_type = framework::OperatorWithKernel::IndicateVarDataType(ctx, {{data_type_arg | to_opmaker_name}});
       {% if kernel["data_type"]["to_complex_flag"][0] %}
   data_type = framework::ToComplexType(data_type);
       {% endif %}
-    {% else %}{# it is an attribute and probably named dtype#}
+    {% elif data_type_arg in attrs %}{# it is an attribute and probably named dtype#}
   auto data_type = framework::proto::VarType::Type(ctx.Attr<int>("{{data_type_arg}}"));
+    {% else %}
+  auto data_type = framework::TransToProtoVarType(phi::{{data_type_arg}});
     {% endif %}
   {% elif kernel["data_type"]["candidates"] | length == 2 %}
     {% set data_type_args = kernel["data_type"]["candidates"] %}
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index 128b078a78662..e941b54fe5030 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -312,16 +312,6 @@
   optional : seed_tensor
   backward : dropout_grad
 
-- op : edit_distance
-  args : (Tensor hyps, Tensor refs, Tensor hypslength, Tensor refslength, bool normalized = false)
-  output : Tensor(sequencenum), Tensor(out)
-  infer_meta :
-    func : EditDistanceInferMeta
-  kernel :
-    func : edit_distance
-    data_type: DataType::FLOAT32
-  optional : hypslength, refslength
-
 - op : einsum
   args : (Tensor[] x, str equation)
   output : Tensor, Tensor[]{x.size()}, Tensor[]{x.size()}
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index 564a7de4e942a..23e509e5dd1a4 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -683,6 +683,16 @@
   extra :
     attrs : [bool fix_seed = false, int seed = 0]
 
+- op : edit_distance
+  inputs :
+    hyps : Hyps
+    refs : Refs
+    hypslength : HypsLength
+    refslength : RefsLength
+  outputs :
+    sequencenum : SequenceNum
+    out : Out
+
 - op : eig
   inputs :
     x : X
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index 72bf1d67a2830..b2c262861ba16 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -565,6 +565,16 @@
     data_type : x
   backward : dot_grad
 
+- op : edit_distance
+  args : (Tensor hyps, Tensor refs, Tensor hypslength, Tensor refslength, bool normalized = false)
+  output : Tensor(sequencenum), Tensor(out)
+  infer_meta :
+    func : EditDistanceInferMeta
+  kernel :
+    func : edit_distance
+    data_type : DataType::FLOAT32
+  optional : hypslength, refslength
+
 - op : eig
   args: (Tensor x)
   output: Tensor(out_w), Tensor(out_v)
diff --git a/paddle/phi/ops/compat/edit_distance_sig.cc b/paddle/phi/ops/compat/edit_distance_sig.cc
deleted file mode 100644
index e558b354e1c2b..0000000000000
--- a/paddle/phi/ops/compat/edit_distance_sig.cc
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/core/compat/op_utils.h"
-
-namespace phi {
-
-KernelSignature EditDistanceOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature("edit_distance",
-                         {"Hyps", "Refs", "HypsLength", "RefsLength"},
-                         {"normalized"},
-                         {"SequenceNum", "Out"});
-}
-
-}  // namespace phi
-
-PD_REGISTER_ARG_MAPPING_FN(edit_distance, phi::EditDistanceOpArgumentMapping);

From 96180fff65243316d64767a77c5e0536c6655e1b Mon Sep 17 00:00:00 2001
From: Meteor Liu <liudongxue01@baidu.com>
Date: Fri, 28 Apr 2023 15:47:38 +0800
Subject: [PATCH 140/405] replace varbase to relevant name or notes as per the
 context (#53431)

---
 .../sharding/group_sharded_stage3.py          |  12 +-
 python/paddle/distributed/parallel.py         |  30 +++--
 python/paddle/fluid/dygraph/base.py           |   2 +-
 .../tests/unittests/test_eager_run_program.py |   5 +-
 .../tests/unittests/test_imperative_group.py  | 106 +++++++++---------
 .../test_view_op_reuse_allocation.py          |   4 +-
 .../paddle/jit/dy2static/partial_program.py   |  15 ++-
 python/paddle/jit/translated_layer.py         |   6 +-
 test/dygraph_to_static/test_list.py           |   6 +-
 9 files changed, 90 insertions(+), 96 deletions(-)

diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py
index d658fc1e82e36..bdf0ec29cd776 100644
--- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py
+++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py
@@ -584,7 +584,7 @@ def _update_params(self):
                 param, "fw_storage"
             ), f"Find {param.name} don't have fw_storage attribute"
 
-            param.fw_storage = _VarBaseWrapper(param)
+            param.fw_storage = _TensorWrapper(param)
             assert param.fw_storage.grad is None
             param.fw_storage._copy_gradient_from(param.bw_storage)
             update_list.append(param)
@@ -1062,17 +1062,17 @@ def _UnsliceParam(param):
     return param
 
 
-def _VarBaseWrapper(param):
-    varbase = param.fw_storage
+def _TensorWrapper(param):
+    var = param.fw_storage
     tmp_param = EagerParamBase(
-        shape=varbase.shape, dtype=varbase.dtype, name="slice@" + param.name
+        shape=var.shape, dtype=var.dtype, name="slice@" + param.name
     )
-    varbase._share_buffer_to(tmp_param)
+    var._share_buffer_to(tmp_param)
     tmp_param.regularizer = param.regularizer
     tmp_param.optimize_attr['learning_rate'] = param.optimize_attr[
         'learning_rate'
     ]
-    varbase._clear()
+    var._clear()
     return tmp_param
 
 
diff --git a/python/paddle/distributed/parallel.py b/python/paddle/distributed/parallel.py
index 30a3e82eaf1b2..1737d0731a90d 100644
--- a/python/paddle/distributed/parallel.py
+++ b/python/paddle/distributed/parallel.py
@@ -146,20 +146,18 @@ def sync_params_buffers(
     for _, param in model._obtain_parameters_buffers().items():
         if not isinstance(param, core.eager.Tensor):
             raise TypeError(
-                "The data type of '%s' must be Varbase or eager.Tensor"
-                % param.name
+                "The data type of '%s' must be core.eager.Tensor" % param.name
             )
 
-        # is_distributed param not need to sync when in mp mode
-        if isinstance(param, core.eager.Tensor):
-            if is_model_parallel:
-                if hasattr(param, "is_distributed") and param.is_distributed:
-                    continue
-
-            # NOTE(shenliang03): Support situations that do not require synchronization parameters,
-            # such as moe's expert parameters
-            if getattr(param, "no_sync", False):
+        if is_model_parallel:
+            if hasattr(param, "is_distributed") and param.is_distributed:
                 continue
+
+        # NOTE(shenliang03): Support situations that do not require synchronization parameters,
+        # such as moe's expert parameters
+        if getattr(param, "no_sync", False):
+            continue
+
         if param.type == core.VarDesc.VarType.VOCAB:
             continue
 
@@ -474,14 +472,14 @@ def check_layer_sparse(sublayer):
                 self.find_unused_parameters,
             )
 
-    def _find_varbase(self, obj):
+    def _find_tensor(self, obj):
         var_type = core.eager.Tensor
         if isinstance(obj, var_type):
             return [obj]
         if isinstance(obj, (list, tuple)):
-            return itertools.chain(*map(self._find_varbase, obj))
+            return itertools.chain(*map(self._find_tensor, obj))
         if isinstance(obj, dict):
-            return itertools.chain(*map(self._find_varbase, obj.values()))
+            return itertools.chain(*map(self._find_tensor, obj.values()))
         return []
 
     @contextmanager
@@ -536,9 +534,7 @@ def forward(self, *inputs, **kwargs):
             and framework._dygraph_tracer()._has_grad
             and self.grad_need_sync
         ):
-            self._reducer.prepare_for_backward(
-                list(self._find_varbase(outputs))
-            )
+            self._reducer.prepare_for_backward(list(self._find_tensor(outputs)))
         return outputs
 
     @deprecated(
diff --git a/python/paddle/fluid/dygraph/base.py b/python/paddle/fluid/dygraph/base.py
index 825cb00f44c66..e76f0f3fa3161 100644
--- a/python/paddle/fluid/dygraph/base.py
+++ b/python/paddle/fluid/dygraph/base.py
@@ -125,7 +125,7 @@ def param_guard(parameters):
 
 def _convert_into_variable(tensor):
     """
-    Convert Varbase into Variable.
+    Convert Tensor into Variable.
     """
     if isinstance(tensor, core.eager.Tensor):
         # Check whether has been created before.
diff --git a/python/paddle/fluid/tests/unittests/test_eager_run_program.py b/python/paddle/fluid/tests/unittests/test_eager_run_program.py
index 416a692038c5e..3ee2aaa9da8f4 100644
--- a/python/paddle/fluid/tests/unittests/test_eager_run_program.py
+++ b/python/paddle/fluid/tests/unittests/test_eager_run_program.py
@@ -57,15 +57,14 @@ def _append_backward_desc(main_program, outs):
 def _create_out(var):
     assert isinstance(var, Variable)
     var_desc = var.desc
-    varbase = None
-    var_base = core.eager.Tensor(
+    out = core.eager.Tensor(
         var_desc.dtype(),
         var_desc.shape(),
         var_desc.name(),
         var_desc.type(),
         False,
     )
-    return var_base
+    return out
 
 
 @switch_to_static_graph
diff --git a/python/paddle/fluid/tests/unittests/test_imperative_group.py b/python/paddle/fluid/tests/unittests/test_imperative_group.py
index da0c5af627b25..2a1fdad88bb87 100644
--- a/python/paddle/fluid/tests/unittests/test_imperative_group.py
+++ b/python/paddle/fluid/tests/unittests/test_imperative_group.py
@@ -20,7 +20,7 @@
 
 
 class TestDataParallelGroup(unittest.TestCase):
-    def create_varbase(self, dtype, shape):
+    def _create_var(self, dtype, shape):
         return paddle.rand(shape=shape, dtype=dtype)
 
     def assign_group_by_size(self, *args):
@@ -30,10 +30,10 @@ def assign_group_by_size(self, *args):
     def test_construct_group0(self):
         # one dtype & one limit capability
         var_list = []
-        var_list.append(self.create_varbase("float32", [2, 50]))
-        var_list.append(self.create_varbase("float32", [2, 100]))
-        var_list.append(self.create_varbase("float32", [2, 50]))
-        var_list.append(self.create_varbase("float32", [2, 25]))
+        var_list.append(self._create_var("float32", [2, 50]))
+        var_list.append(self._create_var("float32", [2, 100]))
+        var_list.append(self._create_var("float32", [2, 50]))
+        var_list.append(self._create_var("float32", [2, 25]))
         res = self.assign_group_by_size(
             var_list, [False, False, False, False], [400]
         )
@@ -42,12 +42,12 @@ def test_construct_group0(self):
     def test_construct_group1(self):
         # multi dtype & one limit capability
         var_list = []
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
         res = self.assign_group_by_size(
             var_list, [False, False, False, False, False, False], [400]
         )
@@ -56,10 +56,10 @@ def test_construct_group1(self):
     def test_construct_group2(self):
         # one dtype & multi limit capability
         var_list = []
-        var_list.append(self.create_varbase("float32", [2, 50]))
-        var_list.append(self.create_varbase("float32", [2, 50]))
-        var_list.append(self.create_varbase("float32", [2, 50]))
-        var_list.append(self.create_varbase("float32", [2, 50]))
+        var_list.append(self._create_var("float32", [2, 50]))
+        var_list.append(self._create_var("float32", [2, 50]))
+        var_list.append(self._create_var("float32", [2, 50]))
+        var_list.append(self._create_var("float32", [2, 50]))
         res = self.assign_group_by_size(
             var_list, [False, False, False, False], [400, 800]
         )
@@ -68,12 +68,12 @@ def test_construct_group2(self):
     def test_construct_group3(self):
         # multi dtype & multi limit capability
         var_list = []
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
         res = self.assign_group_by_size(
             var_list, [False, False, False, False, False, False], [200, 400]
         )
@@ -82,12 +82,12 @@ def test_construct_group3(self):
     def test_construct_group4(self):
         # multi dtype & zero limit capability
         var_list = []
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
         res = self.assign_group_by_size(
             var_list, [False, False, False, False, False, False], [0]
         )
@@ -96,12 +96,12 @@ def test_construct_group4(self):
     def test_construct_group5(self):
         # multi dtype & infinite capability
         var_list = []
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
         res = self.assign_group_by_size(
             var_list, [False, False, False, False, False, False], [10000]
         )
@@ -111,16 +111,16 @@ def test_construct_group6(self):
         # multi dtype & limit capability & multi tensor type
         var_list = []
         var_list.append(
-            self.create_varbase(
+            self._create_var(
                 "float32",
                 [1, 50],
             )
         )
-        var_list.append(self.create_varbase("float64", [1, 25]))
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
+        var_list.append(self._create_var("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
         res = self.assign_group_by_size(
             var_list, [True, False, False, False, False, True], [400]
         )
@@ -129,12 +129,12 @@ def test_construct_group6(self):
     def test_construct_group7(self):
         # multi dtype & multi limit capability & multi tensor type
         var_list = []
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
-        var_list.append(self.create_varbase("float32", [1, 50]))
-        var_list.append(self.create_varbase("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
+        var_list.append(self._create_var("float32", [1, 50]))
+        var_list.append(self._create_var("float64", [1, 25]))
         res = self.assign_group_by_size(
             var_list, [True, False, False, False, False, True], [200, 400]
         )
@@ -143,10 +143,10 @@ def test_construct_group7(self):
     def test_construct_group8(self):
         # one dtype & one limit capability & have tensor_indices
         var_list = []
-        var_list.append(self.create_varbase("float32", [2, 25]))
-        var_list.append(self.create_varbase("float32", [2, 100]))
-        var_list.append(self.create_varbase("float32", [2, 50]))
-        var_list.append(self.create_varbase("float32", [2, 25]))
+        var_list.append(self._create_var("float32", [2, 25]))
+        var_list.append(self._create_var("float32", [2, 100]))
+        var_list.append(self._create_var("float32", [2, 50]))
+        var_list.append(self._create_var("float32", [2, 25]))
         res = self.assign_group_by_size(
             var_list, [False, False, False, False], [400], [3, 0, 1, 2]
         )
@@ -155,10 +155,10 @@ def test_construct_group8(self):
     def test_construct_group9(self):
         # one dtype & one limit capability & have tensor_indices
         var_list = []
-        var_list.append(self.create_varbase("float32", [2, 25]))
-        var_list.append(self.create_varbase("float32", [2, 25]))
-        var_list.append(self.create_varbase("float32", [2, 25]))
-        var_list.append(self.create_varbase("float32", [2, 1000]))
+        var_list.append(self._create_var("float32", [2, 25]))
+        var_list.append(self._create_var("float32", [2, 25]))
+        var_list.append(self._create_var("float32", [2, 25]))
+        var_list.append(self._create_var("float32", [2, 1000]))
         res = self.assign_group_by_size(
             var_list, [False, False, False, True], [300], [1, 0, 2, 3]
         )
diff --git a/python/paddle/fluid/tests/unittests/test_view_op_reuse_allocation.py b/python/paddle/fluid/tests/unittests/test_view_op_reuse_allocation.py
index ca261f63b0093..cee8294541089 100644
--- a/python/paddle/fluid/tests/unittests/test_view_op_reuse_allocation.py
+++ b/python/paddle/fluid/tests/unittests/test_view_op_reuse_allocation.py
@@ -21,8 +21,8 @@
 
 # NOTE(pangyoki): Tensor View Strategy.
 # Refer to `op_function_generator.py`.
-# For view op, a new output varbase will be created, and this varbase will
-# reuse the input varbase's allocation.
+# For view op, a new output tensor will be created, and this tensor will
+# reuse the input tensor's allocation.
 # View APIs include: `squeeze`, `unsqueeze`, `reshape`, `flatten`, `detach`
 class TestDygraphViewReuseAllocation(unittest.TestCase):
     def setUp(self):
diff --git a/python/paddle/jit/dy2static/partial_program.py b/python/paddle/jit/dy2static/partial_program.py
index 2a27c0eb15aa4..ad2e62b9e0461 100644
--- a/python/paddle/jit/dy2static/partial_program.py
+++ b/python/paddle/jit/dy2static/partial_program.py
@@ -913,27 +913,26 @@ def _prepare(self, inputs):
             input_vars.append(var)
 
         # mapping from name(string) -> Tensor
-        out_varbase_map = {}
+        out_tensor_map = {}
 
         def create_out(var_id):
             var = self._outputs[var_id]
             assert isinstance(var, framework.Variable)
             var_desc = var.desc
-            varbase = None
 
-            if var_desc.name() in out_varbase_map:
-                return out_varbase_map[var_desc.name()]
+            if var_desc.name() in out_tensor_map:
+                return out_tensor_map[var_desc.name()]
 
-            var_base = core.eager.Tensor(
+            out = core.eager.Tensor(
                 var_desc.dtype(),
                 var_desc.shape(),
                 var_desc.name(),
                 var_desc.type(),
                 False,
             )
-            var_base.stop_gradient = var.stop_gradient
-            out_varbase_map[var_desc.name()] = var_base
-            return var_base
+            out.stop_gradient = var.stop_gradient
+            out_tensor_map[var_desc.name()] = out
+            return out
 
         # Create Tensor to receive output data.
         out_vars = list(map(create_out, self._outputs.var_ids))
diff --git a/python/paddle/jit/translated_layer.py b/python/paddle/jit/translated_layer.py
index 9f2fe5ee41b0d..06a89ce36401a 100644
--- a/python/paddle/jit/translated_layer.py
+++ b/python/paddle/jit/translated_layer.py
@@ -644,7 +644,7 @@ def _append_backward_desc(self, infer_program_desc):
 # and executor, executes this program. Key points:
 #
 # 1. Data Sharing:
-#   The varBase of the dynamic graph is not in the scope, so before the op
+#   The variable/parameter of the dynamic graph is not in the scope, so before the op
 #   executes the program internally, create persistent variables with the
 #   same name as feed, parameters, and fetch in the scope, and share the
 #   LoDTensor of the op input.
@@ -668,7 +668,7 @@ def _load_persistable_vars_by_program(
     for each_var in persistable_vars:
         orig_each_name = program_holder._suffix_varname_dict[each_var.name()]
         if _is_parameter(each_var, program_holder.infer_program):
-            # create output varbase
+            # create output param
             new_var = framework.EagerParamBase(
                 shape=each_var.shape(),
                 dtype=each_var.dtype(),
@@ -755,7 +755,7 @@ def _load_persistable_vars(
             )
         # get suffix var name, see [why need to append suffix to persistable vars]
         new_name = inv_suffix_varname_dict[name]
-        # create output varbase
+        # create output var or param
         if extra_var_info[name].get('trainable', None) is not None:
             # use default shape and dtype
             new_var = framework.EagerParamBase(
diff --git a/test/dygraph_to_static/test_list.py b/test/dygraph_to_static/test_list.py
index 091d261ed7458..577669279d6b9 100644
--- a/test/dygraph_to_static/test_list.py
+++ b/test/dygraph_to_static/test_list.py
@@ -228,7 +228,7 @@ def init_dygraph_func(self):
             test_list_pop_without_control_flow_2,
         ]
 
-    def varbase_to_numpy(self, res):
+    def result_to_numpy(self, res):
         if isinstance(res, (list, tuple)):
             res = paddle.utils.map_structure(lambda x: x.numpy(), res)
         else:
@@ -248,7 +248,7 @@ def train(self, to_static=False):
                 res = paddle.jit.to_static(self.dygraph_func)(self.input)
             else:
                 res = self.dygraph_func(self.input)
-            return self.varbase_to_numpy(res)
+            return self.result_to_numpy(res)
 
     def test_transformed_static_result(self):
         for dyfunc in self.all_dygraph_funcs:
@@ -294,7 +294,7 @@ def train(self, to_static=False):
                 )
             else:
                 res = self.dygraph_func(self.input, self.iter_num)
-            return self.varbase_to_numpy(res)
+            return self.result_to_numpy(res)
 
 
 class TestListInWhileLoopWithStack(TestListInWhileLoop):

From 6adfcdf62eac32900b7a26091f228533d1ecb4ed Mon Sep 17 00:00:00 2001
From: zqw_1997 <118182234+zhengqiwen1997@users.noreply.github.com>
Date: Fri, 28 Apr 2023 16:09:04 +0800
Subject: [PATCH 141/405] [Zero-Dim] Support output 0D for squeeze, unbind,
 unstack. (#52843)

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* test=allcase

* fix test cases, test=allcase

* fix test cases, test=allcase

* modify the test_squeeze to not use Tensor type axis, test=allcase

* add grad check for unbind and unstack, test=allcase

* check for squeeze axis tensor type, test=allcase

* fix bug, test=allcase
---
 paddle/phi/infermeta/unary.cc                 |   3 +
 .../tests/unittests/test_zero_dim_tensor.py   | 142 ++++++++++++++++++
 2 files changed, 145 insertions(+)

diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc
index 08b2c830d195d..78830c682268c 100644
--- a/paddle/phi/infermeta/unary.cc
+++ b/paddle/phi/infermeta/unary.cc
@@ -3761,6 +3761,9 @@ void SqueezeInferMeta(const MetaTensor& x,
   if (!config.is_runtime && axes.FromTensor()) {
     // compile time infershape, set all elements to -1.
     int output_size = x.dims().size() - axes.GetData().size();
+    if (x.dims().size() == 0 && output_size == -1) {
+      output_size = 0;
+    }
     std::vector<int64_t> vec_out_dims(output_size, -1);
     out->set_dims(phi::make_ddim(vec_out_dims));
   } else {
diff --git a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
index 0e2449e7799c2..b08fe5c9b37d4 100644
--- a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
+++ b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
@@ -2401,6 +2401,56 @@ def test_upsample(self):
         self.assertEqual(out1.shape, [2, 3, 12, 12])
         self.assertEqual(input_x.grad.shape, [2, 3, 6, 6])
 
+    def test_unstack(self):
+        x1 = paddle.full([1], 0)
+        x2 = paddle.full([2], 2)
+        x1.retain_grads()
+        x2.retain_grads()
+        x1.stop_gradient = False
+        x2.stop_gradient = False
+
+        [out1] = paddle.unstack(x1, 0)
+        out1.retain_grads()
+        out1.backward()
+        [out2_1, out2_2] = paddle.unstack(x2, 0)
+        out2 = paddle.add_n([out2_1, out2_2])
+        out2.retain_grads()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out1.numpy(), 0)
+
+        self.assertEqual(out2_1.shape, [])
+        self.assertEqual(out2_1.numpy(), 2)
+        self.assertEqual(out2_2.shape, [])
+        self.assertEqual(out2_2.numpy(), 2)
+        self.assertEqual(x2.grad.shape, [2])
+
+    def test_unbind(self):
+        x1 = paddle.full([1], 0)
+        x2 = paddle.full([2], 2)
+        x1.retain_grads()
+        x2.retain_grads()
+        x1.stop_gradient = False
+        x2.stop_gradient = False
+
+        [out1] = paddle.unbind(x1, 0)
+        out1.retain_grads()
+        out1.backward()
+        [out2_1, out2_2] = paddle.unbind(x2, 0)
+        out2 = paddle.add_n([out2_1, out2_2])
+        out2.retain_grads()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out1.numpy(), 0)
+
+        self.assertEqual(out2_1.shape, [])
+        self.assertEqual(out2_1.numpy(), 2)
+        self.assertEqual(out2_2.shape, [])
+        self.assertEqual(out2_2.numpy(), 2)
+        self.assertEqual(x2.grad.shape, [2])
+
     def test_maseked_select(self):
         x = paddle.rand([])
         x.stop_gradient = False
@@ -2415,6 +2465,26 @@ def test_maseked_select(self):
         self.assertEqual(x.grad.shape, [])
         self.assertEqual(x.grad.numpy(), 1)
 
+    def test_squeeze(self):
+        x1 = paddle.full([], 2)
+        x1.stop_gradient = False
+        x1.retain_grads()
+        out1 = paddle.squeeze(x1, axis=0)
+        out1.retain_grads()
+        out1.backward()
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(x1.grad.shape, [])
+
+        x2 = paddle.full([], 3)
+        x3 = paddle.full([1], 0, dtype='int32')
+        x2.stop_gradient = False
+        x2.retain_grads()
+        out2 = paddle.squeeze(x2, axis=x3)
+        out2.retain_grads()
+        out2.backward()
+        self.assertEqual(out2.shape, [])
+        self.assertEqual(x2.grad.shape, [])
+
     def test_unsqueeze(self):
         x1 = paddle.full([], 2)
         x1.stop_gradient = False
@@ -4242,6 +4312,50 @@ def test_upsample(self):
         self.assertEqual(res1[0].shape, (2, 3, 12, 12))
         self.assertEqual(res1[1].shape, (2, 3, 6, 6))
 
+    @prog_scope()
+    def test_unstack(self):
+        x1 = paddle.full([1], 0, 'float32')
+        x1.stop_gradient = False
+        out1 = paddle.unstack(x1, 0)
+        out1 = paddle.add_n(out1)
+        paddle.static.append_backward(out1)
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, feed={}, fetch_list=[out1, x1.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (1,))
+
+        x2 = paddle.full([2], 2, 'float32')
+        x2.stop_gradient = False
+        out2 = paddle.unstack(x2, 0)
+        out2_sum = paddle.add_n(out2)
+        paddle.static.append_backward(out2_sum)
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, feed={}, fetch_list=[out2_sum, x2.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (2,))
+
+    @prog_scope()
+    def test_unbind(self):
+        x1 = paddle.full([1], 0, 'float32')
+        x1.stop_gradient = False
+        out1 = paddle.unbind(x1, 0)
+        out1 = paddle.add_n(out1)
+        paddle.static.append_backward(out1)
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, feed={}, fetch_list=[out1, x1.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (1,))
+
+        x2 = paddle.full([2], 2, 'float32')
+        x2.stop_gradient = False
+        out2 = paddle.unbind(x2, 0)
+        out2_sum = paddle.add_n(out2)
+        paddle.static.append_backward(out2_sum)
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, feed={}, fetch_list=[out2_sum, x2.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (2,))
+
     @prog_scope()
     def test_maseked_select(self):
         x = paddle.rand([])
@@ -4258,6 +4372,34 @@ def test_maseked_select(self):
         self.assertEqual(res[3].shape, ())
         self.assertEqual(res[3], 1)
 
+    @prog_scope()
+    def test_squeeze(self):
+        x1 = paddle.full([], 2)
+        x1.stop_gradient = False
+        out1 = paddle.squeeze(x1, axis=0)
+        paddle.static.append_backward(out1.sum())
+
+        x2 = paddle.full([], 3)
+        x3 = paddle.full([], 0, dtype='int32')
+        x2.stop_gradient = False
+        out2 = paddle.squeeze(x2, axis=x3)
+        paddle.static.append_backward(out2.sum())
+
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(
+            prog,
+            fetch_list=[
+                out1,
+                out2,
+                x1.grad_name,
+                x2.grad_name,
+            ],
+        )
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, ())
+        self.assertEqual(res[2].shape, ())
+        self.assertEqual(res[3].shape, ())
+
     @prog_scope()
     def test_unsqueeze(self):
         x1 = paddle.full([], 2)

From ef6e8d09300b24bce7812f846640e8bcfda58b9c Mon Sep 17 00:00:00 2001
From: GGBond8488 <33050871+GGBond8488@users.noreply.github.com>
Date: Fri, 28 Apr 2023 16:17:30 +0800
Subject: [PATCH 142/405] =?UTF-8?q?=E3=80=900D=20output=E3=80=91add=5F0D?=
 =?UTF-8?q?=5Foutput=5Fsupport=20(#52857)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add 0d support for dist, trace, paddle.linalg.cond test=allcase

* add_0d_output_support_for_det

* test=allcase

* support_0d_output_for_linalg.norm

* support linalg.norm 0d output, test=allcase

* fix 0D test

* fix zero dim test, test=allcase

* fix 0D test

* fix tets,test=allcase

* fix error,test=allcase

* fix errors ,test=allcase

* add static backward , test=allcase

* add static backwward test, test=allcase

* fix pr-ci-build error;test=document_fix (#53060)

* [Cherry-Pick] Unique support float16&bfloat16 (#53023)

unique支持float16和bfloat16数据类型，并完善相关单测。

* slogdet_support_0D_output

* add new case

* fix tests, test=allcase

* fix p_norm related test, test=allcase

* fix some err, test=allcase

* test=allcase

* move out trace

* open some case, test=allcase

* fix norm all case, test=allcase

* fix some test error, test=allcase

* fix typro,test=allcase

* fix test err, test=allcase

* test=allcase

* test

* fix test error, test=allcase

* fix test error, test=allcase

* fallback norm, test=allcase

---------

Co-authored-by: tianshuo78520a <707759223@qq.com>
Co-authored-by: Zhang Zheng <32410583+ZzSean@users.noreply.github.com>
---
 paddle/phi/infermeta/binary.cc                |   2 +-
 paddle/phi/infermeta/unary.cc                 |   1 -
 .../impl/determinant_grad_kernel_impl.h       |   4 +-
 .../kernels/impl/determinant_kernel_impl.h    |   2 +-
 .../tests/unittests/test_zero_dim_tensor.py   | 243 ++++++++++++++++++
 test/autograd/test_orig2prim.py               |   7 +-
 6 files changed, 253 insertions(+), 6 deletions(-)

diff --git a/paddle/phi/infermeta/binary.cc b/paddle/phi/infermeta/binary.cc
index 228d50f3c4a9e..efb6458601ba9 100644
--- a/paddle/phi/infermeta/binary.cc
+++ b/paddle/phi/infermeta/binary.cc
@@ -990,7 +990,7 @@ void DistInferMeta(const MetaTensor& x,
                         "The Input(Y) has not been initialized properly. The "
                         "shape of Input(Y) = [%s].",
                         y_dims));
-  out->set_dims({1});
+  out->set_dims(phi::make_ddim({}));
   out->set_dtype(x.dtype());
 }
 
diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc
index 78830c682268c..845fe95563900 100644
--- a/paddle/phi/infermeta/unary.cc
+++ b/paddle/phi/infermeta/unary.cc
@@ -2767,7 +2767,6 @@ void PNormInferMeta(const MetaTensor& x,
     if (reduce_dims.size() == 0) {
       reduce_dims.emplace_back(1);
     }
-
     x_dim[axis] = 1;
   }
 
diff --git a/paddle/phi/kernels/impl/determinant_grad_kernel_impl.h b/paddle/phi/kernels/impl/determinant_grad_kernel_impl.h
index 4d58698c64d22..b17512ad1da87 100644
--- a/paddle/phi/kernels/impl/determinant_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/determinant_grad_kernel_impl.h
@@ -91,10 +91,10 @@ void DeterminantGradKernel(const Context& dev_ctx,
             " input tensor's, but here differ %d",
             input_dims_size - out_grad.dims().size()));
   } else if (input_dims_size == 2) {
-    // input dims size 2 and grad dims size 1 is possible
+    // input dims size 2 and grad dims size 0 is possible
     PADDLE_ENFORCE_EQ(
         out_grad.dims().size(),
-        1,
+        0,
         phi::errors::InvalidArgument(
             "The grad tensor of det dims size should be 2 less than"
             " input tensor's, but here differ %d",
diff --git a/paddle/phi/kernels/impl/determinant_kernel_impl.h b/paddle/phi/kernels/impl/determinant_kernel_impl.h
index 3c437ad659c43..01c54d780b4b0 100644
--- a/paddle/phi/kernels/impl/determinant_kernel_impl.h
+++ b/paddle/phi/kernels/impl/determinant_kernel_impl.h
@@ -126,7 +126,7 @@ void DeterminantKernel(const Context& dev_ctx,
     out->Resize(output_dims);
   } else {
     // when input is a two-dimension matrix, The det value is a number.
-    out->Resize({1});
+    out->Resize(phi::make_ddim({}));
   }
   VLOG(10) << "output dim:" << out->dims();
 }
diff --git a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
index b08fe5c9b37d4..bc8a7b82c5bc8 100644
--- a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
+++ b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
@@ -2616,6 +2616,107 @@ def test_multi_dot(self):
         self.assertEqual(b.grad.shape, [4, 5])
         self.assertEqual(c.grad.shape, [5])
 
+    def test_cov(self):
+        xt = paddle.randn((3, 4))
+        xt.stop_gradient = False
+        xt_1 = paddle.randn((12,))
+        xt_1.stop_gradient = False
+
+        xt_out = paddle.linalg.cov(xt)
+        xt_out.retain_grads()
+        xt_out.backward()
+        self.assertEqual(xt_out.shape, [3, 3])
+        self.assertEqual(xt.grad.shape, [3, 4])
+
+        xt_1_out = paddle.linalg.cov(xt_1)
+        xt_1.retain_grads()
+        xt_1_out.backward()
+        self.assertEqual(xt_1_out.shape, [])
+        self.assertEqual(xt_1.grad.shape, [12])
+
+    def test_det(self):
+        xt = paddle.randn([3, 3, 3])
+        xt.stop_gradient = False
+        xt_1 = paddle.randn([3, 3])
+        xt_1.stop_gradient = False
+
+        xt_out = paddle.linalg.det(xt)
+        xt.retain_grads()
+        xt_out.backward()
+        self.assertEqual(xt_out.shape, [3])
+        self.assertEqual(xt.grad.shape, [3, 3, 3])
+
+        xt_1_out = paddle.linalg.det(xt_1)
+        xt_1.retain_grads()
+        xt_1_out.backward()
+        self.assertEqual(xt_1_out.shape, [])
+        self.assertEqual(xt_1.grad.shape, [3, 3])
+
+    def test_dist(self):
+        x = paddle.to_tensor([[3, 3], [3, 3]], dtype="float32")
+        y = paddle.to_tensor([[3, 3], [3, 1]], dtype="float32")
+        x.stop_gradient = False
+        y.stop_gradient = False
+        out = paddle.dist(x, y, 0)
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        np.testing.assert_allclose(out, np.array(1))
+        self.assertEqual(x.grad.shape, [2, 2])
+        self.assertEqual(y.grad.shape, [2, 2])
+
+    def test_linalg_cond(self):
+        def assert_shape(out):
+            self.assertEqual(out.shape, [])
+
+        # x1 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x1.stop_gradient = False
+        # p = 2 : use paddle.sum
+        # out = paddle.linalg.cond(x1)
+        # assert_shape(out)
+
+        # p = fro : use paddle.sum
+        # x2 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x2.stop_gradient = False
+        # out_fro = paddle.linalg.cond(x2, p='fro')
+        # assert_shape(out_fro)
+
+        # p = nuc : use paddle.sum
+        # x3 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x3.stop_gradient = False
+        # out_nuc = paddle.linalg.cond(x3, p='nuc')
+        # assert_shape(out_nuc)
+
+        # p in (-1, 1) : use paddle.sum
+        # x4 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x4.stop_gradient = False
+        # out_1 = paddle.linalg.cond(x4, p=1)
+        # assert_shape(out_1)
+        # x5 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x5.stop_gradient = False
+        # out_minus_1 = paddle.linalg.cond(x5, p=-1)
+        # assert_shape(out_minus_1)
+
+        # p in (-2, 2)  depends on paddle.sum
+        # x6 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x6.stop_gradient = False
+        # out_2 = paddle.linalg.cond(x6, p=2)
+        # assert_shape(out_2)
+
+        # p in (-inf, inf):use paddle.sum
+        # x8 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x8.stop_gradient = False
+        # out_inf = paddle.linalg.cond(x8, p=float("inf"))
+        # assert_shape(out_inf)
+
+        # depends on paddle.sum
+        # a = paddle.randn([2, 4, 4])
+        # a.stop_gradient = False
+        # a_cond_fro = paddle.linalg.cond(a, p='fro')
+        # a_cond_fro.backward()
+        # self.assertEqual(len(a_cond_fro.shape), 1)
+        # self.assertEqual(a.grad.shape, [2, 4, 4])
+
     def test_trace(self):
         x = paddle.to_tensor([[3, 2], [1, 9]], dtype="float32")
         x.stop_gradient = False
@@ -4723,6 +4824,148 @@ def test_multi_dot(self):
         self.assertEqual(res[2].shape, (4, 5))
         self.assertEqual(res[3].shape, (5,))
 
+    @prog_scope()
+    def test_cov(self):
+        xt_1 = paddle.randn((12,))
+        xt_1.stop_gradient = False
+
+        out = paddle.linalg.cov(xt_1)
+        paddle.static.append_backward(out)
+
+        prog = paddle.static.default_main_program()
+
+        res = self.exe.run(prog, fetch_list=[out, xt_1.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (12,))
+
+    @prog_scope()
+    def test_det(self):
+        xt_1 = paddle.randn((3, 3))
+        xt_1.stop_gradient = False
+
+        out = paddle.linalg.det(xt_1)
+        paddle.static.append_backward(out.sum())
+
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out, xt_1.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (3, 3))
+
+    @prog_scope()
+    def test_dist(self):
+        x = paddle.to_tensor([[3, 3], [3, 3]], dtype="float32")
+        y = paddle.to_tensor([[3, 3], [3, 1]], dtype="float32")
+        x.stop_gradient = False
+        y.stop_gradient = False
+        out = paddle.dist(x, y)
+        paddle.static.append_backward(out)
+
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out, x.grad_name, y.grad_name])
+
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (2, 2))
+        self.assertEqual(res[1].shape, (2, 2))
+        np.testing.assert_array_equal(res[0], np.array(2).astype(np.float32))
+
+    @prog_scope()
+    def test_linalg_cond(self):
+        pass
+        # use paddle.sum
+        # x = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x.stop_gradient = False
+        # out = paddle.linalg.cond(x)
+        # paddle.static.append_backward(out)
+
+        # prog = paddle.static.default_main_program()
+        # res = self.exe.run(prog, fetch_list=[out, x.grad_name])
+
+        # self.assertTrue(res[0].shape, ())
+        # self.assertTrue(res[1].shape, (3, 3))
+
+        # p = fro : use paddle.sum
+        # x2 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x2.stop_gradient = False
+        # out_fro = paddle.linalg.cond(x2, p='fro')
+        # paddle.static.append_backward(out_fro)
+
+        # prog = paddle.static.default_main_program()
+        # res = self.exe.run(prog, fetch_list=[out_fro, x.grad_name])
+
+        # self.assertTrue(res[0].shape, ())
+        # self.assertTrue(res[1].shape, (3, 3))
+
+        # p = nuc : use paddle.sum
+        # x3 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x3.stop_gradient = False
+        # out_nuc = paddle.linalg.cond(x3, p='nuc')
+        # paddle.static.append_backward(out_nuc)
+
+        # prog = paddle.static.default_main_program()
+        # res = self.exe.run(prog, fetch_list=[out_nuc, x.grad_name])
+
+        # self.assertTrue(res[0].shape, ())
+        # self.assertTrue(res[1].shape, (3, 3))
+
+        # p in (-1, 1) : use paddle.sum
+        # x4 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x4.stop_gradient = False
+        # out_1 = paddle.linalg.cond(x4, p=1)
+        # paddle.static.append_backward(out_1)
+
+        # prog = paddle.static.default_main_program()
+        # res = self.exe.run(prog, fetch_list=[out_1, x.grad_name])
+
+        # self.assertTrue(res[0].shape, ())
+        # self.assertTrue(res[1].shape, (3, 3))
+
+        # x5 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x5.stop_gradient = False
+        # out_minus_1 = paddle.linalg.cond(x5, p=-1)
+        # paddle.static.append_backward(out_minus_1)
+
+        # prog = paddle.static.default_main_program()
+        # res = self.exe.run(prog, fetch_list=[out_minus_1, x.grad_name])
+
+        # self.assertTrue(res[0].shape, ())
+        # self.assertTrue(res[1].shape, (3, 3))
+
+        # p in (-2, 2)  depends on paddle.sum
+        # x6 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x6.stop_gradient = False
+        # out_2 = paddle.linalg.cond(x6, p=2)
+        # paddle.static.append_backward(out_2)
+
+        # prog = paddle.static.default_main_program()
+        # res = self.exe.run(prog, fetch_list=[out_2, x.grad_name])
+
+        # self.assertTrue(res[0].shape, ())
+        # self.assertTrue(res[1].shape, (3, 3))
+
+        # p in (-inf, inf):use paddle.sum
+        # x8 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        # x8.stop_gradient = False
+        # out_inf = paddle.linalg.cond(x8, p=float("inf"))
+        # paddle.static.append_backward(out_inf)
+
+        # prog = paddle.static.default_main_program()
+        # res = self.exe.run(prog, fetch_list=[out_inf, x.grad_name])
+
+        # self.assertTrue(res[0].shape, ())
+        # self.assertTrue(res[1].shape, (3, 3))
+
+        # depends on paddle.sum
+        # a = paddle.randn([2, 4, 4])
+        # a.stop_gradient = False
+        # a_cond_fro = paddle.linalg.cond(a, p='fro')
+        # paddle.static.append_backward(a_cond_fro)
+
+        # prog = paddle.static.default_main_program()
+        # res = self.exe.run(prog, fetch_list=[a_cond_fro, a.grad_name])
+
+        # self.assertEqual(res[0].shape, (2,))
+        # self.assertEqual(res[1].shape, (2, 4, 4))
+
     @prog_scope()
     def test_trace(self):
         x = paddle.to_tensor([[3, 2], [1, 9]], dtype="float32")
diff --git a/test/autograd/test_orig2prim.py b/test/autograd/test_orig2prim.py
index 2f8f2a9648bfd..8a42255d711ee 100644
--- a/test/autograd/test_orig2prim.py
+++ b/test/autograd/test_orig2prim.py
@@ -463,7 +463,12 @@ def init_data(self):
         }
 
         self.orig2prim_args = (X,)
-        self.all_ops = ['p_norm', 'reshape_p', 'abs_p', 'reduce_sum_p']
+        self.all_ops = [
+            'p_norm',
+            'reshape_p',
+            'abs_p',
+            'reduce_sum_p',
+        ]
         self.out_map = {0: self.output['Out']}
 
 

From a53ee944e26f93fe473fdf0005bba56fe161ca37 Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Fri, 28 Apr 2023 17:22:29 +0800
Subject: [PATCH 143/405] [test]mv fluid op cinn to test/cpp/fluid/cinn
 (#53443)

---
 paddle/fluid/operators/cinn/CMakeLists.txt    | 50 -------------------
 test/cpp/CMakeLists.txt                       |  1 +
 test/cpp/fluid/CMakeLists.txt                 |  3 ++
 test/cpp/fluid/cinn/CMakeLists.txt            | 46 +++++++++++++++++
 .../cinn/cinn_instruction_run_op_test.cc      |  2 +-
 .../fluid}/cinn/cinn_launch_context_test.cc   |  0
 .../cpp/fluid}/cinn/cinn_launch_op_test.cc    |  2 +-
 .../cpp/fluid}/cinn/test_helper.h             |  0
 8 files changed, 52 insertions(+), 52 deletions(-)
 create mode 100644 test/cpp/fluid/CMakeLists.txt
 create mode 100644 test/cpp/fluid/cinn/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/cinn/cinn_instruction_run_op_test.cc (98%)
 rename {paddle/fluid/operators => test/cpp/fluid}/cinn/cinn_launch_context_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/cinn/cinn_launch_op_test.cc (99%)
 rename {paddle/fluid/operators => test/cpp/fluid}/cinn/test_helper.h (100%)

diff --git a/paddle/fluid/operators/cinn/CMakeLists.txt b/paddle/fluid/operators/cinn/CMakeLists.txt
index 3a97e275689ed..b700b2798fcd8 100644
--- a/paddle/fluid/operators/cinn/CMakeLists.txt
+++ b/paddle/fluid/operators/cinn/CMakeLists.txt
@@ -29,53 +29,3 @@ set(CINN_OP_DEPS
     cinn_launch_context
     transform_type)
 register_operators(DEPS ${CINN_OP_DEPS})
-
-if(WITH_TESTING)
-  cc_test_old(
-    cinn_launch_context_test
-    SRCS
-    cinn_launch_context_test.cc
-    DEPS
-    ddim
-    lod_tensor
-    scope
-    proto_desc
-    graph
-    cinn_launch_context
-    cinn_instruction_run_op
-    cinn)
-  target_link_libraries(cinn_launch_context_test ${PYTHON_LIBRARIES})
-  set_tests_properties(cinn_launch_context_test PROPERTIES LABELS
-                                                           "RUN_TYPE=CINN")
-
-  set(CINN_RUN_ENVIRONMENT
-      "OMP_NUM_THREADS=1;runtime_include_dir=${PADDLE_BINARY_DIR}/third_party/CINN/src/external_cinn/cinn/runtime/cuda"
-  )
-  # cc_test_old(
-  #   cinn_launch_op_test
-  #   SRCS
-  #   cinn_launch_op_test.cc
-  #   DEPS
-  #   cinn_compiler
-  #   cinn_launch_op
-  #   cinn_instruction_run_op
-  #   elementwise_add_op
-  #   gflags)
-  # set_tests_properties(
-  #   cinn_launch_op_test PROPERTIES LABELS "RUN_TYPE=CINN" ENVIRONMENT
-  #                                  "${CINN_RUN_ENVIRONMENT}")
-
-  cc_test_old(
-    cinn_instruction_run_op_test
-    SRCS
-    cinn_instruction_run_op_test.cc
-    DEPS
-    cinn_compiler
-    cinn_launch_op
-    cinn_instruction_run_op
-    elementwise_add_op)
-  target_link_libraries(cinn_instruction_run_op_test ${PYTHON_LIBRARIES})
-  set_tests_properties(
-    cinn_instruction_run_op_test PROPERTIES LABELS "RUN_TYPE=CINN" ENVIRONMENT
-                                            "${CINN_RUN_ENVIRONMENT}")
-endif()
diff --git a/test/cpp/CMakeLists.txt b/test/cpp/CMakeLists.txt
index 2f395cb62ed65..60bdac586c49b 100644
--- a/test/cpp/CMakeLists.txt
+++ b/test/cpp/CMakeLists.txt
@@ -6,3 +6,4 @@ add_subdirectory(imperative)
 add_subdirectory(ir)
 add_subdirectory(inference)
 add_subdirectory(eager)
+add_subdirectory(fluid)
diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt
new file mode 100644
index 0000000000000..e2d08eac64b73
--- /dev/null
+++ b/test/cpp/fluid/CMakeLists.txt
@@ -0,0 +1,3 @@
+if(WITH_CINN)
+  add_subdirectory(cinn)
+endif()
diff --git a/test/cpp/fluid/cinn/CMakeLists.txt b/test/cpp/fluid/cinn/CMakeLists.txt
new file mode 100644
index 0000000000000..f396d1c58cccc
--- /dev/null
+++ b/test/cpp/fluid/cinn/CMakeLists.txt
@@ -0,0 +1,46 @@
+cc_test_old(
+  cinn_launch_context_test
+  SRCS
+  cinn_launch_context_test.cc
+  DEPS
+  ddim
+  lod_tensor
+  scope
+  proto_desc
+  graph
+  cinn_launch_context
+  cinn_instruction_run_op
+  cinn)
+target_link_libraries(cinn_launch_context_test ${PYTHON_LIBRARIES})
+set_tests_properties(cinn_launch_context_test PROPERTIES LABELS "RUN_TYPE=CINN")
+
+set(CINN_RUN_ENVIRONMENT
+    "OMP_NUM_THREADS=1;runtime_include_dir=${PADDLE_BINARY_DIR}/third_party/CINN/src/external_cinn/cinn/runtime/cuda"
+)
+# cc_test_old(
+#   cinn_launch_op_test
+#   SRCS
+#   cinn_launch_op_test.cc
+#   DEPS
+#   cinn_compiler
+#   cinn_launch_op
+#   cinn_instruction_run_op
+#   elementwise_add_op
+#   gflags)
+# set_tests_properties(
+#   cinn_launch_op_test PROPERTIES LABELS "RUN_TYPE=CINN" ENVIRONMENT
+#                                  "${CINN_RUN_ENVIRONMENT}")
+
+cc_test_old(
+  cinn_instruction_run_op_test
+  SRCS
+  cinn_instruction_run_op_test.cc
+  DEPS
+  cinn_compiler
+  cinn_launch_op
+  cinn_instruction_run_op
+  elementwise_add_op)
+target_link_libraries(cinn_instruction_run_op_test ${PYTHON_LIBRARIES})
+set_tests_properties(
+  cinn_instruction_run_op_test PROPERTIES LABELS "RUN_TYPE=CINN" ENVIRONMENT
+                                          "${CINN_RUN_ENVIRONMENT}")
diff --git a/paddle/fluid/operators/cinn/cinn_instruction_run_op_test.cc b/test/cpp/fluid/cinn/cinn_instruction_run_op_test.cc
similarity index 98%
rename from paddle/fluid/operators/cinn/cinn_instruction_run_op_test.cc
rename to test/cpp/fluid/cinn/cinn_instruction_run_op_test.cc
index 8b6fa57c5d2b6..36086a0bd1fab 100644
--- a/paddle/fluid/operators/cinn/cinn_instruction_run_op_test.cc
+++ b/test/cpp/fluid/cinn/cinn_instruction_run_op_test.cc
@@ -20,11 +20,11 @@ limitations under the License. */
 #include "paddle/fluid/framework/op_registry.h"
 #include "paddle/fluid/framework/paddle2cinn/cinn_compiler.h"
 #include "paddle/fluid/framework/scope.h"
-#include "paddle/fluid/operators/cinn/test_helper.h"
 #include "paddle/fluid/platform/cpu_helper.h"
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/platform/init.h"
 #include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/fluid/cinn/test_helper.h"
 
 USE_OP_ITSELF(cinn_launch);
 USE_OP_ITSELF(cinn_instruction_run);
diff --git a/paddle/fluid/operators/cinn/cinn_launch_context_test.cc b/test/cpp/fluid/cinn/cinn_launch_context_test.cc
similarity index 100%
rename from paddle/fluid/operators/cinn/cinn_launch_context_test.cc
rename to test/cpp/fluid/cinn/cinn_launch_context_test.cc
diff --git a/paddle/fluid/operators/cinn/cinn_launch_op_test.cc b/test/cpp/fluid/cinn/cinn_launch_op_test.cc
similarity index 99%
rename from paddle/fluid/operators/cinn/cinn_launch_op_test.cc
rename to test/cpp/fluid/cinn/cinn_launch_op_test.cc
index 9764c31200f9e..5765a2c50269e 100644
--- a/paddle/fluid/operators/cinn/cinn_launch_op_test.cc
+++ b/test/cpp/fluid/cinn/cinn_launch_op_test.cc
@@ -24,12 +24,12 @@ limitations under the License. */
 #include "paddle/fluid/framework/op_registry.h"
 #include "paddle/fluid/framework/paddle2cinn/cinn_compiler.h"
 #include "paddle/fluid/framework/scope.h"
-#include "paddle/fluid/operators/cinn/test_helper.h"
 #include "paddle/fluid/platform/cpu_helper.h"
 #include "paddle/fluid/platform/init.h"
 #include "paddle/phi/core/ddim.h"
 #include "paddle/phi/core/flags.h"
 #include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/fluid/cinn/test_helper.h"
 
 USE_OP_ITSELF(cinn_launch);
 USE_OP_ITSELF(cinn_instruction_run);
diff --git a/paddle/fluid/operators/cinn/test_helper.h b/test/cpp/fluid/cinn/test_helper.h
similarity index 100%
rename from paddle/fluid/operators/cinn/test_helper.h
rename to test/cpp/fluid/cinn/test_helper.h

From d611e48c90d1a9145f97956ca2e5faea7a4a16bd Mon Sep 17 00:00:00 2001
From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com>
Date: Fri, 28 Apr 2023 18:45:44 +0800
Subject: [PATCH 144/405] Dropout optimize & clean broadcast inT and
 ElementwiseType (#52969)

* change judgement for DropoutGradGPUKernelDriver

* add UnrollerWithoutVecSize and after this Loaddata to be refined

* pass unittest

* use same unroller with XPU

* BroadcastWithInt64Index

* BroadcastDataLoader template partial specialization

* fix compile errs in ROCms

* clean ElementwiseT and InT for BroadcastKernel

* default axis and clean inT

* remove redundant fast divmod computation

* optimize drop_nd & drop_nd_grad

* optimize BroadcastDataLoader bf16 fp16

* rm InT etc. after merge develop

* delete constexpr for windows ci

* fix conflict

* fix conflic with develop

* fix conflic

* new clean

* clean
---
 .../elementwise/elementwise_op_broadcast.cu.h |  14 +--
 .../elementwise/elementwise_op_function.h     |  11 +-
 .../elementwise/elementwise_op_impl.cu.h      |   2 -
 paddle/fluid/operators/fused/attn_gemm.h      |   4 +-
 paddle/fluid/operators/fused/attn_gemm_int8.h |   8 +-
 paddle/fluid/operators/fused/fmha_ref.h       |  22 ++--
 .../operators/fused/fused_gate_attention.h    |   8 +-
 .../fluid/operators/fused_token_prune_op.cu   |   3 +-
 paddle/fluid/operators/reduce_ops/reduce_op.h |  11 +-
 paddle/phi/kernels/cpu/bitwise_kernel.cc      |  18 ++--
 paddle/phi/kernels/cpu/compare_kernel.cc      |   6 +-
 paddle/phi/kernels/cpu/dirichlet_kernel.cc    |   4 +-
 .../kernels/cpu/elementwise_divide_kernel.cc  |   4 +-
 paddle/phi/kernels/cpu/elementwise_kernel.cc  |   2 +-
 .../phi/kernels/cpu/layer_norm_grad_kernel.cc |  80 +++++---------
 paddle/phi/kernels/cpu/layer_norm_kernel.cc   |  22 ++--
 paddle/phi/kernels/cpu/logical_kernel.cc      |   2 +-
 .../phi/kernels/cpu/matrix_rank_tol_kernel.cc |  11 +-
 paddle/phi/kernels/funcs/broadcast_function.h | 101 +++++++++---------
 paddle/phi/kernels/funcs/dropout_impl.cu.h    |  93 ++++------------
 paddle/phi/kernels/funcs/elementwise_base.h   |   5 +-
 paddle/phi/kernels/fusion/gpu/attn_gemm.h     |   4 +-
 paddle/phi/kernels/fusion/gpu/fmha_ref.h      |  22 ++--
 paddle/phi/kernels/gpu/dirichlet_kernel.cu    |   4 +-
 .../gpu/elementwise_divide_grad_kernel.cu     |  21 ++--
 paddle/phi/kernels/gpu/elementwise_grad.h     |  51 +++++----
 .../kernels/gpu/elementwise_grad_kernel.cu    |  42 ++++----
 paddle/phi/kernels/gpu/expand_as_kernel.cu    |   3 +-
 paddle/phi/kernels/gpu/expand_kernel.cu       |   3 +-
 .../phi/kernels/gpu/matrix_rank_tol_kernel.cu |   5 +-
 .../phi/kernels/gpu/reduce_amin_amax_common.h |  12 +--
 paddle/phi/kernels/gpu/reduce_grad.h          |  16 ++-
 .../phi/kernels/gpu/reduce_max_grad_kernel.cu |   8 +-
 .../kernels/gpu/reduce_mean_grad_kernel.cu    |   4 +-
 .../phi/kernels/gpu/reduce_min_grad_kernel.cu |   8 +-
 .../phi/kernels/gpu/reduce_sum_grad_kernel.cu |   2 +-
 .../gpu/squared_l2_norm_grad_kernel.cu        |   3 +-
 paddle/phi/kernels/gpu/tile_kernel.cu         |   8 +-
 .../phi/kernels/gpu/viterbi_decode_kernel.cu  |   3 +-
 paddle/phi/kernels/impl/complex_kernel_impl.h |   6 +-
 .../impl/elementwise_grad_kernel_impl.h       |   8 +-
 .../kernels/impl/elementwise_kernel_impl.h    |  16 +--
 paddle/phi/kernels/impl/lu_kernel_impl.h      |  12 +--
 .../phi/kernels/impl/set_value_kernel_impl.h  |   2 -
 paddle/phi/kernels/kps/bitwise_kernel.cu      |  23 ++--
 paddle/phi/kernels/kps/compare_kernel.cu      |   3 +-
 paddle/phi/kernels/kps/elementwise_kernel.cu  |   4 +-
 paddle/phi/kernels/kps/logical_kernel.cu      |  31 +++---
 .../kernels/legacy/cpu/elementwise_kernel.cc  |  16 +--
 .../legacy/kps/elementwise_raw_kernel.cu      |  21 ++--
 .../cpp/phi/kernels/test_ternary_broadcast.cu |   3 +-
 51 files changed, 334 insertions(+), 461 deletions(-)

diff --git a/paddle/fluid/operators/elementwise/elementwise_op_broadcast.cu.h b/paddle/fluid/operators/elementwise/elementwise_op_broadcast.cu.h
index 7ca62b1969e26..94ce32cdaf182 100644
--- a/paddle/fluid/operators/elementwise/elementwise_op_broadcast.cu.h
+++ b/paddle/fluid/operators/elementwise/elementwise_op_broadcast.cu.h
@@ -19,17 +19,13 @@
 namespace paddle {
 namespace operators {
 
-template <ElementwiseType ET,
-          typename InT,
-          typename OutT,
-          typename Functor,
-          int NumOuts = 1>
+template <typename OutT, typename Functor, int NumOuts = 1>
 void LaunchElementwiseCudaKernel(
     const KPDevice &ctx,
     const std::vector<const phi::DenseTensor *> &ins,
     std::vector<phi::DenseTensor *> *outs,
-    int axis,
-    Functor func) {
+    Functor func,
+    int axis = -1) {
   std::vector<const phi::DenseTensor *> pt_inputs;
   std::vector<phi::DenseTensor *> pt_outputs;
   // TODO(YuanRisheng) *_tmp for cache DenseTensor, because the temporary
@@ -53,8 +49,8 @@ void LaunchElementwiseCudaKernel(
   for (int i = 0; i < pt_outputs_tmp.size(); i++) {
     pt_outputs.push_back(pt_outputs_tmp[i].get());
   }
-  phi::funcs::BroadcastKernel<ET, InT, OutT, Functor, NumOuts>(
-      ctx, pt_inputs, &pt_outputs, axis, func);
+  phi::funcs::BroadcastKernel<OutT, Functor, NumOuts>(
+      ctx, pt_inputs, &pt_outputs, func, axis);
 }
 
 }  // namespace operators
diff --git a/paddle/fluid/operators/elementwise/elementwise_op_function.h b/paddle/fluid/operators/elementwise/elementwise_op_function.h
index b1d5f13bf858b..c69acb89750c9 100644
--- a/paddle/fluid/operators/elementwise/elementwise_op_function.h
+++ b/paddle/fluid/operators/elementwise/elementwise_op_function.h
@@ -188,7 +188,7 @@ void ElementwiseComputeEx(const framework::ExecutionContext &ctx,
   z->mutable_data<OutType>(ctx.GetPlace());
   const auto &dev_ctx = ctx.template device_context<DeviceContext>();
   phi::funcs::ElementwiseCompute<Functor, T, OutType>(
-      dev_ctx, *x, *y, axis, func, z);
+      dev_ctx, *x, *y, func, z, axis);
 }
 
 // FusedElemwiseAndAct
@@ -1596,7 +1596,7 @@ static inline std::vector<int> GetReduceDim(const framework::DDim &in,
 
 #if defined(__NVCC__) || defined(__HIPCC__)
 
-template <ElementwiseType ET, typename T, typename Functor>
+template <typename T, typename Functor>
 void GetGradXAndYOut(const phi::GPUContext &dev_ctx,
                      const platform::Place &place,
                      int axis,
@@ -1605,11 +1605,11 @@ void GetGradXAndYOut(const phi::GPUContext &dev_ctx,
                      phi::DenseTensor *dx,
                      phi::DenseTensor *dy,
                      Functor func) {
-  phi::GetGradXAndYOut<ET, T, Functor>(
+  phi::GetGradXAndYOut<T, Functor>(
       dev_ctx, place, axis, ins, *dout, dx, dy, func);
 }
 
-template <ElementwiseType ET, typename T, typename Functor>
+template <typename T, typename Functor>
 void GetGradXOrYOut(const phi::GPUContext &dev_ctx,
                     const platform::Place &place,
                     int axis,
@@ -1617,8 +1617,7 @@ void GetGradXOrYOut(const phi::GPUContext &dev_ctx,
                     const phi::DenseTensor *dout,
                     phi::DenseTensor *dxy,
                     Functor func) {
-  phi::GetGradXOrYOut<ET, T, Functor>(
-      dev_ctx, place, axis, ins, *dout, dxy, func);
+  phi::GetGradXOrYOut<T, Functor>(dev_ctx, place, axis, ins, *dout, dxy, func);
 }
 
 #endif
diff --git a/paddle/fluid/operators/elementwise/elementwise_op_impl.cu.h b/paddle/fluid/operators/elementwise/elementwise_op_impl.cu.h
index 2680c8ecc5e52..47e317831409a 100644
--- a/paddle/fluid/operators/elementwise/elementwise_op_impl.cu.h
+++ b/paddle/fluid/operators/elementwise/elementwise_op_impl.cu.h
@@ -23,8 +23,6 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
-using ElementwiseType = phi::ElementwiseType;
-
 template <typename OutT, typename Functor, int NumOuts = 1>
 void LaunchSameDimsElementwiseCudaKernel(
     const KPDevice &ctx,
diff --git a/paddle/fluid/operators/fused/attn_gemm.h b/paddle/fluid/operators/fused/attn_gemm.h
index 9709f60bbc1ce..277e29c4d59ce 100644
--- a/paddle/fluid/operators/fused/attn_gemm.h
+++ b/paddle/fluid/operators/fused/attn_gemm.h
@@ -109,8 +109,8 @@ class AttnMatMul {
       // bias_out = output + bias
       std::vector<const phi::DenseTensor*> ins = {output, bias};
       std::vector<phi::DenseTensor*> outs = {bias_out};
-      phi::funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-          dev_ctx_, ins, &outs, -1, phi::funcs::AddFunctor<T>());
+      phi::funcs::BroadcastKernel<T>(
+          dev_ctx_, ins, &outs, phi::funcs::AddFunctor<T>());
     }
   }
 
diff --git a/paddle/fluid/operators/fused/attn_gemm_int8.h b/paddle/fluid/operators/fused/attn_gemm_int8.h
index c61a7f60d4359..705cb8ece418e 100644
--- a/paddle/fluid/operators/fused/attn_gemm_int8.h
+++ b/paddle/fluid/operators/fused/attn_gemm_int8.h
@@ -85,8 +85,8 @@ class AttnMatmulINT8 {
       // bias_out = output + bias
       std::vector<const phi::DenseTensor*> ins = {output, bias};
       std::vector<phi::DenseTensor*> outs = {bias_out};
-      phi::funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-          dev_ctx_, ins, &outs, -1, phi::funcs::AddFunctor<T>());
+      phi::funcs::BroadcastKernel<T>(
+          dev_ctx_, ins, &outs, phi::funcs::AddFunctor<T>());
       PADDLE_ENFORCE_EQ(cudaGetLastError(),
                         cudaSuccess,
                         platform::errors::Fatal(
@@ -139,8 +139,8 @@ class AttnMatmulINT8 {
       // bias_out = output + bias
       std::vector<const phi::DenseTensor*> ins = {output, bias};
       std::vector<phi::DenseTensor*> outs = {bias_out};
-      phi::funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-          dev_ctx_, ins, &outs, -1, phi::funcs::AddFunctor<T>());
+      phi::funcs::BroadcastKernel<T>(
+          dev_ctx_, ins, &outs, phi::funcs::AddFunctor<T>());
       PADDLE_ENFORCE_EQ(cudaGetLastError(),
                         cudaSuccess,
                         platform::errors::Fatal(
diff --git a/paddle/fluid/operators/fused/fmha_ref.h b/paddle/fluid/operators/fused/fmha_ref.h
index 1d83c7a62b1d9..843b5009a6fcc 100644
--- a/paddle/fluid/operators/fused/fmha_ref.h
+++ b/paddle/fluid/operators/fused/fmha_ref.h
@@ -255,12 +255,11 @@ class FMHARef {
         ins.emplace_back(src_mask_tensor);
         outs.emplace_back(src_mask_out_tensor);
         int elewise_add_axis = -1;
-        phi::funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-            dev_ctx_,
-            ins,
-            &outs,
-            elewise_add_axis,
-            phi::funcs::AddFunctor<T>());
+        phi::funcs::BroadcastKernel<T>(dev_ctx_,
+                                       ins,
+                                       &outs,
+                                       phi::funcs::AddFunctor<T>(),
+                                       elewise_add_axis);
 
         phi::SoftmaxForwardCUDAKernelDriver<T>(
             dev_ctx_, *src_mask_out_tensor, softmax_axis, softmax_out_tensor);
@@ -432,12 +431,11 @@ class FMHARef {
         ins.emplace_back(src_mask_tensor);
         outs.emplace_back(src_mask_out_tensor);
         int elewise_add_axis = -1;
-        phi::funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-            dev_ctx_,
-            ins,
-            &outs,
-            elewise_add_axis,
-            phi::funcs::AddFunctor<T>());
+        phi::funcs::BroadcastKernel<T>(dev_ctx_,
+                                       ins,
+                                       &outs,
+                                       phi::funcs::AddFunctor<T>(),
+                                       elewise_add_axis);
 
         phi::SoftmaxForwardCUDAKernelDriver<T>(
             dev_ctx_, *src_mask_out_tensor, softmax_axis, softmax_out_tensor);
diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index c8c4733df2e2e..105647baf1c35 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -689,13 +689,13 @@ class FMHAGateRef {
       std::vector<const phi::DenseTensor*> ins = {
           qk_out, src_mask, nonbatched_bias};
       std::vector<phi::DenseTensor*> outs = {qk_out};
-      phi::funcs::BroadcastKernel<phi::ElementwiseType::kTernary, T, T>(
-          dev_ctx_, ins, &outs, -1, TernaryAddFunctor<T>());
+      phi::funcs::BroadcastKernel<T>(
+          dev_ctx_, ins, &outs, TernaryAddFunctor<T>());
     } else {
       std::vector<const phi::DenseTensor*> ins = {qk_out, src_mask};
       std::vector<phi::DenseTensor*> outs = {qk_out};
-      phi::funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-          dev_ctx_, ins, &outs, -1, phi::funcs::AddFunctor<T>());
+      phi::funcs::BroadcastKernel<T>(
+          dev_ctx_, ins, &outs, phi::funcs::AddFunctor<T>());
     }
     phi::SoftmaxForwardCUDAKernelDriver<T>(dev_ctx_, *qk_out, -1, softmax_out);
   }
diff --git a/paddle/fluid/operators/fused_token_prune_op.cu b/paddle/fluid/operators/fused_token_prune_op.cu
index 434c072e5aa6a..8f0a53611f3b2 100644
--- a/paddle/fluid/operators/fused_token_prune_op.cu
+++ b/paddle/fluid/operators/fused_token_prune_op.cu
@@ -141,8 +141,7 @@ class FusedTokenPruneOpCUDAKernel : public framework::OpKernel<T> {
     ins.emplace_back(attn);
     ins.emplace_back(mask);
     outs.emplace_back(&attn_tmp);
-    LaunchElementwiseCudaKernel<ElementwiseType::kBinary, T, T>(
-        dev_ctx, ins, &outs, -1, AttnMaskFunctor<T>());
+    LaunchElementwiseCudaKernel<T>(dev_ctx, ins, &outs, AttnMaskFunctor<T>());
 
     // 2. Reduce sum
     const std::vector<int64_t> reduce_dims{1, 2};
diff --git a/paddle/fluid/operators/reduce_ops/reduce_op.h b/paddle/fluid/operators/reduce_ops/reduce_op.h
index 3349400a2f93b..5cea4fa9e0573 100644
--- a/paddle/fluid/operators/reduce_ops/reduce_op.h
+++ b/paddle/fluid/operators/reduce_ops/reduce_op.h
@@ -834,12 +834,11 @@ class ReduceCudaGradKernel : public framework::OpKernel<T> {
     }
 
     using MPType = typename kps::details::MPTypeTrait<T>::Type;
-    phi::ReduceGrad<T, TransformOp<T, MPType>>(
-        dev_ctx,
-        pt_d_out.get(),
-        pt_d_x.get(),
-        pt_out_dtype,
-        TransformOp<T, MPType>(reduce_num));
+    phi::ReduceGrad<TransformOp<T, MPType>>(dev_ctx,
+                                            pt_d_out.get(),
+                                            pt_d_x.get(),
+                                            pt_out_dtype,
+                                            TransformOp<T, MPType>(reduce_num));
   }
 };
 
diff --git a/paddle/phi/kernels/cpu/bitwise_kernel.cc b/paddle/phi/kernels/cpu/bitwise_kernel.cc
index 80424ef624f61..a6297efd9cd3e 100644
--- a/paddle/phi/kernels/cpu/bitwise_kernel.cc
+++ b/paddle/phi/kernels/cpu/bitwise_kernel.cc
@@ -24,15 +24,15 @@ limitations under the License. */
 
 namespace phi {
 
-#define DEFINE_BITWISE_KERNEL(op_type)                                    \
-  template <typename T, typename Context>                                 \
-  void Bitwise##op_type##Kernel(const Context& dev_ctx,                   \
-                                const DenseTensor& x,                     \
-                                const DenseTensor& y,                     \
-                                DenseTensor* out) {                       \
-    funcs::Bitwise##op_type##Functor<T> func;                             \
-    funcs::ElementwiseCompute<funcs::Bitwise##op_type##Functor<T>, T, T>( \
-        dev_ctx, x, y, -1, func, out);                                    \
+#define DEFINE_BITWISE_KERNEL(op_type)                                 \
+  template <typename T, typename Context>                              \
+  void Bitwise##op_type##Kernel(const Context& dev_ctx,                \
+                                const DenseTensor& x,                  \
+                                const DenseTensor& y,                  \
+                                DenseTensor* out) {                    \
+    funcs::Bitwise##op_type##Functor<T> func;                          \
+    funcs::ElementwiseCompute<funcs::Bitwise##op_type##Functor<T>, T>( \
+        dev_ctx, x, y, func, out);                                     \
   }
 
 DEFINE_BITWISE_KERNEL(And)
diff --git a/paddle/phi/kernels/cpu/compare_kernel.cc b/paddle/phi/kernels/cpu/compare_kernel.cc
index cf8eb47fb427f..0fd1332d76c64 100644
--- a/paddle/phi/kernels/cpu/compare_kernel.cc
+++ b/paddle/phi/kernels/cpu/compare_kernel.cc
@@ -33,10 +33,10 @@ inline void CompareKernelImpl(const Context& ctx,
   ctx.template Alloc<bool>(out);
   if (x.dims().size() >= y.dims().size()) {
     funcs::ElementwiseCompute<Functor, T, bool>(
-        ctx, x, y, axis, Functor(), out);
+        ctx, x, y, Functor(), out, axis);
   } else {
     funcs::ElementwiseCompute<InverseFunctor, T, bool>(
-        ctx, x, y, axis, InverseFunctor(), out);
+        ctx, x, y, InverseFunctor(), out, axis);
   }
 }
 
@@ -59,7 +59,7 @@ inline void CompareAllKernelImpl(const Context& ctx,
       tmp_data[0] = Functor()(x.data<T>()[0], y.data<T>()[0]);
     } else {
       funcs::ElementwiseCompute<Functor, T, bool>(
-          ctx, x, y, 0, Functor(), &tmp);
+          ctx, x, y, Functor(), &tmp, 0);
     }
     auto tmp_flat = EigenVector<bool>::Flatten(tmp);
     auto out_es = EigenScalar<bool>::From(*out);
diff --git a/paddle/phi/kernels/cpu/dirichlet_kernel.cc b/paddle/phi/kernels/cpu/dirichlet_kernel.cc
index c124920dfa0db..855e6bdfe1e1f 100644
--- a/paddle/phi/kernels/cpu/dirichlet_kernel.cc
+++ b/paddle/phi/kernels/cpu/dirichlet_kernel.cc
@@ -91,8 +91,8 @@ struct DirichletSampler<CPUContext, T> {
         true,
         false);
 
-    funcs::ElementwiseCompute<funcs::DivideFunctor<T>, T, T>(
-        dev_ctx, gamma_samples, gamma_sum, -1, funcs::DivideFunctor<T>(), out);
+    funcs::ElementwiseCompute<funcs::DivideFunctor<T>, T>(
+        dev_ctx, gamma_samples, gamma_sum, funcs::DivideFunctor<T>(), out);
   }
 };
 
diff --git a/paddle/phi/kernels/cpu/elementwise_divide_kernel.cc b/paddle/phi/kernels/cpu/elementwise_divide_kernel.cc
index 40d0e863ea50e..8dbbfc13b81e2 100644
--- a/paddle/phi/kernels/cpu/elementwise_divide_kernel.cc
+++ b/paddle/phi/kernels/cpu/elementwise_divide_kernel.cc
@@ -38,10 +38,10 @@ void DivideRawKernel(const Context& dev_ctx,
     auto y_dims = y.dims();
     if (x_dims.size() >= y_dims.size()) {
       funcs::ElementwiseCompute<funcs::DivideFunctor<T>, T>(
-          dev_ctx, x, y, axis, funcs::DivideFunctor<T>(), out);
+          dev_ctx, x, y, funcs::DivideFunctor<T>(), out, axis);
     } else {
       funcs::ElementwiseCompute<funcs::InverseDivideFunctor<T>, T>(
-          dev_ctx, x, y, axis, funcs::InverseDivideFunctor<T>(), out);
+          dev_ctx, x, y, funcs::InverseDivideFunctor<T>(), out, axis);
     }
   }
 }
diff --git a/paddle/phi/kernels/cpu/elementwise_kernel.cc b/paddle/phi/kernels/cpu/elementwise_kernel.cc
index 9b564679b354e..5af36cfbf2009 100644
--- a/paddle/phi/kernels/cpu/elementwise_kernel.cc
+++ b/paddle/phi/kernels/cpu/elementwise_kernel.cc
@@ -75,7 +75,7 @@ void HeavisideKernel(const Context& dev_ctx,
   // allocate memory for out
   dev_ctx.template Alloc<T>(out);
   funcs::ElementwiseCompute<funcs::ElementwiseHeavisideFunctor<T>, T>(
-      dev_ctx, x, y, -1, funcs::ElementwiseHeavisideFunctor<T>(), out);
+      dev_ctx, x, y, funcs::ElementwiseHeavisideFunctor<T>(), out);
 }
 
 }  // namespace phi
diff --git a/paddle/phi/kernels/cpu/layer_norm_grad_kernel.cc b/paddle/phi/kernels/cpu/layer_norm_grad_kernel.cc
index c42e423ba2d34..630e786b571bc 100644
--- a/paddle/phi/kernels/cpu/layer_norm_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/layer_norm_grad_kernel.cc
@@ -68,20 +68,15 @@ void LayerNormGradKernel(const Context& dev_ctx,
     temp_norm.Resize(matrix_shape);
     dev_ctx.template Alloc<T>(&temp_norm);
     // get x_norm
-    phi::funcs::ElementwiseCompute<funcs::SubtractFunctor<T>, T, T>(
-        dev_ctx,
-        x_tmp,
-        mean,
-        /*axis*/ 0,
-        funcs::SubtractFunctor<T>(),
-        &temp_norm);
-    phi::funcs::ElementwiseCompute<funcs::DivAndSqrtFunctor<T>, T, T>(
+    phi::funcs::ElementwiseCompute<funcs::SubtractFunctor<T>, T>(
+        dev_ctx, x_tmp, mean, funcs::SubtractFunctor<T>(), &temp_norm, 0);
+    phi::funcs::ElementwiseCompute<funcs::DivAndSqrtFunctor<T>, T>(
         dev_ctx,
         temp_norm,
         variance,
-        /*axis*/ 0,
         funcs::DivAndSqrtFunctor<T>(static_cast<T>(epsilon)),
-        &temp_norm);
+        &temp_norm,
+        0);
   }
 
   if (d_bias) {
@@ -90,8 +85,8 @@ void LayerNormGradKernel(const Context& dev_ctx,
   }
   if (d_scale) {
     dev_ctx.template Alloc<T>(d_scale);
-    phi::funcs::ElementwiseCompute<funcs::MultiplyFunctor<T>, T, T>(
-        dev_ctx, temp_norm, d_y, 0, funcs::MultiplyFunctor<T>(), &temp);
+    phi::funcs::ElementwiseCompute<funcs::MultiplyFunctor<T>, T>(
+        dev_ctx, temp_norm, d_y, funcs::MultiplyFunctor<T>(), &temp, 0);
     colwise_sum(dev_ctx, temp, d_scale);
   }
 
@@ -107,70 +102,45 @@ void LayerNormGradKernel(const Context& dev_ctx,
 
     if (d_scale) {
       // dy_dx
-      phi::funcs::ElementwiseCompute<funcs::MultiplyFunctor<T>, T, T>(
-          dev_ctx, d_y, *scale, /*axis*/ 1, funcs::MultiplyFunctor<T>(), &temp);
+      phi::funcs::ElementwiseCompute<funcs::MultiplyFunctor<T>, T>(
+          dev_ctx, d_y, *scale, funcs::MultiplyFunctor<T>(), &temp, 1);
       phi::Copy<Context>(dev_ctx, temp, dev_ctx.GetPlace(), false, d_x);
 
       // dy_dmean_dx
       row_mean(dev_ctx, temp, &temp_vec);
-      phi::funcs::ElementwiseCompute<funcs::SubtractFunctor<T>, T, T>(
-          dev_ctx,
-          *d_x,
-          temp_vec,
-          /*axis*/ 0,
-          funcs::SubtractFunctor<T>(),
-          d_x);
+      phi::funcs::ElementwiseCompute<funcs::SubtractFunctor<T>, T>(
+          dev_ctx, *d_x, temp_vec, funcs::SubtractFunctor<T>(), d_x, 0);
 
       // dy_var_dx
-      phi::funcs::ElementwiseCompute<funcs::MultiplyFunctor<T>, T, T>(
-          dev_ctx,
-          temp,
-          temp_norm,
-          /*axis*/ 0,
-          funcs::MultiplyFunctor<T>(),
-          &temp);
+      phi::funcs::ElementwiseCompute<funcs::MultiplyFunctor<T>, T>(
+          dev_ctx, temp, temp_norm, funcs::MultiplyFunctor<T>(), &temp, 0);
     } else {
       // dy_dx
       phi::Copy<Context>(dev_ctx, d_y, dev_ctx.GetPlace(), false, d_x);
 
       // dy_dmean_dx
       row_mean(dev_ctx, d_y, &temp_vec);
-      phi::funcs::ElementwiseCompute<funcs::SubtractFunctor<T>, T, T>(
-          dev_ctx,
-          *d_x,
-          temp_vec,
-          /*axis*/ 0,
-          funcs::SubtractFunctor<T>(),
-          d_x);
+      phi::funcs::ElementwiseCompute<funcs::SubtractFunctor<T>, T>(
+          dev_ctx, *d_x, temp_vec, funcs::SubtractFunctor<T>(), d_x, 0);
 
       // dy_var_dx
-      phi::funcs::ElementwiseCompute<funcs::MultiplyFunctor<T>, T, T>(
-          dev_ctx,
-          d_y,
-          temp_norm,
-          /*axis*/ 0,
-          funcs::MultiplyFunctor<T>(),
-          &temp);
+      phi::funcs::ElementwiseCompute<funcs::MultiplyFunctor<T>, T>(
+          dev_ctx, d_y, temp_norm, funcs::MultiplyFunctor<T>(), &temp, 0);
     }
     // dy_var_dx
     row_mean(dev_ctx, temp, &temp_vec);
-    phi::funcs::ElementwiseCompute<funcs::MultiplyFunctor<T>, T, T>(
-        dev_ctx,
-        temp_norm,
-        temp_vec,
-        /*axis*/ 0,
-        funcs::MultiplyFunctor<T>(),
-        &temp);
-    phi::funcs::ElementwiseCompute<funcs::SubtractFunctor<T>, T, T>(
-        dev_ctx, *d_x, temp, /*axis*/ 0, funcs::SubtractFunctor<T>(), d_x);
-
-    phi::funcs::ElementwiseCompute<funcs::DivAndSqrtFunctor<T>, T, T>(
+    phi::funcs::ElementwiseCompute<funcs::MultiplyFunctor<T>, T>(
+        dev_ctx, temp_norm, temp_vec, funcs::MultiplyFunctor<T>(), &temp, 0);
+    phi::funcs::ElementwiseCompute<funcs::SubtractFunctor<T>, T>(
+        dev_ctx, *d_x, temp, funcs::SubtractFunctor<T>(), d_x, 0);
+
+    phi::funcs::ElementwiseCompute<funcs::DivAndSqrtFunctor<T>, T>(
         dev_ctx,
         *d_x,
         variance,
-        /*axis*/ 0,
         funcs::DivAndSqrtFunctor<T>(static_cast<T>(epsilon)),
-        d_x);
+        d_x,
+        0);
     d_x->Resize(dx_dim);
   }
 }
diff --git a/paddle/phi/kernels/cpu/layer_norm_kernel.cc b/paddle/phi/kernels/cpu/layer_norm_kernel.cc
index 1c82866f0bbda..2a93d03b4abc1 100644
--- a/paddle/phi/kernels/cpu/layer_norm_kernel.cc
+++ b/paddle/phi/kernels/cpu/layer_norm_kernel.cc
@@ -67,30 +67,30 @@ void LayerNormKernel(const Context& dev_ctx,
 
   // get variance
 
-  phi::funcs::ElementwiseCompute<funcs::SubAndSquareFunctor<T>, T, T>(
-      dev_ctx, x_tmp, *mean, 0, funcs::SubAndSquareFunctor<T>(), &out);
+  phi::funcs::ElementwiseCompute<funcs::SubAndSquareFunctor<T>, T>(
+      dev_ctx, x_tmp, *mean, funcs::SubAndSquareFunctor<T>(), &out, 0);
 
   row_mean(dev_ctx, out, var);
 
   // get x_norm
-  phi::funcs::ElementwiseCompute<funcs::SubtractFunctor<T>, T, T>(
-      dev_ctx, x_tmp, *mean, 0, funcs::SubtractFunctor<T>(), &out);
+  phi::funcs::ElementwiseCompute<funcs::SubtractFunctor<T>, T>(
+      dev_ctx, x_tmp, *mean, funcs::SubtractFunctor<T>(), &out, 0);
 
-  phi::funcs::ElementwiseCompute<funcs::DivAndSqrtFunctor<T>, T, T>(
+  phi::funcs::ElementwiseCompute<funcs::DivAndSqrtFunctor<T>, T>(
       dev_ctx,
       out,
       *var,
-      0,
       funcs::DivAndSqrtFunctor<T>(static_cast<T>(epsilon)),
-      &out);
+      &out,
+      0);
 
   if (scale) {
-    phi::funcs::ElementwiseCompute<funcs::MultiplyFunctor<T>, T, T>(
-        dev_ctx, out, *scale, 1, funcs::MultiplyFunctor<T>(), &out);
+    phi::funcs::ElementwiseCompute<funcs::MultiplyFunctor<T>, T>(
+        dev_ctx, out, *scale, funcs::MultiplyFunctor<T>(), &out, 1);
   }
   if (bias) {
-    phi::funcs::ElementwiseCompute<funcs::AddFunctor<T>, T, T>(
-        dev_ctx, out, *bias, 1, funcs::AddFunctor<T>(), &out);
+    phi::funcs::ElementwiseCompute<funcs::AddFunctor<T>, T>(
+        dev_ctx, out, *bias, funcs::AddFunctor<T>(), &out, 1);
   }
 #else
   PADDLE_ENFORCE_EQ(mean->numel(),
diff --git a/paddle/phi/kernels/cpu/logical_kernel.cc b/paddle/phi/kernels/cpu/logical_kernel.cc
index 3a669883f7837..38c927e976f8c 100644
--- a/paddle/phi/kernels/cpu/logical_kernel.cc
+++ b/paddle/phi/kernels/cpu/logical_kernel.cc
@@ -32,7 +32,7 @@ namespace phi {
                              DenseTensor* out) {                          \
     funcs::Logical##type##Functor<T> binary_func;                         \
     funcs::ElementwiseCompute<funcs::Logical##type##Functor<T>, T, bool>( \
-        dev_ctx, x, y, -1, binary_func, out);                             \
+        dev_ctx, x, y, binary_func, out);                                 \
   }
 
 DEFINE_LOGICAL_BINARY_KERNEL(And)
diff --git a/paddle/phi/kernels/cpu/matrix_rank_tol_kernel.cc b/paddle/phi/kernels/cpu/matrix_rank_tol_kernel.cc
index fbb16138567d1..e1188fda486c7 100644
--- a/paddle/phi/kernels/cpu/matrix_rank_tol_kernel.cc
+++ b/paddle/phi/kernels/cpu/matrix_rank_tol_kernel.cc
@@ -132,11 +132,10 @@ void MatrixRankTolKernel(const Context& dev_ctx,
   DenseTensor tol_tensor;
   tol_tensor.Resize(dim_out);
   dev_ctx.template Alloc<T>(&tol_tensor);
-  funcs::ElementwiseCompute<GreaterElementFunctor<T>, T, T>(
+  funcs::ElementwiseCompute<GreaterElementFunctor<T>, T>(
       dev_ctx,
       atol_tensor,
       rtol_tensor,
-      -1,
       GreaterElementFunctor<T>(),
       &tol_tensor);
 
@@ -151,17 +150,17 @@ void MatrixRankTolKernel(const Context& dev_ctx,
         dev_ctx,
         eigenvalue_tensor,
         tol_tensor,
-        axis,
         funcs::GreaterThanFunctor<T, int64_t>(),
-        &compare_result);
+        &compare_result,
+        axis);
   } else {
     funcs::ElementwiseCompute<funcs::LessThanFunctor<T, int64_t>, T, int>(
         dev_ctx,
         eigenvalue_tensor,
         tol_tensor,
-        axis,
         funcs::LessThanFunctor<T, int64_t>(),
-        &compare_result);
+        &compare_result,
+        axis);
   }
 
   phi::SumKernel<int64_t>(dev_ctx,
diff --git a/paddle/phi/kernels/funcs/broadcast_function.h b/paddle/phi/kernels/funcs/broadcast_function.h
index b4125ab555079..d0f3fba392a86 100644
--- a/paddle/phi/kernels/funcs/broadcast_function.h
+++ b/paddle/phi/kernels/funcs/broadcast_function.h
@@ -189,45 +189,29 @@ struct BroadcastDataLoader<Index, VecSize, false, kElementwise> {
   }
 };
 
-// Common broadcast data loader.
-template <int Index, int VecSize, bool IsBoundary>
-struct BroadcastDataLoader<Index, VecSize, IsBoundary, kBroadcast> {
-  template <typename Array1, typename Array2, typename Array3, typename ArgsT>
-  static __device__ __forceinline__ void Apply(const Array1 &ins,
-                                               ArgsT *args,
-                                               const Array2 &configs,
-                                               const Array3 &use_broadcast,
-                                               const int block_offset,
-                                               const int num,
-                                               const uint32_t numel) {
+template <int Index, int VecSize>
+struct BroadcastDataInit {
+  template <typename ArgsT>
+  static __device__ __forceinline__ void Apply(ArgsT *args) {
     using Type = std::tuple_element_t<Index, ArgsT>;
-    uint32_t index_bc[VecSize];
 #pragma unroll
     for (int k = 0; k < VecSize; ++k) {
-      index_bc[k] = 0;
       std::get<Index>(args[k]) = static_cast<Type>(1);
     }
+  }
+};
 
-    uint32_t thread_offset = block_offset + threadIdx.x * VecSize;
-#pragma unroll
-    for (int k = 0; k < VecSize; ++k) {
-      uint32_t idx = thread_offset + k;
-      if (IsBoundary && idx == numel) {
-        break;
-      }
-#pragma unroll
-      for (int i = 0; i < phi::DDim::kMaxRank; ++i) {
-        if (i == configs[0].rank) break;
-        auto fast_divmoder = configs[0].divmoders[i].Divmod(idx);
-        idx = fast_divmoder.val[0];
-        index_bc[k] += fast_divmoder.val[1] * configs[Index].strides[i];
-      }
-    }
-
+template <int Index, int VecSize>
+struct BroadcastDataSetter {
+  template <typename Array, typename ArgsT>
+  static __device__ __forceinline__ void Apply(const Array &ins,
+                                               ArgsT *args,
+                                               uint32_t index_bc[][VecSize]) {
+    using Type = std::tuple_element_t<Index, ArgsT>;
 #pragma unroll
     for (int k = 0; k < VecSize; ++k) {
       std::get<Index>(args[k]) =
-          reinterpret_cast<const _ptr_ Type *>(ins[Index])[index_bc[k]];
+          reinterpret_cast<const _ptr_ Type *>(ins[Index])[index_bc[Index][k]];
     }
   }
 };
@@ -285,8 +269,30 @@ __device__ void VectorizedBroadcastKernelImpl(
   __simd__ ArgsT args[VecSize];
   __simd__ ConditionalT<OutT, NumOuts> result[VecSize];
 
-  BcUnroller<BroadcastDataLoader, IsBoundary, LoadType, VecSize, Arity>::step(
-      ins, args, configs, use_broadcast, block_offset, num, numel);
+  if (LoadType == kBroadcast) {
+    uint32_t index_bc[Arity][VecSize] = {0};
+    Unroller<BroadcastDataInit, VecSize, Arity>::step(args);
+    uint32_t thread_offset = block_offset + threadIdx.x * VecSize;
+#pragma unroll
+    for (int k = 0; k < VecSize; ++k) {
+      uint32_t idx = thread_offset + k;
+      if (IsBoundary && idx == numel) break;
+#pragma unroll
+      for (int i = 0; i < phi::DDim::kMaxRank; ++i) {
+        if (i == configs[0].rank) break;
+        auto fast_divmoder = configs[0].divmoders[i].Divmod(idx);
+        idx = fast_divmoder.val[0];
+#pragma unroll
+        for (int j = 0; j < Arity; ++j) {
+          index_bc[j][k] += fast_divmoder.val[1] * configs[j].strides[i];
+        }
+      }
+    }
+    Unroller<BroadcastDataSetter, VecSize, Arity>::step(ins, args, index_bc);
+  } else {
+    BcUnroller<BroadcastDataLoader, IsBoundary, LoadType, VecSize, Arity>::step(
+        ins, args, configs, use_broadcast, block_offset, num, numel);
+  }
 
   SameDimsElementwisePrimitiveCaller<ConditionalT<OutT, NumOuts>,
                                      VecSize,
@@ -783,11 +789,7 @@ struct LaunchBroadcastKernelWithInt64IndexHelper<OutT,
 };
 #endif
 
-template <ElementwiseType ET,
-          typename OutT,
-          typename Functor,
-          int kArity,
-          int NumOuts = 1>
+template <typename OutT, typename Functor, int kArity, int NumOuts = 1>
 void BroadcastKernelForDifferentVecSize(
     const KPDevice &ctx,
     const std::vector<const DenseTensor *> &ins,
@@ -922,16 +924,12 @@ void BroadcastKernelForDifferentVecSize(
   }
 }
 
-template <ElementwiseType ET,
-          typename InT,
-          typename OutT,
-          typename Functor,
-          int NumOuts = 1>
+template <typename OutT, typename Functor, int NumOuts = 1>
 void BroadcastKernel(const KPDevice &ctx,
                      const std::vector<const DenseTensor *> &ins,
                      std::vector<DenseTensor *> *outs,
-                     int axis,
-                     Functor func) {
+                     Functor func,
+                     int axis = -1) {
   // When there are multiple inputs, the outputs's rank should be equal the
   // maximum rank of all inputs.
   using Traits = phi::funcs::FunctionTraits<Functor>;
@@ -968,7 +966,7 @@ void BroadcastKernel(const KPDevice &ctx,
     max_rank = std::max(max_rank, (*outs)[0]->dims().size());
   }
   axis = axis == -1 ? max_rank - min_rank : axis;
-  BroadcastKernelForDifferentVecSize<ET, OutT, Functor, kArity, NumOuts>(
+  BroadcastKernelForDifferentVecSize<OutT, Functor, kArity, NumOuts>(
       ctx, ins, outs, axis, func);
 }
 
@@ -976,15 +974,14 @@ template <typename Functor, typename T, typename OutType = T>
 void ElementwiseCompute(const GPUContext &dev_ctx,
                         const DenseTensor &x,
                         const DenseTensor &y,
-                        int axis,
                         Functor func,
-                        DenseTensor *z) {
+                        DenseTensor *z,
+                        int axis = -1) {
   std::vector<const DenseTensor *> ins = {&x, &y};
   std::vector<DenseTensor *> outs = {z};
   dev_ctx.template Alloc<OutType>(z);
 
-  BroadcastKernel<ElementwiseType::kBinary, T, OutType, Functor, 1>(
-      dev_ctx, ins, &outs, axis, func);
+  BroadcastKernel<OutType, Functor, 1>(dev_ctx, ins, &outs, func, axis);
 }
 
 template <typename DeviceContext,
@@ -999,7 +996,7 @@ void DefaultElementwiseOperator(const DeviceContext &dev_ctx,
   auto x_dims = x.dims();
   auto y_dims = y.dims();
   dev_ctx.template Alloc<T>(z);
-  funcs::ElementwiseCompute<Functor, T>(dev_ctx, x, y, axis, Functor(), z);
+  funcs::ElementwiseCompute<Functor, T>(dev_ctx, x, y, Functor(), z, axis);
 }
 
 #else
@@ -1017,10 +1014,10 @@ void DefaultElementwiseOperator(const DeviceContext &dev_ctx,
   auto y_dims = y.dims();
   dev_ctx.template Alloc<T>(z);
   if (x_dims.size() >= y_dims.size()) {
-    funcs::ElementwiseCompute<Functor, T>(dev_ctx, x, y, axis, Functor(), z);
+    funcs::ElementwiseCompute<Functor, T>(dev_ctx, x, y, Functor(), z, axis);
   } else {
     funcs::ElementwiseCompute<InverseFunctor, T>(
-        dev_ctx, x, y, axis, InverseFunctor(), z);
+        dev_ctx, x, y, InverseFunctor(), z, axis);
   }
 }
 #endif
diff --git a/paddle/phi/kernels/funcs/dropout_impl.cu.h b/paddle/phi/kernels/funcs/dropout_impl.cu.h
index 2ae9d3c02f63d..48a7008463c96 100644
--- a/paddle/phi/kernels/funcs/dropout_impl.cu.h
+++ b/paddle/phi/kernels/funcs/dropout_impl.cu.h
@@ -191,25 +191,19 @@ __global__ void VectorizedRandomGenerator(const size_t n,
 }
 
 template <typename T>
-__global__ void DropOutNdForwardKernel(
-    const size_t n,
-    uint64_t seed,
-    const float dropout_prob,
-    const T* src,
-    uint8_t* mask,
-    uint64_t increment,
-    size_t main_offset,
-    DstFunctor<T> dst_functor,
-    MaskFunctor<T> mask_functor,
-    T* y,
-    int64_t N,
-    kps::details::BroadcastConfig broadcast_config,
-    const uint64_t* seed_ptr) {
+__global__ void VectorizedGeneratorMask(const size_t n,
+                                        uint64_t seed,
+                                        const float dropout_prob,
+                                        const T* src,
+                                        uint8_t* mask,
+                                        uint64_t increment,
+                                        size_t main_offset,
+                                        MaskFunctor<T> mask_functor,
+
+                                        const uint64_t* seed_ptr) {
   // Vectorized Generate Mask
   // kCount is 4 for curand_uniform4 is used
-  if (seed_ptr) {
-    seed = seed_ptr[0];
-  }
+  if (seed_ptr) seed = seed_ptr[0];
 
   constexpr int kCount = phi::funcs::uniform_distribution<float>::kReturnsCount;
   size_t idx = static_cast<size_t>(BLOCK_ID_X * BLOCK_NUM_X);
@@ -259,22 +253,6 @@ __global__ void DropOutNdForwardKernel(
     kps::WriteData<uint8_t, kCount, 1, true>(
         mask + fix, &mask_result[0], remainder);
   }
-  // Broadcast mask data and do elementwise operaiton with DstFunctor
-  CUDA_KERNEL_LOOP(i, N) {
-    uint32_t offset = 0u;
-    uint32_t idx = i;
-    // Use (j < phi::DDim::kMaxRank) conditiion rather than
-    // (j < broadcast_config.rank) for (#pragma unroll)
-#pragma unroll
-    for (int j = 0; j < phi::DDim::kMaxRank; ++j) {
-      if (j == broadcast_config.rank) break;
-      auto fast_divmoder = broadcast_config.divmoders[j].Divmod(idx);
-      idx = fast_divmoder.val[0];
-      offset += broadcast_config.strides[j] * fast_divmoder.val[1];
-    }
-    __syncthreads();
-    y[i] = dst_functor(src[i], mask[offset]);
-  }
 }
 
 template <typename T, typename MT>
@@ -348,18 +326,6 @@ void DropoutFwGPUKernelDriver(
         size / (block_size * kVecSize) * (block_size * kVecSize);
 
     if (is_dropout_nd) {
-      auto dst_functor =
-          DstFunctor<T>(1.0f - dropout_prob, upscale_in_train, x_numel);
-
-      std::vector<int64_t> out_dims =
-          std::move(phi::vectorize<int64_t>(x.dims()));
-      std::vector<int64_t> in_dims =
-          std::move(phi::vectorize<int64_t>(mask->dims()));
-      std::reverse(out_dims.begin(), out_dims.end());
-      std::reverse(in_dims.begin(), in_dims.end());
-      kps::details::BroadcastConfig broadcast_config(
-          out_dims, in_dims, x.dims().size());
-
       auto mask_functor = MaskFunctor<T>(1.0f - dropout_prob);
       bool copy_in_kernel = GetSeedDataAndIncrement(dev_ctx,
                                                     seed,
@@ -372,20 +338,22 @@ void DropoutFwGPUKernelDriver(
       const uint64_t* seed_ptr =
           copy_in_kernel ? seed->data<uint64_t>() : nullptr;
 
-      DropOutNdForwardKernel<T>
+      VectorizedGeneratorMask<T>
           <<<grid_size, block_size, 0, stream>>>(size,
                                                  seed_data,
                                                  dropout_prob,
                                                  x_data,
                                                  mask_data,
+
                                                  increment,
                                                  main_offset,
-                                                 dst_functor,
                                                  mask_functor,
-                                                 y_data,
-                                                 y->numel(),
-                                                 broadcast_config,
                                                  seed_ptr);
+      auto dst_functor =
+          DstFunctor<T>(1.0f - dropout_prob, upscale_in_train, x_numel);
+      std::vector<const phi::DenseTensor*> ins = {&x, mask};
+      std::vector<phi::DenseTensor*> outs = {y};
+      phi::funcs::BroadcastKernel<T>(dev_ctx, ins, &outs, dst_functor);
     } else {
       bool copy_in_kernel = GetSeedDataAndIncrement(
           dev_ctx, seed, is_fix_seed, seed_val, offset, &seed_data, &increment);
@@ -469,30 +437,13 @@ void DropoutGradGPUKernelDriver(const phi::GPUContext& dev_ctx,
       MT factor = upscale_in_train
                       ? static_cast<MT>(1.0f / (1.0f - dropout_prob))
                       : static_cast<MT>(1.0f);
+
+      std::vector<const phi::DenseTensor*> ins = {&grad_y, &mask};
+      std::vector<phi::DenseTensor*> outs = {grad_x};
       if (is_dropout_nd) {
-        phi::DenseTensor broadcasted_mask;
-
-        broadcasted_mask.Resize(grad_y.dims());
-        dev_ctx.template Alloc<uint8_t>(&broadcasted_mask);
-
-        std::vector<const phi::DenseTensor*> broadcast_ins = {&mask};
-        std::vector<phi::DenseTensor*> broadcast_outs = {&broadcasted_mask};
-        phi::funcs::BroadcastKernel<phi::ElementwiseType::kUnary,
-                                    uint8_t,
-                                    uint8_t>(dev_ctx,
-                                             broadcast_ins,
-                                             &broadcast_outs,
-                                             -1,
-                                             kps::IdentityFunctor<uint8_t>());
-
-        std::vector<const phi::DenseTensor*> ins = {&grad_y, &broadcasted_mask};
-        std::vector<phi::DenseTensor*> outs = {grad_x};
-        phi::funcs::ElementwiseKernel<T>(
+        phi::funcs::BroadcastKernel<T>(
             dev_ctx, ins, &outs, CudaDropoutGradFunctor<T>(factor));
-
       } else {
-        std::vector<const phi::DenseTensor*> ins = {&grad_y, &mask};
-        std::vector<phi::DenseTensor*> outs = {grad_x};
         phi::funcs::ElementwiseKernel<T>(
             dev_ctx, ins, &outs, CudaDropoutGradFunctor<T>(factor));
       }
diff --git a/paddle/phi/kernels/funcs/elementwise_base.h b/paddle/phi/kernels/funcs/elementwise_base.h
index 0f6c225d89b2a..274ac1cc32c05 100644
--- a/paddle/phi/kernels/funcs/elementwise_base.h
+++ b/paddle/phi/kernels/funcs/elementwise_base.h
@@ -35,7 +35,6 @@ namespace kps = phi::kps;
 
 namespace phi {
 
-enum ElementwiseType { kUnary = 1, kBinary = 2, kTernary = 3 };
 /* Packing scalar type T(float, int etc.) into Array<T, NumOuts> type
    for supporting multiple-output feature in elementwise system.*/
 template <class T, int Num>
@@ -369,9 +368,9 @@ template <typename Functor, typename T, typename OutType = T>
 void ElementwiseCompute(const CPUContext &dev_ctx,
                         const DenseTensor &x,
                         const DenseTensor &y,
-                        int axis,
                         Functor func,
-                        DenseTensor *z) {
+                        DenseTensor *z,
+                        int axis = -1) {
   dev_ctx.Alloc<OutType>(z);
   auto x_dims = x.dims();
   auto y_dims = y.dims();
diff --git a/paddle/phi/kernels/fusion/gpu/attn_gemm.h b/paddle/phi/kernels/fusion/gpu/attn_gemm.h
index 01544436e4d7e..a96601dddacac 100644
--- a/paddle/phi/kernels/fusion/gpu/attn_gemm.h
+++ b/paddle/phi/kernels/fusion/gpu/attn_gemm.h
@@ -112,8 +112,8 @@ class AttnMatMul {
       // bias_out = output + bias
       std::vector<const phi::DenseTensor*> ins = {output, bias};
       std::vector<phi::DenseTensor*> outs = {bias_out};
-      phi::funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-          dev_ctx_, ins, &outs, -1, phi::funcs::AddFunctor<T>());
+      phi::funcs::BroadcastKernel<T>(
+          dev_ctx_, ins, &outs, phi::funcs::AddFunctor<T>());
     }
   }
 
diff --git a/paddle/phi/kernels/fusion/gpu/fmha_ref.h b/paddle/phi/kernels/fusion/gpu/fmha_ref.h
index 207be7b5d7bda..a41582c7076b3 100644
--- a/paddle/phi/kernels/fusion/gpu/fmha_ref.h
+++ b/paddle/phi/kernels/fusion/gpu/fmha_ref.h
@@ -258,12 +258,11 @@ class FMHARef {
         ins.emplace_back(src_mask_tensor);
         outs.emplace_back(src_mask_out_tensor);
         int elewise_add_axis = -1;
-        phi::funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-            dev_ctx_,
-            ins,
-            &outs,
-            elewise_add_axis,
-            phi::funcs::AddFunctor<T>());
+        phi::funcs::BroadcastKernel<T>(dev_ctx_,
+                                       ins,
+                                       &outs,
+                                       phi::funcs::AddFunctor<T>(),
+                                       elewise_add_axis);
 
         phi::SoftmaxForwardCUDAKernelDriver<T>(
             dev_ctx_, *src_mask_out_tensor, softmax_axis, softmax_out_tensor);
@@ -435,12 +434,11 @@ class FMHARef {
         ins.emplace_back(src_mask_tensor);
         outs.emplace_back(src_mask_out_tensor);
         int elewise_add_axis = -1;
-        phi::funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-            dev_ctx_,
-            ins,
-            &outs,
-            elewise_add_axis,
-            phi::funcs::AddFunctor<T>());
+        phi::funcs::BroadcastKernel<T>(dev_ctx_,
+                                       ins,
+                                       &outs,
+                                       phi::funcs::AddFunctor<T>(),
+                                       elewise_add_axis);
 
         phi::SoftmaxForwardCUDAKernelDriver<T>(
             dev_ctx_, *src_mask_out_tensor, softmax_axis, softmax_out_tensor);
diff --git a/paddle/phi/kernels/gpu/dirichlet_kernel.cu b/paddle/phi/kernels/gpu/dirichlet_kernel.cu
index eacbab800578b..09d6a402e701a 100644
--- a/paddle/phi/kernels/gpu/dirichlet_kernel.cu
+++ b/paddle/phi/kernels/gpu/dirichlet_kernel.cu
@@ -106,8 +106,8 @@ struct DirichletSampler<GPUContext, T> {
         {new_shape.size() - 1},
         true,
         false);
-    funcs::ElementwiseCompute<funcs::DivideFunctor<T>, T, T>(
-        dev_ctx, gamma_samples, gamma_sum, -1, funcs::DivideFunctor<T>(), out);
+    funcs::ElementwiseCompute<funcs::DivideFunctor<T>, T>(
+        dev_ctx, gamma_samples, gamma_sum, funcs::DivideFunctor<T>(), out);
   }
 };
 }  // namespace phi
diff --git a/paddle/phi/kernels/gpu/elementwise_divide_grad_kernel.cu b/paddle/phi/kernels/gpu/elementwise_divide_grad_kernel.cu
index 57bf6da4060d3..58cdc63b4a3dc 100644
--- a/paddle/phi/kernels/gpu/elementwise_divide_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/elementwise_divide_grad_kernel.cu
@@ -37,22 +37,21 @@ void DivideGradKernel(const Context& dev_ctx,
   const auto place = dev_ctx.GetPlace();
   if (dx != nullptr && dy != nullptr) {
     std::vector<const DenseTensor*> ins = {&dout, &out, &y};
-    GetGradXAndYOut<ElementwiseType::kTernary, T>(
-        dev_ctx,
-        place,
-        axis,
-        ins,
-        dout,
-        dx,
-        dy,
-        funcs::DivGradXYFunctor<T, T>());
+    GetGradXAndYOut<T>(dev_ctx,
+                       place,
+                       axis,
+                       ins,
+                       dout,
+                       dx,
+                       dy,
+                       funcs::DivGradXYFunctor<T, T>());
   } else if (dx != nullptr && dy == nullptr) {
     std::vector<const DenseTensor*> ins = {&dout, &y};
-    GetGradXOrYOut<ElementwiseType::kBinary, T>(
+    GetGradXOrYOut<T>(
         dev_ctx, place, axis, ins, dout, dx, funcs::DivGradXFunctor<T>());
   } else if (dy != nullptr && dx == nullptr) {
     std::vector<const DenseTensor*> ins = {&dout, &out, &y};
-    GetGradXOrYOut<ElementwiseType::kTernary, T>(
+    GetGradXOrYOut<T>(
         dev_ctx, place, axis, ins, dout, dy, funcs::DivGradYFunctor<T>());
   }
 }
diff --git a/paddle/phi/kernels/gpu/elementwise_grad.h b/paddle/phi/kernels/gpu/elementwise_grad.h
index 8440ed2b1222b..6152425f27221 100644
--- a/paddle/phi/kernels/gpu/elementwise_grad.h
+++ b/paddle/phi/kernels/gpu/elementwise_grad.h
@@ -35,7 +35,7 @@ void ReduceWrapper(const GPUContext &dev_ctx,
       dev_ctx, *src, dst, kps::IdentityFunctor<T>(), reduce_dims);
 }
 
-template <ElementwiseType ET, typename T, typename Functor>
+template <typename T, typename Functor>
 void GetGradXAndYOut(const GPUContext &dev_ctx,
                      const Place &place,
                      int axis,
@@ -67,8 +67,7 @@ void GetGradXAndYOut(const GPUContext &dev_ctx,
     outs = {&tmp_dx, &tmp_dy};
   }
 
-  funcs::BroadcastKernel<ET, T, T, decltype(func), 2>(
-      dev_ctx, ins, &outs, axis, func);
+  funcs::BroadcastKernel<T, decltype(func), 2>(dev_ctx, ins, &outs, func, axis);
 
   if (dx->dims() != dout.dims() && dy->dims() == dout.dims()) {
     ReduceWrapper<T>(dev_ctx, axis, &tmp_dx, dx);
@@ -80,7 +79,7 @@ void GetGradXAndYOut(const GPUContext &dev_ctx,
   }
 }
 
-template <ElementwiseType ET, typename T, typename Functor>
+template <typename T, typename Functor>
 void GetGradXOrYOut(const GPUContext &dev_ctx,
                     const Place &place,
                     int axis,
@@ -100,7 +99,7 @@ void GetGradXOrYOut(const GPUContext &dev_ctx,
     outs = {dxy};
   }
 
-  funcs::BroadcastKernel<ET, T, T>(dev_ctx, ins, &outs, axis, func);
+  funcs::BroadcastKernel<T>(dev_ctx, ins, &outs, func, axis);
   if (dxy->dims() != dout.dims()) {
     ReduceWrapper<T>(dev_ctx, axis, &tmp_dxy, dxy);
   }
@@ -342,22 +341,21 @@ void ElementwiseDivGrad(const GPUContext &dev_ctx,
   const auto place = dev_ctx.GetPlace();
   if (dx != nullptr && dy != nullptr) {
     std::vector<const DenseTensor *> ins = {&dout, &out, &y};
-    GetGradXAndYOut<ElementwiseType::kTernary, T>(
-        dev_ctx,
-        place,
-        axis,
-        ins,
-        dout,
-        dx,
-        dy,
-        funcs::DivGradXYFunctor<T, T>());
+    GetGradXAndYOut<T>(dev_ctx,
+                       place,
+                       axis,
+                       ins,
+                       dout,
+                       dx,
+                       dy,
+                       funcs::DivGradXYFunctor<T, T>());
   } else if (dx != nullptr && dy == nullptr) {
     std::vector<const DenseTensor *> ins = {&dout, &y};
-    GetGradXOrYOut<ElementwiseType::kBinary, T>(
+    GetGradXOrYOut<T>(
         dev_ctx, place, axis, ins, dout, dx, funcs::DivGradXFunctor<T>());
   } else if (dy != nullptr && dx == nullptr) {
     std::vector<const DenseTensor *> ins = {&dout, &out, &y};
-    GetGradXOrYOut<ElementwiseType::kTernary, T>(
+    GetGradXOrYOut<T>(
         dev_ctx, place, axis, ins, dout, dy, funcs::DivGradYFunctor<T>());
   }
 }
@@ -380,22 +378,21 @@ void ElementwiseMulGrad(const GPUContext &dev_ctx,
 
   if (dx != nullptr && dy != nullptr) {
     std::vector<const DenseTensor *> ins = {&dout, &y, &x};
-    GetGradXAndYOut<ElementwiseType::kTernary, T>(
-        dev_ctx,
-        place,
-        axis,
-        ins,
-        dout,
-        dx,
-        dy,
-        funcs::MultiplyGradXYFunctor<T, T>());
+    GetGradXAndYOut<T>(dev_ctx,
+                       place,
+                       axis,
+                       ins,
+                       dout,
+                       dx,
+                       dy,
+                       funcs::MultiplyGradXYFunctor<T, T>());
   } else if (dx != nullptr && dy == nullptr) {
     std::vector<const DenseTensor *> ins = {&dout, &y};
-    GetGradXOrYOut<ElementwiseType::kBinary, T>(
+    GetGradXOrYOut<T>(
         dev_ctx, place, axis, ins, dout, dx, funcs::MultiplyGradFunctor<T>());
   } else if (dx == nullptr && dy != nullptr) {
     std::vector<const DenseTensor *> ins = {&dout, &x};
-    GetGradXOrYOut<ElementwiseType::kBinary, T>(
+    GetGradXOrYOut<T>(
         dev_ctx, place, axis, ins, dout, dy, funcs::MultiplyGradFunctor<T>());
   }
 }
diff --git a/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu b/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu
index b69434c82da8c..efd15adc38346 100644
--- a/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu
@@ -37,22 +37,21 @@ void MaximumGradKernel(const Context& dev_ctx,
   int axis = -1;
   if (dx != nullptr && dy != nullptr) {
     std::vector<const DenseTensor*> ins = {&x, &y, &dout};
-    GetGradXAndYOut<ElementwiseType::kTernary, T>(
-        dev_ctx,
-        place,
-        axis,
-        ins,
-        dout,
-        dx,
-        dy,
-        funcs::MaxGradXYFunctor<T, T>());
+    GetGradXAndYOut<T>(dev_ctx,
+                       place,
+                       axis,
+                       ins,
+                       dout,
+                       dx,
+                       dy,
+                       funcs::MaxGradXYFunctor<T, T>());
   } else if (dx != nullptr && dy == nullptr) {
     std::vector<const DenseTensor*> ins = {&x, &y, &dout};
-    GetGradXOrYOut<ElementwiseType::kBinary, T>(
+    GetGradXOrYOut<T>(
         dev_ctx, place, axis, ins, dout, dx, funcs::MaxGradXFunctor<T>());
   } else if (dy != nullptr && dx == nullptr) {
     std::vector<const DenseTensor*> ins = {&x, &y, &dout};
-    GetGradXOrYOut<ElementwiseType::kTernary, T>(
+    GetGradXOrYOut<T>(
         dev_ctx, place, axis, ins, dout, dy, funcs::MaxGradYFunctor<T>());
   }
 }
@@ -68,22 +67,21 @@ void MinimumGradKernel(const Context& dev_ctx,
   int axis = -1;
   if (dx != nullptr && dy != nullptr) {
     std::vector<const DenseTensor*> ins = {&x, &y, &dout};
-    GetGradXAndYOut<ElementwiseType::kTernary, T>(
-        dev_ctx,
-        place,
-        axis,
-        ins,
-        dout,
-        dx,
-        dy,
-        funcs::MinGradXYFunctor<T, T>());
+    GetGradXAndYOut<T>(dev_ctx,
+                       place,
+                       axis,
+                       ins,
+                       dout,
+                       dx,
+                       dy,
+                       funcs::MinGradXYFunctor<T, T>());
   } else if (dx != nullptr && dy == nullptr) {
     std::vector<const DenseTensor*> ins = {&x, &y, &dout};
-    GetGradXOrYOut<ElementwiseType::kBinary, T>(
+    GetGradXOrYOut<T>(
         dev_ctx, place, axis, ins, dout, dx, funcs::MinGradXFunctor<T>());
   } else if (dy != nullptr && dx == nullptr) {
     std::vector<const DenseTensor*> ins = {&x, &y, &dout};
-    GetGradXOrYOut<ElementwiseType::kTernary, T>(
+    GetGradXOrYOut<T>(
         dev_ctx, place, axis, ins, dout, dy, funcs::MinGradYFunctor<T>());
   }
 }
diff --git a/paddle/phi/kernels/gpu/expand_as_kernel.cu b/paddle/phi/kernels/gpu/expand_as_kernel.cu
index f87a2417ced19..de47024f29581 100644
--- a/paddle/phi/kernels/gpu/expand_as_kernel.cu
+++ b/paddle/phi/kernels/gpu/expand_as_kernel.cu
@@ -74,8 +74,7 @@ void ExpandAsKernel(const Context& ctx,
   ctx.template Alloc<T>(out);
   std::vector<const DenseTensor*> ins = {&x};
   std::vector<DenseTensor*> outs = {out};
-  phi::funcs::BroadcastKernel<ElementwiseType::kUnary, T, T>(
-      ctx, ins, &outs, -1, kps::IdentityFunctor<T>());
+  phi::funcs::BroadcastKernel<T>(ctx, ins, &outs, kps::IdentityFunctor<T>());
 }
 
 }  // namespace phi
diff --git a/paddle/phi/kernels/gpu/expand_kernel.cu b/paddle/phi/kernels/gpu/expand_kernel.cu
index b2f973b0a8896..456aa9b3c5a34 100644
--- a/paddle/phi/kernels/gpu/expand_kernel.cu
+++ b/paddle/phi/kernels/gpu/expand_kernel.cu
@@ -73,8 +73,7 @@ void ExpandKernel(const Context& ctx,
   ctx.template Alloc<T>(out);
   std::vector<const DenseTensor*> ins = {&x};
   std::vector<DenseTensor*> outs = {out};
-  phi::funcs::BroadcastKernel<ElementwiseType::kUnary, T, T>(
-      ctx, ins, &outs, -1, kps::IdentityFunctor<T>());
+  phi::funcs::BroadcastKernel<T>(ctx, ins, &outs, kps::IdentityFunctor<T>());
 }
 
 }  // namespace phi
diff --git a/paddle/phi/kernels/gpu/matrix_rank_tol_kernel.cu b/paddle/phi/kernels/gpu/matrix_rank_tol_kernel.cu
index 620341f338eef..e4ee1f342131a 100644
--- a/paddle/phi/kernels/gpu/matrix_rank_tol_kernel.cu
+++ b/paddle/phi/kernels/gpu/matrix_rank_tol_kernel.cu
@@ -407,11 +407,10 @@ void MatrixRankTolKernel(const Context& dev_ctx,
   tol_tensor.Resize(dim_out);
   dev_ctx.template Alloc<T>(&tol_tensor);
 
-  funcs::ElementwiseCompute<GreaterElementFunctor<T>, T, T>(
+  funcs::ElementwiseCompute<GreaterElementFunctor<T>, T>(
       dev_ctx,
       atol_tensor,
       rtol_tensor,
-      -1,
       GreaterElementFunctor<T>(),
       &tol_tensor);
 
@@ -421,12 +420,10 @@ void MatrixRankTolKernel(const Context& dev_ctx,
   compare_result.Resize(detail::NewAxisDim(dim_out, k));
   dev_ctx.template Alloc<int64_t>(&compare_result);
 
-  int axis = -1;
   funcs::ElementwiseCompute<funcs::GreaterThanFunctor<T, int64_t>, T, int64_t>(
       dev_ctx,
       eigenvalue_tensor,
       tol_tensor,
-      axis,
       funcs::GreaterThanFunctor<T, int64_t>(),
       &compare_result);
 
diff --git a/paddle/phi/kernels/gpu/reduce_amin_amax_common.h b/paddle/phi/kernels/gpu/reduce_amin_amax_common.h
index 5ba779d027a38..04befb29b2de1 100644
--- a/paddle/phi/kernels/gpu/reduce_amin_amax_common.h
+++ b/paddle/phi/kernels/gpu/reduce_amin_amax_common.h
@@ -78,8 +78,8 @@ void ReduceCudaAMaxAMinGrad(const Context& dev_ctx,
   // 1. equal_out = Equal(x, y)
   std::vector<const phi::DenseTensor*> equal_inputs = {&new_y, new_in_tensor};
   std::vector<phi::DenseTensor*> equal_outputs = {&equal_out_tensor};
-  funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-      dev_ctx, equal_inputs, &equal_outputs, 0, funcs::EqualFunctor<T>());
+  funcs::BroadcastKernel<T>(
+      dev_ctx, equal_inputs, &equal_outputs, funcs::EqualFunctor<T>(), 0);
   // 2. equal_count = reduceSum(equal_out)
   using MPType = typename kps::details::MPTypeTrait<T>::Type;
   phi::funcs::
@@ -95,15 +95,15 @@ void ReduceCudaAMaxAMinGrad(const Context& dev_ctx,
   std::vector<const phi::DenseTensor*> mul_inputs = {&new_dout,
                                                      &equal_out_tensor};
   std::vector<phi::DenseTensor*> mul_outputs = {&equal_out_tensor};
-  funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-      dev_ctx, mul_inputs, &mul_outputs, 0, funcs::MultiplyFunctor<T>());
+  funcs::BroadcastKernel<T>(
+      dev_ctx, mul_inputs, &mul_outputs, funcs::MultiplyFunctor<T>(), 0);
 
   // 4. dx = Div(dx, equal_out)
   std::vector<const phi::DenseTensor*> grad_inputs = {&equal_out_tensor,
                                                       equal_count};
   std::vector<phi::DenseTensor*> grad_outputs = {new_dx_tensor};
-  funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-      dev_ctx, grad_inputs, &grad_outputs, 0, funcs::DivideFunctor<T>());
+  funcs::BroadcastKernel<T>(
+      dev_ctx, grad_inputs, &grad_outputs, funcs::DivideFunctor<T>(), 0);
   delete equal_out;
   delete equal_count;
 }
diff --git a/paddle/phi/kernels/gpu/reduce_grad.h b/paddle/phi/kernels/gpu/reduce_grad.h
index 01f91924645fa..7e01c1ae84391 100644
--- a/paddle/phi/kernels/gpu/reduce_grad.h
+++ b/paddle/phi/kernels/gpu/reduce_grad.h
@@ -28,7 +28,7 @@
 
 namespace phi {
 
-template <typename InT, typename Functor>
+template <typename Functor>
 void ReduceGrad(const GPUContext& dev_ctx,
                 DenseTensor* d_out,
                 DenseTensor* d_x,
@@ -36,14 +36,13 @@ void ReduceGrad(const GPUContext& dev_ctx,
                 Functor functor) {
   std::vector<const DenseTensor*> inputs = {d_out};
   std::vector<DenseTensor*> outputs = {d_x};
-  PD_VISIT_ALL_TYPES(
-      out_dtype, "BroadcastKernel", ([&] {
-        funcs::BroadcastKernel<phi::ElementwiseType::kUnary, InT, data_t>(
-            dev_ctx, inputs, &outputs, 0, functor);
-      }));
+  PD_VISIT_ALL_TYPES(out_dtype, "BroadcastKernel", ([&] {
+                       funcs::BroadcastKernel<data_t>(
+                           dev_ctx, inputs, &outputs, functor, 0);
+                     }));
 }
 
-template <typename T, typename OutT, typename Context, typename Functor>
+template <typename OutT, typename Context, typename Functor>
 void ReduceGradKernel(const Context& dev_ctx,
                       const DenseTensor& x,
                       const DenseTensor& out_grad,
@@ -79,8 +78,7 @@ void ReduceGradKernel(const Context& dev_ctx,
   auto pt_d_x = *d_x;
   std::vector<const DenseTensor*> inputs = {&pt_d_out};
   std::vector<DenseTensor*> outputs = {&pt_d_x};
-  funcs::BroadcastKernel<phi::ElementwiseType::kUnary, T, OutT>(
-      dev_ctx, inputs, &outputs, 0, functor);
+  funcs::BroadcastKernel<OutT>(dev_ctx, inputs, &outputs, functor, 0);
 }
 
 }  // namespace phi
diff --git a/paddle/phi/kernels/gpu/reduce_max_grad_kernel.cu b/paddle/phi/kernels/gpu/reduce_max_grad_kernel.cu
index 7b4472c522318..6bee38abe1fd0 100644
--- a/paddle/phi/kernels/gpu/reduce_max_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/reduce_max_grad_kernel.cu
@@ -62,14 +62,14 @@ void ReduceMaxGradKernel(const Context& dev_ctx,
   // 1. equal_out = Equal(x, y)
   std::vector<const phi::DenseTensor*> equal_inputs = {&new_out, &x};
   std::vector<phi::DenseTensor*> equal_outputs = {equal_out};
-  funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-      dev_ctx, equal_inputs, &equal_outputs, 0, funcs::EqualFunctor<T>());
+  funcs::BroadcastKernel<T>(
+      dev_ctx, equal_inputs, &equal_outputs, funcs::EqualFunctor<T>(), 0);
 
   // 2. dx = dout * 1
   std::vector<const phi::DenseTensor*> mul_inputs = {&new_out_grad, equal_out};
   std::vector<phi::DenseTensor*> mul_outputs = {x_grad};
-  funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-      dev_ctx, mul_inputs, &mul_outputs, 0, funcs::MultiplyFunctor<T>());
+  funcs::BroadcastKernel<T>(
+      dev_ctx, mul_inputs, &mul_outputs, funcs::MultiplyFunctor<T>(), 0);
   delete equal_out;
 }
 }  // namespace phi
diff --git a/paddle/phi/kernels/gpu/reduce_mean_grad_kernel.cu b/paddle/phi/kernels/gpu/reduce_mean_grad_kernel.cu
index 6519f6b08556e..0eac15902af04 100644
--- a/paddle/phi/kernels/gpu/reduce_mean_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/reduce_mean_grad_kernel.cu
@@ -53,8 +53,8 @@ void ReduceMeanGradKernel(const Context& dev_ctx,
   std::vector<DenseTensor*> outputs = {x_grad};
 
   using MPType = typename kps::details::MPTypeTrait<T>::Type;
-  funcs::BroadcastKernel<phi::ElementwiseType::kUnary, T, T>(
-      dev_ctx, inputs, &outputs, 0, kps::DivideFunctor<T, MPType>(reduce_num));
+  funcs::BroadcastKernel<T>(
+      dev_ctx, inputs, &outputs, kps::DivideFunctor<T, MPType>(reduce_num), 0);
 }
 
 }  // namespace phi
diff --git a/paddle/phi/kernels/gpu/reduce_min_grad_kernel.cu b/paddle/phi/kernels/gpu/reduce_min_grad_kernel.cu
index 86cccc5e03b1c..7a650dc9640a2 100644
--- a/paddle/phi/kernels/gpu/reduce_min_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/reduce_min_grad_kernel.cu
@@ -62,14 +62,14 @@ void ReduceMinGradKernel(const Context& dev_ctx,
   // 1. equal_out = Equal(x, y)
   std::vector<const phi::DenseTensor*> equal_inputs = {&new_out, &x};
   std::vector<phi::DenseTensor*> equal_outputs = {equal_out};
-  funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-      dev_ctx, equal_inputs, &equal_outputs, 0, funcs::EqualFunctor<T>());
+  funcs::BroadcastKernel<T>(
+      dev_ctx, equal_inputs, &equal_outputs, funcs::EqualFunctor<T>(), 0);
 
   // 2. dx = dout * 1
   std::vector<const phi::DenseTensor*> mul_inputs = {&new_out_grad, equal_out};
   std::vector<phi::DenseTensor*> mul_outputs = {x_grad};
-  funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-      dev_ctx, mul_inputs, &mul_outputs, 0, funcs::MultiplyFunctor<T>());
+  funcs::BroadcastKernel<T>(
+      dev_ctx, mul_inputs, &mul_outputs, funcs::MultiplyFunctor<T>(), 0);
   delete equal_out;
 }
 }  // namespace phi
diff --git a/paddle/phi/kernels/gpu/reduce_sum_grad_kernel.cu b/paddle/phi/kernels/gpu/reduce_sum_grad_kernel.cu
index 15215c05d6361..9ee6d530374db 100644
--- a/paddle/phi/kernels/gpu/reduce_sum_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/reduce_sum_grad_kernel.cu
@@ -48,7 +48,7 @@ void ReduceSumGradKernel(const Context& dev_ctx,
   // call ReduceGrad
   dev_ctx.Alloc(x_grad, x.dtype());
   using MPType = typename kps::details::MPTypeTrait<T>::Type;
-  phi::ReduceGrad<T, kps::IdentityFunctor<T, MPType>>(
+  phi::ReduceGrad<kps::IdentityFunctor<T, MPType>>(
       dev_ctx,
       &new_out_grad,
       x_grad,
diff --git a/paddle/phi/kernels/gpu/squared_l2_norm_grad_kernel.cu b/paddle/phi/kernels/gpu/squared_l2_norm_grad_kernel.cu
index 4557d44f15021..1d491b41ecb55 100644
--- a/paddle/phi/kernels/gpu/squared_l2_norm_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/squared_l2_norm_grad_kernel.cu
@@ -46,8 +46,7 @@ void SquaredL2NormGradKernel(const Context& dev_ctx,
   std::vector<const DenseTensor*> ins{&x, &dout};
   std::vector<DenseTensor*> outs{dx};
 
-  funcs::BroadcastKernel<ElementwiseType::kBinary, T, T>(
-      dev_ctx, ins, &outs, -1, phi::DoubleMulFunctor<T>());
+  funcs::BroadcastKernel<T>(dev_ctx, ins, &outs, phi::DoubleMulFunctor<T>());
 }
 }  // namespace phi
 
diff --git a/paddle/phi/kernels/gpu/tile_kernel.cu b/paddle/phi/kernels/gpu/tile_kernel.cu
index be825eea499c8..7861a2bdf01f8 100644
--- a/paddle/phi/kernels/gpu/tile_kernel.cu
+++ b/paddle/phi/kernels/gpu/tile_kernel.cu
@@ -78,8 +78,8 @@ void TileKernel(const Context& dev_ctx,
         tmp_out.Resize(make_ddim(vec_x_dims));
         dev_ctx.template Alloc<T>(&tmp_out);
         std::vector<DenseTensor*> outs = {&tmp_out};
-        phi::funcs::BroadcastKernel<ElementwiseType::kUnary, T, T>(
-            dev_ctx, ins, &outs, i, kps::IdentityFunctor<T>());
+        phi::funcs::BroadcastKernel<T>(
+            dev_ctx, ins, &outs, kps::IdentityFunctor<T>(), i);
         tmp_out.Resize(out_dims);
         new_x = tmp_out;
       }
@@ -89,8 +89,8 @@ void TileKernel(const Context& dev_ctx,
       out->Resize(make_ddim(vec_x_dims));
       dev_ctx.template Alloc<T>(out);
       std::vector<DenseTensor*> outs = {out};
-      phi::funcs::BroadcastKernel<ElementwiseType::kUnary, T, T>(
-          dev_ctx, ins, &outs, i, kps::IdentityFunctor<T>());
+      phi::funcs::BroadcastKernel<T>(
+          dev_ctx, ins, &outs, kps::IdentityFunctor<T>(), i);
       out->Resize(out_dims);
     }
   }
diff --git a/paddle/phi/kernels/gpu/viterbi_decode_kernel.cu b/paddle/phi/kernels/gpu/viterbi_decode_kernel.cu
index 0a78df87a8ca1..be630f85ce07d 100644
--- a/paddle/phi/kernels/gpu/viterbi_decode_kernel.cu
+++ b/paddle/phi/kernels/gpu/viterbi_decode_kernel.cu
@@ -91,8 +91,7 @@ struct BinaryOperation {
                   DenseTensor* output) {
     std::vector<const DenseTensor*> ins{&lhs, &rhs};
     std::vector<DenseTensor*> outs{output};
-    phi::funcs::BroadcastKernel<phi::ElementwiseType::kBinary, T, T>(
-        dev_ctx, ins, &outs, 0, BinaryFunctor<T>());
+    phi::funcs::BroadcastKernel<T>(dev_ctx, ins, &outs, BinaryFunctor<T>(), 0);
   }
 };
 
diff --git a/paddle/phi/kernels/impl/complex_kernel_impl.h b/paddle/phi/kernels/impl/complex_kernel_impl.h
index 8bd7823411964..ebbbda04a01c2 100644
--- a/paddle/phi/kernels/impl/complex_kernel_impl.h
+++ b/paddle/phi/kernels/impl/complex_kernel_impl.h
@@ -90,16 +90,16 @@ void ComplexKernel(const Context& dev_ctx,
 // facility functions
 #if defined(__NVCC__) || defined(__HIPCC__)
   phi::funcs::ElementwiseCompute<RealAndImagToComplexFunctor<T>, T, C>(
-      dev_ctx, x, y, /*axis*/ -1, RealAndImagToComplexFunctor<T>(), out);
+      dev_ctx, x, y, RealAndImagToComplexFunctor<T>(), out);
 #else
   auto x_dims = x.dims();
   auto y_dims = y.dims();
   if (x_dims.size() >= y_dims.size()) {
     phi::funcs::ElementwiseCompute<RealAndImagToComplexFunctor<T>, T, C>(
-        dev_ctx, x, y, /*axis*/ -1, RealAndImagToComplexFunctor<T>(), out);
+        dev_ctx, x, y, RealAndImagToComplexFunctor<T>(), out);
   } else {
     phi::funcs::ElementwiseCompute<ImagAndRealToComplexFunctor<T>, T, C>(
-        dev_ctx, x, y, /*axis*/ -1, ImagAndRealToComplexFunctor<T>(), out);
+        dev_ctx, x, y, ImagAndRealToComplexFunctor<T>(), out);
   }
 #endif
 }
diff --git a/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h b/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h
index 1d069775f225d..15f99a58fa5a5 100644
--- a/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h
@@ -76,15 +76,15 @@ void AddDoubleGradImpl(const Context& dev_ctx,
     auto ddy_dims = ddy_safe.dims();
     if (ddx_dims.size() >= ddy_dims.size()) {
       funcs::ElementwiseCompute<funcs::AddFunctor<T>, T>(
-          dev_ctx, ddx_safe, ddy_safe, axis, funcs::AddFunctor<T>(), ddout);
+          dev_ctx, ddx_safe, ddy_safe, funcs::AddFunctor<T>(), ddout, axis);
     } else {
       funcs::ElementwiseCompute<funcs::InverseAddFunctor<T>, T>(
           dev_ctx,
           ddx_safe,
           ddy_safe,
-          axis,
           funcs::InverseAddFunctor<T>(),
-          ddout);
+          ddout,
+          axis);
     }
   }
 }
@@ -107,7 +107,7 @@ void SubtractDoubleGradImpl(const Context& dev_ctx,
 
     dev_ctx.template Alloc<T>(ddout);
     funcs::ElementwiseCompute<funcs::SubtractFunctor<T>, T>(
-        dev_ctx, ddx_safe, ddy_safe, axis, funcs::SubtractFunctor<T>(), ddout);
+        dev_ctx, ddx_safe, ddy_safe, funcs::SubtractFunctor<T>(), ddout, axis);
   }
 }
 
diff --git a/paddle/phi/kernels/impl/elementwise_kernel_impl.h b/paddle/phi/kernels/impl/elementwise_kernel_impl.h
index 5a30fec36c3b0..0121f35b3cecb 100644
--- a/paddle/phi/kernels/impl/elementwise_kernel_impl.h
+++ b/paddle/phi/kernels/impl/elementwise_kernel_impl.h
@@ -39,10 +39,10 @@ namespace phi {
       auto y_dims = y.dims();                                               \
       if (x_dims.size() >= y_dims.size()) {                                 \
         funcs::ElementwiseCompute<funcs::name##Functor<T>, T>(              \
-            dev_ctx, x, y, axis, funcs::name##Functor<T>(), out);           \
+            dev_ctx, x, y, funcs::name##Functor<T>(), out, axis);           \
       } else {                                                              \
         funcs::ElementwiseCompute<funcs::Inverse##name##Functor<T>, T>(     \
-            dev_ctx, x, y, axis, funcs::Inverse##name##Functor<T>(), out);  \
+            dev_ctx, x, y, funcs::Inverse##name##Functor<T>(), out, axis);  \
       }                                                                     \
     }                                                                       \
   }
@@ -62,8 +62,8 @@ namespace phi {
     inputs.emplace_back(&y);                                         \
     outputs.emplace_back(out);                                       \
     dev_ctx.template Alloc<T>(out);                                  \
-    funcs::BroadcastKernel<ElementwiseType::kBinary, T, T>(          \
-        dev_ctx, inputs, &outputs, axis, funcs::name##Functor<T>()); \
+    funcs::BroadcastKernel<T>(                                       \
+        dev_ctx, inputs, &outputs, funcs::name##Functor<T>(), axis); \
   }
 
 template <typename T, typename Context>
@@ -72,8 +72,8 @@ void FMaxKernel(const Context& dev_ctx,
                 const DenseTensor& y,
                 DenseTensor* out) {
   dev_ctx.template Alloc<T>(out);
-  funcs::ElementwiseCompute<funcs::FMaxFunctor<T>, T, T>(
-      dev_ctx, x, y, -1, funcs::FMaxFunctor<T>(), out);
+  funcs::ElementwiseCompute<funcs::FMaxFunctor<T>, T>(
+      dev_ctx, x, y, funcs::FMaxFunctor<T>(), out);
 }
 
 template <typename T, typename Context>
@@ -82,8 +82,8 @@ void FMinKernel(const Context& dev_ctx,
                 const DenseTensor& y,
                 DenseTensor* out) {
   dev_ctx.template Alloc<T>(out);
-  funcs::ElementwiseCompute<funcs::FMinFunctor<T>, T, T>(
-      dev_ctx, x, y, -1, funcs::FMinFunctor<T>(), out);
+  funcs::ElementwiseCompute<funcs::FMinFunctor<T>, T>(
+      dev_ctx, x, y, funcs::FMinFunctor<T>(), out);
 }
 
 }  // namespace phi
diff --git a/paddle/phi/kernels/impl/lu_kernel_impl.h b/paddle/phi/kernels/impl/lu_kernel_impl.h
index 5315e36b47172..5663484362a8e 100644
--- a/paddle/phi/kernels/impl/lu_kernel_impl.h
+++ b/paddle/phi/kernels/impl/lu_kernel_impl.h
@@ -153,12 +153,8 @@ void SetValueCompute(const Context& dev_ctx,
   slice_tensor.Resize(slice_dims_for_assign);
   if (value_tensor != nullptr) {
     CheckIsDimsMatch(slice_dims_for_assign, value_tensor->dims());
-    phi::funcs::ElementwiseCompute<SubFunctor<T>, T, T>(dev_ctx,
-                                                        slice_tensor,
-                                                        *value_tensor,
-                                                        -1,
-                                                        SubFunctor<T>(),
-                                                        &slice_tensor);
+    phi::funcs::ElementwiseCompute<SubFunctor<T>, T>(
+        dev_ctx, slice_tensor, *value_tensor, SubFunctor<T>(), &slice_tensor);
   } else {
     DenseTensor value_t(dtype);
     auto value_dims = phi::make_ddim(shape);
@@ -166,8 +162,8 @@ void SetValueCompute(const Context& dev_ctx,
 
     value_t.Resize(value_dims);
     dev_ctx.template Alloc<T>(&value_t);
-    phi::funcs::ElementwiseCompute<SubFunctor<T>, T, T>(
-        dev_ctx, slice_tensor, value_t, -1, SubFunctor<T>(), &slice_tensor);
+    phi::funcs::ElementwiseCompute<SubFunctor<T>, T>(
+        dev_ctx, slice_tensor, value_t, SubFunctor<T>(), &slice_tensor);
   }
   slice_tensor.Resize(slice_dims);
 
diff --git a/paddle/phi/kernels/impl/set_value_kernel_impl.h b/paddle/phi/kernels/impl/set_value_kernel_impl.h
index 9956260a79836..2c545ac06ada1 100644
--- a/paddle/phi/kernels/impl/set_value_kernel_impl.h
+++ b/paddle/phi/kernels/impl/set_value_kernel_impl.h
@@ -204,7 +204,6 @@ void SetValueImpl(const Context& dev_ctx,
         dev_ctx,
         slice_tensor,
         value,
-        -1,
         funcs::SubtractFunctor<T>(),
         &slice_tensor);
   } else {
@@ -212,7 +211,6 @@ void SetValueImpl(const Context& dev_ctx,
         dev_ctx,
         slice_tensor,
         value,
-        -1,
         funcs::InverseSubtractFunctor<T>(),
         &slice_tensor);
   }
diff --git a/paddle/phi/kernels/kps/bitwise_kernel.cu b/paddle/phi/kernels/kps/bitwise_kernel.cu
index 285b18927af80..fcdc7c95e9151 100644
--- a/paddle/phi/kernels/kps/bitwise_kernel.cu
+++ b/paddle/phi/kernels/kps/bitwise_kernel.cu
@@ -25,18 +25,17 @@ limitations under the License. */
 #include "paddle/phi/kernels/funcs/broadcast_function.h"
 namespace phi {
 
-#define DEFINE_BITWISE_KERNEL(op_type)                      \
-  template <typename T, typename Context>                   \
-  void Bitwise##op_type##Kernel(const Context& dev_ctx,     \
-                                const DenseTensor& x,       \
-                                const DenseTensor& y,       \
-                                DenseTensor* out) {         \
-    dev_ctx.template Alloc<T>(out);                         \
-    funcs::Bitwise##op_type##Functor<T> func;               \
-    std::vector<const DenseTensor*> ins = {&x, &y};         \
-    std::vector<DenseTensor*> outs = {out};                 \
-    funcs::BroadcastKernel<ElementwiseType::kBinary, T, T>( \
-        dev_ctx, ins, &outs, -1, func);                     \
+#define DEFINE_BITWISE_KERNEL(op_type)                    \
+  template <typename T, typename Context>                 \
+  void Bitwise##op_type##Kernel(const Context& dev_ctx,   \
+                                const DenseTensor& x,     \
+                                const DenseTensor& y,     \
+                                DenseTensor* out) {       \
+    dev_ctx.template Alloc<T>(out);                       \
+    funcs::Bitwise##op_type##Functor<T> func;             \
+    std::vector<const DenseTensor*> ins = {&x, &y};       \
+    std::vector<DenseTensor*> outs = {out};               \
+    funcs::BroadcastKernel<T>(dev_ctx, ins, &outs, func); \
   }
 
 DEFINE_BITWISE_KERNEL(And)
diff --git a/paddle/phi/kernels/kps/compare_kernel.cu b/paddle/phi/kernels/kps/compare_kernel.cu
index 5397bf41efb4a..50de82cd004aa 100644
--- a/paddle/phi/kernels/kps/compare_kernel.cu
+++ b/paddle/phi/kernels/kps/compare_kernel.cu
@@ -55,8 +55,7 @@ inline void CompareKernelImpl(const Context& ctx,
   ctx.template Alloc<bool>(out);
   std::vector<const DenseTensor*> ins{&x, &y};
   std::vector<DenseTensor*> outs{out};
-  funcs::BroadcastKernel<ElementwiseType::kBinary, T, bool>(
-      ctx, ins, &outs, axis, Functor());
+  funcs::BroadcastKernel<bool>(ctx, ins, &outs, Functor(), axis);
 }
 
 #ifndef PADDLE_WITH_XPU_KP
diff --git a/paddle/phi/kernels/kps/elementwise_kernel.cu b/paddle/phi/kernels/kps/elementwise_kernel.cu
index 550e579ac1159..e56cb4d845911 100644
--- a/paddle/phi/kernels/kps/elementwise_kernel.cu
+++ b/paddle/phi/kernels/kps/elementwise_kernel.cu
@@ -72,8 +72,8 @@ void HeavisideKernel(const Context& dev_ctx,
   inputs.emplace_back(&y);
   outputs.emplace_back(out);
   dev_ctx.template Alloc<T>(out);
-  funcs::BroadcastKernel<ElementwiseType::kBinary, T, T>(
-      dev_ctx, inputs, &outputs, -1, funcs::ElementwiseHeavisideFunctor<T>());
+  funcs::BroadcastKernel<T>(
+      dev_ctx, inputs, &outputs, funcs::ElementwiseHeavisideFunctor<T>());
 }
 
 template <typename T, typename Context>
diff --git a/paddle/phi/kernels/kps/logical_kernel.cu b/paddle/phi/kernels/kps/logical_kernel.cu
index 570ac0e676ecb..cd9e12fe36777 100755
--- a/paddle/phi/kernels/kps/logical_kernel.cu
+++ b/paddle/phi/kernels/kps/logical_kernel.cu
@@ -25,20 +25,17 @@
 
 namespace phi {
 
-#define DEFINE_LOGICAL_BINARY_KERNEL(type)                               \
-  template <typename T, typename Context>                                \
-  void Logical##type##Kernel(const Context& dev_ctx,                     \
-                             const DenseTensor& x,                       \
-                             const DenseTensor& y,                       \
-                             DenseTensor* out) {                         \
-    using InT = typename funcs::Logical##type##Functor<T>::ELEMENT_TYPE; \
-    using OutT = bool;                                                   \
-    dev_ctx.template Alloc<bool>(out);                                   \
-    funcs::Logical##type##Functor<T> binary_func;                        \
-    std::vector<const DenseTensor*> ins = {&x, &y};                      \
-    std::vector<DenseTensor*> outs = {out};                              \
-    funcs::BroadcastKernel<ElementwiseType::kBinary, InT, OutT>(         \
-        dev_ctx, ins, &outs, -1, binary_func);                           \
+#define DEFINE_LOGICAL_BINARY_KERNEL(type)                          \
+  template <typename T, typename Context>                           \
+  void Logical##type##Kernel(const Context& dev_ctx,                \
+                             const DenseTensor& x,                  \
+                             const DenseTensor& y,                  \
+                             DenseTensor* out) {                    \
+    dev_ctx.template Alloc<bool>(out);                              \
+    funcs::Logical##type##Functor<T> binary_func;                   \
+    std::vector<const DenseTensor*> ins = {&x, &y};                 \
+    std::vector<DenseTensor*> outs = {out};                         \
+    funcs::BroadcastKernel<bool>(dev_ctx, ins, &outs, binary_func); \
   }
 
 DEFINE_LOGICAL_BINARY_KERNEL(And)
@@ -50,15 +47,11 @@ template <typename T, typename Context>
 void LogicalNotKernel(const Context& dev_ctx,
                       const DenseTensor& x,
                       DenseTensor* out) {
-  using InT = typename funcs::LogicalNotFunctor<T>::ELEMENT_TYPE;
-  using OutT = bool;
-
   dev_ctx.template Alloc<bool>(out);
   funcs::LogicalNotFunctor<T> unary_func;
   std::vector<const DenseTensor*> ins = {&x};
   std::vector<DenseTensor*> outs = {out};
-  funcs::BroadcastKernel<ElementwiseType::kUnary, InT, OutT>(
-      dev_ctx, ins, &outs, -1, unary_func);
+  funcs::BroadcastKernel<bool>(dev_ctx, ins, &outs, unary_func);
 }
 
 }  // namespace phi
diff --git a/paddle/phi/kernels/legacy/cpu/elementwise_kernel.cc b/paddle/phi/kernels/legacy/cpu/elementwise_kernel.cc
index a976cb2a00933..6d051863a8e41 100644
--- a/paddle/phi/kernels/legacy/cpu/elementwise_kernel.cc
+++ b/paddle/phi/kernels/legacy/cpu/elementwise_kernel.cc
@@ -30,7 +30,7 @@ void MaximumRawKernel(const Context& dev_ctx,
   // allocate memory for out
   dev_ctx.template Alloc<T>(out);
   funcs::ElementwiseCompute<funcs::MaximumFunctor<T>, T>(
-      dev_ctx, x, y, axis, funcs::MaximumFunctor<T>(), out);
+      dev_ctx, x, y, funcs::MaximumFunctor<T>(), out, axis);
 }
 
 template <typename T, typename Context>
@@ -42,7 +42,7 @@ void MinimumRawKernel(const Context& dev_ctx,
   // allocate memory for out
   dev_ctx.template Alloc<T>(out);
   funcs::ElementwiseCompute<funcs::MinimumFunctor<T>, T>(
-      dev_ctx, x, y, axis, funcs::MinimumFunctor<T>(), out);
+      dev_ctx, x, y, funcs::MinimumFunctor<T>(), out, axis);
 }
 
 template <typename T, typename Context>
@@ -57,10 +57,10 @@ void RemainderRawKernel(const Context& dev_ctx,
   auto y_dims = y.dims();
   if (x_dims.size() >= y_dims.size()) {
     funcs::ElementwiseCompute<funcs::RemainderFunctor<T>, T>(
-        dev_ctx, x, y, axis, funcs::RemainderFunctor<T>(), out);
+        dev_ctx, x, y, funcs::RemainderFunctor<T>(), out, axis);
   } else {
     funcs::ElementwiseCompute<funcs::InverseRemainderFunctor<T>, T>(
-        dev_ctx, x, y, axis, funcs::InverseRemainderFunctor<T>(), out);
+        dev_ctx, x, y, funcs::InverseRemainderFunctor<T>(), out, axis);
   }
 }
 
@@ -76,10 +76,10 @@ void FloorDivideRawKernel(const Context& dev_ctx,
   auto y_dims = y.dims();
   if (x_dims.size() >= y_dims.size()) {
     funcs::ElementwiseCompute<funcs::FloorDivideFunctor<T>, T>(
-        dev_ctx, x, y, axis, funcs::FloorDivideFunctor<T>(), out);
+        dev_ctx, x, y, funcs::FloorDivideFunctor<T>(), out, axis);
   } else {
     funcs::ElementwiseCompute<funcs::InverseFloorDivideFunctor<T>, T>(
-        dev_ctx, x, y, axis, funcs::InverseFloorDivideFunctor<T>(), out);
+        dev_ctx, x, y, funcs::InverseFloorDivideFunctor<T>(), out, axis);
   }
 }
 
@@ -95,10 +95,10 @@ void ElementwisePowRawKernel(const Context& dev_ctx,
   auto y_dims = y.dims();
   if (x_dims.size() >= y_dims.size()) {
     funcs::ElementwiseCompute<funcs::ElementwisePowFunctor<T>, T>(
-        dev_ctx, x, y, axis, funcs::ElementwisePowFunctor<T>(), out);
+        dev_ctx, x, y, funcs::ElementwisePowFunctor<T>(), out, axis);
   } else {
     funcs::ElementwiseCompute<funcs::ElementwiseInversePowFunctor<T>, T>(
-        dev_ctx, x, y, axis, funcs::ElementwiseInversePowFunctor<T>(), out);
+        dev_ctx, x, y, funcs::ElementwiseInversePowFunctor<T>(), out, axis);
   }
 }
 
diff --git a/paddle/phi/kernels/legacy/kps/elementwise_raw_kernel.cu b/paddle/phi/kernels/legacy/kps/elementwise_raw_kernel.cu
index 95cf5d4333e8d..8fab237f0a5ee 100644
--- a/paddle/phi/kernels/legacy/kps/elementwise_raw_kernel.cu
+++ b/paddle/phi/kernels/legacy/kps/elementwise_raw_kernel.cu
@@ -36,8 +36,8 @@ void MaximumRawKernel(const Context& dev_ctx,
   inputs.emplace_back(&y);
   outputs.emplace_back(out);
   dev_ctx.template Alloc<T>(out);
-  funcs::BroadcastKernel<ElementwiseType::kBinary, T, T>(
-      dev_ctx, inputs, &outputs, axis, funcs::MaximumFunctor<T>());
+  funcs::BroadcastKernel<T>(
+      dev_ctx, inputs, &outputs, funcs::MaximumFunctor<T>(), axis);
 }
 
 template <typename T, typename Context>
@@ -54,8 +54,8 @@ void MinimumRawKernel(const Context& dev_ctx,
   inputs.emplace_back(&y);
   outputs.emplace_back(out);
   dev_ctx.template Alloc<T>(out);
-  funcs::BroadcastKernel<ElementwiseType::kBinary, T, T>(
-      dev_ctx, inputs, &outputs, axis, funcs::MinimumFunctor<T>());
+  funcs::BroadcastKernel<T>(
+      dev_ctx, inputs, &outputs, funcs::MinimumFunctor<T>(), axis);
 }
 
 template <typename T, typename Context>
@@ -72,8 +72,8 @@ void RemainderRawKernel(const Context& dev_ctx,
   inputs.emplace_back(&y);
   outputs.emplace_back(out);
   dev_ctx.template Alloc<T>(out);
-  funcs::BroadcastKernel<ElementwiseType::kBinary, T, T>(
-      dev_ctx, inputs, &outputs, axis, funcs::RemainderFunctor<T>());
+  funcs::BroadcastKernel<T>(
+      dev_ctx, inputs, &outputs, funcs::RemainderFunctor<T>(), axis);
 }
 
 template <typename T, typename Context>
@@ -90,8 +90,8 @@ void FloorDivideRawKernel(const Context& dev_ctx,
   inputs.emplace_back(&y);
   outputs.emplace_back(out);
   dev_ctx.template Alloc<T>(out);
-  funcs::BroadcastKernel<ElementwiseType::kBinary, T, T>(
-      dev_ctx, inputs, &outputs, axis, funcs::FloorDivideFunctor<T>());
+  funcs::BroadcastKernel<T>(
+      dev_ctx, inputs, &outputs, funcs::FloorDivideFunctor<T>(), axis);
 }
 
 template <typename T, typename Context>
@@ -108,8 +108,8 @@ void ElementwisePowRawKernel(const Context& dev_ctx,
   inputs.emplace_back(&y);
   outputs.emplace_back(out);
   dev_ctx.template Alloc<T>(out);
-  funcs::BroadcastKernel<ElementwiseType::kBinary, T, T>(
-      dev_ctx, inputs, &outputs, axis, funcs::ElementwisePowFunctor<T>());
+  funcs::BroadcastKernel<T>(
+      dev_ctx, inputs, &outputs, funcs::ElementwisePowFunctor<T>(), axis);
 }
 
 }  // namespace phi
@@ -174,4 +174,5 @@ PD_REGISTER_KERNEL(elementwise_pow_raw,
                    float16,
                    int64_t,
                    bfloat16) {}
+
 #endif
diff --git a/test/cpp/phi/kernels/test_ternary_broadcast.cu b/test/cpp/phi/kernels/test_ternary_broadcast.cu
index d1faa89a1ed5b..09598e637909a 100644
--- a/test/cpp/phi/kernels/test_ternary_broadcast.cu
+++ b/test/cpp/phi/kernels/test_ternary_broadcast.cu
@@ -89,8 +89,7 @@ void TestCase(const phi::GPUContext& dev_ctx,
       d_in1.get(), d_in2.get(), d_in3.get()};
   std::vector<phi::DenseTensor*> outputs{d_out.get()};
   for (int i = 0; i < times; ++i) {
-    phi::funcs::BroadcastKernel<phi::ElementwiseType::kTernary, T, T>(
-        dev_ctx, inputs, &outputs, -1, compute);
+    phi::funcs::BroadcastKernel<T>(dev_ctx, inputs, &outputs, compute);
   }
   dev_ctx.Wait();
 }

From 09f8e31dbcc43c853201a755aea653a3003b2029 Mon Sep 17 00:00:00 2001
From: HongyuJia <jiahongyu@baidu.com>
Date: Fri, 28 Apr 2023 20:55:23 +0800
Subject: [PATCH 145/405] [CINN Support 0D-Tensor] CINN hack squeeze2 with
 trick temporarily (#53454)

---
 .../cinn_zero_tensor_trick_pass.cc            | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.cc b/paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.cc
index 9c4e6192be424..de84742146cab 100644
--- a/paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.cc
+++ b/paddle/fluid/framework/paddle2cinn/cinn_zero_tensor_trick_pass.cc
@@ -32,6 +32,32 @@ void CinnZeroTensorTrickPass::ApplyImpl(ir::Graph* graph) const {
                                                           "assign_value",
                                                           "gaussian_random",
                                                           "set_value"};
+  // NOTE: Hack squeeze2 0D-Tensor input
+  // If squeeze2 inputs 0D-Tensor and axes, The 0D-Tensor's shape will convert
+  // to 1D-Tensor, which could lead error. We hack squeeze2's axes attribute to
+  // resolve this. Change 0D-Tensor input to 1D-Tensor input and then make
+  // axes->axes[: -1]
+  for (const ir::Node* n : graph->Nodes()) {
+    if (n->IsOp() && n->Op()->Type() == "unsqueeze2") {
+      if (n->Op()->HasAttr("axes")) {
+        auto axes =
+            PADDLE_GET_CONST(std::vector<int32_t>, n->Op()->GetAttr("axes"));
+        for (const ir::Node* var : n->inputs) {
+          if (var->Var() &&
+              var->Var()->GetType() == proto::VarType::LOD_TENSOR) {
+            std::vector<int64_t> shape = var->Var()->GetShape();
+            if (shape.empty()) {
+              axes.pop_back();
+              n->Op()->SetAttr("axes", axes);
+              VLOG(4) << "unsqueeze2 axes dims is full, fix dim -> dim[:-1] to "
+                         "avoid 0D-Tensor input error";
+            }
+          }
+        }
+      }
+    }
+  }
+
   for (const ir::Node* n : graph->Nodes()) {
     if (n->IsOp() && op_cases_fix_attr.count(n->Op()->Type())) {
       if (n->Op()->HasAttr("shape")) {

From 7757087408f96028fad65f8d8b4afb8a77f15f68 Mon Sep 17 00:00:00 2001
From: zhurou603 <32234672+zhurou603@users.noreply.github.com>
Date: Fri, 28 Apr 2023 21:10:16 +0800
Subject: [PATCH 146/405] support profiler Distributed Summary for CustomDevice
 (#53405)

---
 python/paddle/profiler/profiler_statistic.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/python/paddle/profiler/profiler_statistic.py b/python/paddle/profiler/profiler_statistic.py
index 69fd3c15130e3..df1cfe49cd9d2 100755
--- a/python/paddle/profiler/profiler_statistic.py
+++ b/python/paddle/profiler/profiler_statistic.py
@@ -472,7 +472,11 @@ def parse(self, nodetrees):
                     for runtimenode in hostnode.runtime_node:
                         for devicenode in runtimenode.device_node:
                             if devicenode.type == TracerEventType.Kernel:
-                                if 'nccl' in devicenode.name.lower():
+                                kernel_name = devicenode.name.lower()
+                                if (
+                                    'nccl' in kernel_name
+                                    or 'xccl' in kernel_name
+                                ):
                                     self.gpu_communication_range.append(
                                         (devicenode.start_ns, devicenode.end_ns)
                                     )

From 0c2ab714befeaac35a3df5d92cfe7cb1631ce716 Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Sat, 29 Apr 2023 00:42:15 +0800
Subject: [PATCH 147/405] [tests]mv fluid benchmark to tests (#53426)

* [tests]mv fluid benchmark to tests

* [test]Add placeholder

* [test]Add placeholder
---
 paddle/fluid/operators/CMakeLists.txt         |  1 -
 test/cpp/fluid/CMakeLists.txt                 | 20 +++++++++++++++++++
 .../cpp/fluid}/benchmark/CMakeLists.txt       |  0
 .../cpp/fluid}/benchmark/op_tester.cc         |  2 +-
 .../cpp/fluid}/benchmark/op_tester.h          |  2 +-
 .../cpp/fluid}/benchmark/op_tester_config.cc  |  2 +-
 .../cpp/fluid}/benchmark/op_tester_config.h   |  0
 7 files changed, 23 insertions(+), 4 deletions(-)
 rename {paddle/fluid/operators => test/cpp/fluid}/benchmark/CMakeLists.txt (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/benchmark/op_tester.cc (99%)
 rename {paddle/fluid/operators => test/cpp/fluid}/benchmark/op_tester.h (97%)
 rename {paddle/fluid/operators => test/cpp/fluid}/benchmark/op_tester_config.cc (92%)
 rename {paddle/fluid/operators => test/cpp/fluid}/benchmark/op_tester_config.h (100%)

diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt
index e4add9ae4bd3c..984dacfa8bf88 100644
--- a/paddle/fluid/operators/CMakeLists.txt
+++ b/paddle/fluid/operators/CMakeLists.txt
@@ -194,7 +194,6 @@ if (WITH_PYTHON)
 endif()
 
 set(GLOB_OP_LIB ${OP_LIBRARY} CACHE INTERNAL "Global OP library")
-add_subdirectory(benchmark)
 
 cc_test_old(op_debug_string_test SRCS op_debug_string_test.cc DEPS elementwise_add_op ${COMMON_OP_DEPS})
 
diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt
index e2d08eac64b73..cabad612e0d07 100644
--- a/test/cpp/fluid/CMakeLists.txt
+++ b/test/cpp/fluid/CMakeLists.txt
@@ -1,3 +1,23 @@
+add_subdirectory(benchmark)
 if(WITH_CINN)
   add_subdirectory(cinn)
 endif()
+# add_subdirectory(controlflow)
+# add_subdirectory(detection)
+# add_subdirectory(dlnne)
+# add_subdirectory(elementwise)
+# add_subdirectory(fused)
+# if(WITH_LITE)
+#   add_subdirectory(lite)
+# endif()
+# add_subdirectory(math)
+# if(WITH_MKLDNN)
+#   add_subdirectory(mkldnn)
+# endif()
+# add_subdirectory(nccl)
+# if(WITH_PSCORE)
+#   add_subdirectory(pscore)
+# endif()
+# add_subdirectory(prim_ops)
+# add_subdirectory(reduce_ops)
+# add_subdirectory(tensorrt)
diff --git a/paddle/fluid/operators/benchmark/CMakeLists.txt b/test/cpp/fluid/benchmark/CMakeLists.txt
similarity index 100%
rename from paddle/fluid/operators/benchmark/CMakeLists.txt
rename to test/cpp/fluid/benchmark/CMakeLists.txt
diff --git a/paddle/fluid/operators/benchmark/op_tester.cc b/test/cpp/fluid/benchmark/op_tester.cc
similarity index 99%
rename from paddle/fluid/operators/benchmark/op_tester.cc
rename to test/cpp/fluid/benchmark/op_tester.cc
index 20c6b848248ff..482cdf4348124 100644
--- a/paddle/fluid/operators/benchmark/op_tester.cc
+++ b/test/cpp/fluid/benchmark/op_tester.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/operators/benchmark/op_tester.h"
+#include "test/cpp/fluid/benchmark/op_tester.h"
 
 #include <fstream>
 
diff --git a/paddle/fluid/operators/benchmark/op_tester.h b/test/cpp/fluid/benchmark/op_tester.h
similarity index 97%
rename from paddle/fluid/operators/benchmark/op_tester.h
rename to test/cpp/fluid/benchmark/op_tester.h
index 2135febc47f32..de8f62cfe07cd 100644
--- a/paddle/fluid/operators/benchmark/op_tester.h
+++ b/test/cpp/fluid/benchmark/op_tester.h
@@ -21,8 +21,8 @@ limitations under the License. */
 
 #include "paddle/fluid/framework/op_desc.h"
 #include "paddle/fluid/framework/operator.h"
-#include "paddle/fluid/operators/benchmark/op_tester_config.h"
 #include "paddle/phi/core/ddim.h"
+#include "test/cpp/fluid/benchmark/op_tester_config.h"
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/benchmark/op_tester_config.cc b/test/cpp/fluid/benchmark/op_tester_config.cc
similarity index 92%
rename from paddle/fluid/operators/benchmark/op_tester_config.cc
rename to test/cpp/fluid/benchmark/op_tester_config.cc
index a7370e30f2f65..ba9997e846afe 100644
--- a/paddle/fluid/operators/benchmark/op_tester_config.cc
+++ b/test/cpp/fluid/benchmark/op_tester_config.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/operators/benchmark/op_tester_config.h"
+#include "test/cpp/fluid/benchmark/op_tester_config.h"
 
 #include <fstream>
 
diff --git a/paddle/fluid/operators/benchmark/op_tester_config.h b/test/cpp/fluid/benchmark/op_tester_config.h
similarity index 100%
rename from paddle/fluid/operators/benchmark/op_tester_config.h
rename to test/cpp/fluid/benchmark/op_tester_config.h

From 0271d9e7f9fca474193bc11e9cc26ed1f566c588 Mon Sep 17 00:00:00 2001
From: HydrogenSulfate <490868991@qq.com>
Date: Sun, 30 Apr 2023 03:09:28 +0800
Subject: [PATCH 148/405] fix test_autograd_dynamic (#53473)

Co-authored-by: zhwesky2010 <zhouwei25@baidu.com>
---
 test/autograd/test_autograd_dynamic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/autograd/test_autograd_dynamic.py b/test/autograd/test_autograd_dynamic.py
index e8e5d8b626c5f..2e6033a6ee937 100644
--- a/test/autograd/test_autograd_dynamic.py
+++ b/test/autograd/test_autograd_dynamic.py
@@ -630,7 +630,7 @@ def func(x):
 
     def func_ndim_bigger_than_2(self):
         def func(x):
-            return (x * x).sum()
+            return (x * x).sum([1, 2, 3])
 
         with self.assertRaises(ValueError):
             x = paddle.ones([3, 3, 3, 3])

From ddf94ae4adaad7e20cb117b962dc8901e47e0e06 Mon Sep 17 00:00:00 2001
From: zhwesky2010 <1183042833@qq.com>
Date: Sun, 30 Apr 2023 14:01:08 +0800
Subject: [PATCH 149/405] [Zero-Dim] Support paddle.sum/mean/loss api output
 0D,test=allcase (#52739)

---
 .../composite_backward_api.h                  |   2 +-
 paddle/phi/infermeta/unary.cc                 |   7 +-
 .../distributed/auto_parallel/completion.py   |  20 ++-
 .../distributed/auto_parallel/engine.py       |   2 +-
 .../auto_parallel/operators/common.py         |   2 +-
 .../auto_parallel/operators/dist_default.py   |   4 +-
 .../auto_parallel/operators/dist_eltwise.py   |   4 +-
 .../auto_parallel/operators/dist_embedding.py |   2 +-
 .../auto_parallel/operators/dist_matmul.py    |  18 +--
 .../auto_parallel/operators/dist_reshape.py   |  12 +-
 .../auto_parallel/operators/dist_scale.py     |   4 +-
 .../auto_parallel/operators/dist_softmax.py   |   4 +-
 .../auto_parallel/operators/dist_transpose.py |   4 +-
 .../auto_parallel/tuner/rule_based_tuner.py   |  12 +-
 .../paddle/distributed/auto_parallel/utils.py |  13 +-
 .../hybrid_parallel_optimizer.py              |  49 ++++---
 .../sharding/group_sharded_utils.py           |  55 ++++----
 .../distributed/fleet/metrics/metric.py       |   6 +-
 .../distributed/passes/auto_parallel_amp.py   |  46 +++++--
 .../distributed/passes/auto_parallel_fp16.py  |   9 +-
 .../passes/auto_parallel_grad_clip.py         |   4 +-
 .../passes/auto_parallel_quantization.py      |   4 +-
 .../tests/unittests/check_nan_inf_base.py     |   6 +-
 .../tests/unittests/seresnext_test_base.py    |  14 +-
 .../fluid/tests/unittests/test_argsort_op.py  |   5 +-
 .../paddle/fluid/tests/unittests/test_cond.py |   2 +-
 .../unittests/test_cosine_embedding_loss.py   |   8 +-
 .../test_eager_deletion_recurrent_op.py       |   4 +-
 .../unittests/test_fetch_lod_tensor_array.py  |   8 +-
 .../unittests/test_fuse_all_reduce_pass.py    |  10 +-
 .../tests/unittests/test_fuse_bn_act_pass.py  |   4 +-
 .../unittests/test_fuse_bn_add_act_pass.py    |   4 +-
 .../test_fuse_elewise_add_act_pass.py         |  10 +-
 .../unittests/test_fuse_optimizer_pass.py     |  10 +-
 .../test_fuse_relu_depthwise_conv_pass.py     |  10 +-
 .../tests/unittests/test_gradient_clip.py     |   2 +-
 .../unittests/test_hinge_embedding_loss.py    |  12 +-
 .../unittests/test_ir_memory_optimize_pass.py |   7 +-
 .../fluid/tests/unittests/test_l1_loss.py     |   4 +-
 .../fluid/tests/unittests/test_mse_loss.py    |  12 +-
 .../fluid/tests/unittests/test_nan_inf.py     |   4 -
 .../test_parallel_executor_run_cinn.py        |   2 +-
 ...llel_executor_seresnext_with_reduce_cpu.py |  48 ++++---
 .../tests/unittests/test_recurrent_op.py      |   4 +-
 .../unittests/test_resnet50_with_cinn.py      |   2 +-
 .../tests/unittests/test_run_program_op.py    |   2 +-
 .../tests/unittests/test_zero_dim_tensor.py   | 128 ++++++++++++------
 .../incubate/autograd/composite_rules.py      |   2 +-
 .../distributed/models/moe/grad_clip.py       |  12 +-
 test/auto_parallel/test_while_op_partition.py |   2 +-
 .../test_autograd_functional_static.py        |   4 +-
 .../test_multi_precision_fp16_train.py        |   2 +-
 .../test_distribution_transform.py            |   2 +-
 test/legacy_test/test_async_read_write.py     |   2 +-
 .../vjp/eager/test_comp_eager_sum_grad.py     |   2 +-
 ...test_standalone_cuda_graph_multi_stream.py |   2 +-
 test/xpu/test_zero_dim_tensor_xpu.py          |   3 +-
 57 files changed, 380 insertions(+), 258 deletions(-)

diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
index be825f52362ee..78c4ee61f3d98 100644
--- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
+++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
@@ -365,7 +365,7 @@ void sum_grad(const Tensor& x,
     if (!keepdim) {
       auto axis_ = std::vector<int64_t>();
       if (reduce_all) {
-        for (int64_t i = 1; i < x_dim_size; i++) {
+        for (int64_t i = 0; i < x_dim_size; i++) {
           axis_.push_back(i);
         }
       } else {
diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc
index 845fe95563900..1d1d83339829b 100644
--- a/paddle/phi/infermeta/unary.cc
+++ b/paddle/phi/infermeta/unary.cc
@@ -4004,9 +4004,6 @@ DDim OriginReduceInferDim(const MetaTensor& x,
       out_dim_vector.push_back(x.dims().at(i));
     }
   }
-  if (x_rank > 0 && out_dim_vector.size() == 0) {
-    out_dim_vector.push_back(1);
-  }
 
   DDim out_dim = phi::make_ddim(out_dim_vector);
   return out_dim;
@@ -4023,14 +4020,14 @@ DDim OriginReduceInferDimForIntArrayAxis(const MetaTensor& x,
     if (keep_dim) {
       vec_dim = std::vector<int64_t>(x.dims().size(), 1);
     } else {
-      vec_dim = {1};
+      vec_dim = {};
     }
   } else {
     if (keep_dim) {
       vec_dim = std::vector<int64_t>(x.dims().size(), -1);
     } else {
       auto x_rank = static_cast<size_t>(x.dims().size());
-      if (vec_axis.size() >= x_rank) {
+      if (vec_axis.size() > x_rank) {
         vec_dim = {-1};
       } else {
         vec_dim = std::vector<int64_t>(x.dims().size() - vec_axis.size(), -1);
diff --git a/python/paddle/distributed/auto_parallel/completion.py b/python/paddle/distributed/auto_parallel/completion.py
index 91ffc556c78bd..5f2ab7e102b0b 100644
--- a/python/paddle/distributed/auto_parallel/completion.py
+++ b/python/paddle/distributed/auto_parallel/completion.py
@@ -1688,7 +1688,7 @@ def complete_update_annotation(self, serial_main_program):
                                 world_ranks
                             )
                             out_dist_attr.dims_mapping = [
-                                -1 for _ in range(len(out_var.shape))
+                                -1 for _ in out_var.shape
                             ]
                             self._dist_context.set_tensor_dist_attr_for_program(
                                 out_var, out_dist_attr
@@ -1732,7 +1732,9 @@ def complete_update_annotation(self, serial_main_program):
                                 len(out_var.shape) == 1
                                 and out_var.shape[0] == 1
                             )
-                            out_dist_attr.dims_mapping = [-1]
+                            out_dist_attr.dims_mapping = [
+                                -1 for _ in out_var.shape
+                            ]
                         self._dist_context.set_tensor_dist_attr_for_program(
                             out_var, out_dist_attr
                         )
@@ -1802,16 +1804,20 @@ def complete_update_annotation(self, serial_main_program):
                         param.name, ref_dims_mapping
                     )
                     learning_var = vars[op.input("LearningRate")[0]]
-                    op_dist_attr.set_input_dims_mapping(learning_var.name, [-1])
+                    op_dist_attr.set_input_dims_mapping(
+                        learning_var.name, [-1 for _ in learning_var.shape]
+                    )
                     op_dist_attr.set_output_dims_mapping(
-                        learning_var.name, [-1]
+                        learning_var.name, [-1 for _ in learning_var.shape]
                     )
 
                     if not learning_rate_completed:
                         learning_rate_completed = True
                         var_dist_attr = TensorDistAttr()
                         var_dist_attr.process_mesh = ProcessMesh(world_ranks)
-                        var_dist_attr.dims_mapping = [-1]
+                        var_dist_attr.dims_mapping = [
+                            -1 for _ in learning_var.shape
+                        ]
                         self._dist_context.set_tensor_dist_attr_for_program(
                             learning_var, var_dist_attr
                         )
@@ -1841,10 +1847,10 @@ def complete_update_annotation(self, serial_main_program):
                         ):
                             input_var_attr.dims_mapping = [-1]
                             op_dist_attr.set_input_dims_mapping(
-                                input_var.name, [-1]
+                                input_var.name, [-1 for _ in input_var.shape]
                             )
                             op_dist_attr.set_output_dims_mapping(
-                                input_var.name, [-1]
+                                input_var.name, [-1 for _ in input_var.shape]
                             )
                         else:
                             input_var_attr.dims_mapping = ref_dims_mapping
diff --git a/python/paddle/distributed/auto_parallel/engine.py b/python/paddle/distributed/auto_parallel/engine.py
index 567723aa29321..cb735fe11b86e 100644
--- a/python/paddle/distributed/auto_parallel/engine.py
+++ b/python/paddle/distributed/auto_parallel/engine.py
@@ -511,7 +511,7 @@ def _prepare_logger(
             loss_indices = fetch_indices[group_idx]
             assert len(loss_indices) <= 1
             for idx in loss_indices:
-                logs["loss"] = outs[idx][0]
+                logs["loss"] = outs[idx]
             group_idx += 1
             # logging metrics
             dist_context = self._dist_contexts[mode]
diff --git a/python/paddle/distributed/auto_parallel/operators/common.py b/python/paddle/distributed/auto_parallel/operators/common.py
index 209319f861a77..987c533a012eb 100644
--- a/python/paddle/distributed/auto_parallel/operators/common.py
+++ b/python/paddle/distributed/auto_parallel/operators/common.py
@@ -393,7 +393,7 @@ def get_data_parallel_group(dist_ctx, op, act_grad_names, rank):
 
     for var_name in act_grad_names:
         var_dim_mapping = op_dist_attr.get_input_dims_mapping(var_name)
-        # consider that the variable's shape is None
+        # consider that the variable's shape is [], which is 0D
         # TODO utilize the batch_dim attr instead of "0" in future
         batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
 
diff --git a/python/paddle/distributed/auto_parallel/operators/dist_default.py b/python/paddle/distributed/auto_parallel/operators/dist_default.py
index 2e7ad3d12e0b6..bad92a02c63f0 100644
--- a/python/paddle/distributed/auto_parallel/operators/dist_default.py
+++ b/python/paddle/distributed/auto_parallel/operators/dist_default.py
@@ -159,7 +159,9 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
                 ):
                     var_dim_mapping = dist_attr.get_input_dims_mapping(varname)
                     mesh_shape = process_mesh.shape
-                    batch_size_axis = var_dim_mapping[0]
+                    batch_size_axis = (
+                        var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
+                    )
                     if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1:
                         need_gradient_allreduce = True
                         break
diff --git a/python/paddle/distributed/auto_parallel/operators/dist_eltwise.py b/python/paddle/distributed/auto_parallel/operators/dist_eltwise.py
index 3e0924d143ff6..5c11dfba08fe1 100644
--- a/python/paddle/distributed/auto_parallel/operators/dist_eltwise.py
+++ b/python/paddle/distributed/auto_parallel/operators/dist_eltwise.py
@@ -101,7 +101,9 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
                 ):
                     var_dim_mapping = dist_attr.get_input_dims_mapping(varname)
                     mesh_shape = process_mesh.shape
-                    batch_size_axis = var_dim_mapping[0]
+                    batch_size_axis = (
+                        var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
+                    )
                     if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1:
                         need_gradient_allreduce = True
                         break
diff --git a/python/paddle/distributed/auto_parallel/operators/dist_embedding.py b/python/paddle/distributed/auto_parallel/operators/dist_embedding.py
index 7176341feedfb..4f13c89bb14fd 100644
--- a/python/paddle/distributed/auto_parallel/operators/dist_embedding.py
+++ b/python/paddle/distributed/auto_parallel/operators/dist_embedding.py
@@ -252,7 +252,7 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
             backward_op.input("Ids")[0]
         )
         mesh_shape = process_mesh.shape
-        batch_size_axis = var_dim_mapping[0]
+        batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
         if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1:
             parallel_axis = batch_size_axis
             attrs = {"use_calc_stream": True}
diff --git a/python/paddle/distributed/auto_parallel/operators/dist_matmul.py b/python/paddle/distributed/auto_parallel/operators/dist_matmul.py
index f3e1c74771b73..8825e14d9aba7 100644
--- a/python/paddle/distributed/auto_parallel/operators/dist_matmul.py
+++ b/python/paddle/distributed/auto_parallel/operators/dist_matmul.py
@@ -651,7 +651,7 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
             backward_op.input("X")[0]
         )
         mesh_shape = process_mesh.shape
-        batch_size_axis = var_dim_mapping[0]
+        batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
         if (
             batch_size_axis > -1
             and mesh_shape[batch_size_axis] > 1
@@ -1028,7 +1028,7 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
             backward_op.input("X")[0]
         )
         mesh_shape = process_mesh.shape
-        batch_size_axis = var_dim_mapping[0]
+        batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
         if (
             batch_size_axis > -1
             and mesh_shape[batch_size_axis] > 1
@@ -1365,7 +1365,7 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
             backward_op.input("X")[0]
         )
         mesh_shape = process_mesh.shape
-        batch_size_axis = var_dim_mapping[0]
+        batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
         if (
             batch_size_axis > -1
             and mesh_shape[batch_size_axis] > 1
@@ -1552,7 +1552,7 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
             backward_op.input("X")[0]
         )
         mesh_shape = process_mesh.shape
-        batch_size_axis = var_dim_mapping[0]
+        batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
         if (
             batch_size_axis > -1
             and mesh_shape[batch_size_axis] > 1
@@ -1929,7 +1929,7 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
             backward_op.input("X")[0]
         )
         mesh_shape = process_mesh.shape
-        batch_size_axis = var_dim_mapping[0]
+        batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
         if (
             batch_size_axis > -1
             and mesh_shape[batch_size_axis] > 1
@@ -2264,7 +2264,7 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
             backward_op.input("X")[0]
         )
         mesh_shape = process_mesh.shape
-        batch_size_axis = var_dim_mapping[0]
+        batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
         if (
             batch_size_axis > -1
             and mesh_shape[batch_size_axis] > 1
@@ -2449,7 +2449,7 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
             backward_op.input("X")[0]
         )
         mesh_shape = process_mesh.shape
-        batch_size_axis = var_dim_mapping[0]
+        batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
         if (
             batch_size_axis > -1
             and mesh_shape[batch_size_axis] > 1
@@ -2832,7 +2832,7 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
             backward_op.input("X")[0]
         )
         mesh_shape = process_mesh.shape
-        batch_size_axis = var_dim_mapping[0]
+        batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
         if (
             batch_size_axis > -1
             and mesh_shape[batch_size_axis] > 1
@@ -3178,7 +3178,7 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
             backward_op.input("X")[0]
         )
         mesh_shape = process_mesh.shape
-        batch_size_axis = var_dim_mapping[0]
+        batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
         if (
             batch_size_axis > -1
             and mesh_shape[batch_size_axis] > 1
diff --git a/python/paddle/distributed/auto_parallel/operators/dist_reshape.py b/python/paddle/distributed/auto_parallel/operators/dist_reshape.py
index d1e590c379ec3..267e8437abacc 100644
--- a/python/paddle/distributed/auto_parallel/operators/dist_reshape.py
+++ b/python/paddle/distributed/auto_parallel/operators/dist_reshape.py
@@ -120,7 +120,9 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
                     var_dim_mapping = dist_attr.get_input_dims_mapping(varname)
 
                     mesh_shape = process_mesh.shape
-                    batch_size_axis = var_dim_mapping[0]
+                    batch_size_axis = (
+                        var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
+                    )
                     if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1:
                         parallel_axis = batch_size_axis
                         attrs = {"use_calc_stream": True}
@@ -377,7 +379,9 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
                     var_dim_mapping = dist_attr.get_input_dims_mapping(varname)
 
                     mesh_shape = process_mesh.shape
-                    batch_size_axis = var_dim_mapping[0]
+                    batch_size_axis = (
+                        var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
+                    )
                     if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1:
                         parallel_axis = batch_size_axis
                         attrs = {"use_calc_stream": True}
@@ -637,7 +641,9 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
                     var_dim_mapping = dist_attr.get_input_dims_mapping(varname)
 
                     mesh_shape = process_mesh.shape
-                    batch_size_axis = var_dim_mapping[0]
+                    batch_size_axis = (
+                        var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
+                    )
                     if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1:
                         parallel_axis = batch_size_axis
                         attrs = {"use_calc_stream": True}
diff --git a/python/paddle/distributed/auto_parallel/operators/dist_scale.py b/python/paddle/distributed/auto_parallel/operators/dist_scale.py
index e95e001b89000..66a35b1eadb68 100644
--- a/python/paddle/distributed/auto_parallel/operators/dist_scale.py
+++ b/python/paddle/distributed/auto_parallel/operators/dist_scale.py
@@ -100,7 +100,9 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
                 ):
                     var_dim_mapping = dist_attr.get_input_dims_mapping(varname)
                     mesh_shape = process_mesh.shape
-                    batch_size_axis = var_dim_mapping[0]
+                    batch_size_axis = (
+                        var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
+                    )
                     if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1:
                         need_gradient_allreduce = True
                         break
diff --git a/python/paddle/distributed/auto_parallel/operators/dist_softmax.py b/python/paddle/distributed/auto_parallel/operators/dist_softmax.py
index d5c3802e50689..814464d659865 100644
--- a/python/paddle/distributed/auto_parallel/operators/dist_softmax.py
+++ b/python/paddle/distributed/auto_parallel/operators/dist_softmax.py
@@ -94,7 +94,9 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
                     var_dim_mapping = dist_attr.get_input_dims_mapping(varname)
 
                     mesh_shape = process_mesh.shape
-                    batch_size_axis = var_dim_mapping[0]
+                    batch_size_axis = (
+                        var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
+                    )
                     if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1:
                         parallel_axis = batch_size_axis
                         attrs = {"use_calc_stream": True}
diff --git a/python/paddle/distributed/auto_parallel/operators/dist_transpose.py b/python/paddle/distributed/auto_parallel/operators/dist_transpose.py
index 5fa1f3a7bac39..762c5b9209cd5 100644
--- a/python/paddle/distributed/auto_parallel/operators/dist_transpose.py
+++ b/python/paddle/distributed/auto_parallel/operators/dist_transpose.py
@@ -183,7 +183,9 @@ def calc_bwd_cost(self, dist_op, ctx, cluster):
                     var_dim_mapping = dist_attr.get_input_dims_mapping(varname)
 
                     mesh_shape = process_mesh.shape
-                    batch_size_axis = var_dim_mapping[0]
+                    batch_size_axis = (
+                        var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
+                    )
                     if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1:
                         parallel_axis = batch_size_axis
                         attrs = {"use_calc_stream": True}
diff --git a/python/paddle/distributed/auto_parallel/tuner/rule_based_tuner.py b/python/paddle/distributed/auto_parallel/tuner/rule_based_tuner.py
index 9f71163f6d78e..5ef0e87293345 100644
--- a/python/paddle/distributed/auto_parallel/tuner/rule_based_tuner.py
+++ b/python/paddle/distributed/auto_parallel/tuner/rule_based_tuner.py
@@ -1727,7 +1727,9 @@ def _complete_sub_update_program(self, sub_program_dist_context):
                                     len(out_var.shape) == 1
                                     and out_var.shape[0] == 1
                                 )
-                                out_dist_attr.dims_mapping = [-1]
+                                out_dist_attr.dims_mapping = [
+                                    -1 for _ in out_var.shape
+                                ]
                             sub_program_dist_context.set_tensor_dist_attr_for_program(
                                 out_var, out_dist_attr
                             )
@@ -1798,17 +1800,19 @@ def _complete_sub_update_program(self, sub_program_dist_context):
                         )
                         learning_var = vars[op.input("LearningRate")[0]]
                         op_dist_attr.set_input_dims_mapping(
-                            learning_var.name, [-1]
+                            learning_var.name, [-1 for i in learning_var.shape]
                         )
                         op_dist_attr.set_output_dims_mapping(
-                            learning_var.name, [-1]
+                            learning_var.name, [-1 for i in learning_var.shape]
                         )
 
                         if not learning_rate_completed:
                             learning_rate_completed = True
                             var_dist_attr = TensorDistAttr()
                             var_dist_attr.process_mesh = world_ranks
-                            var_dist_attr.dims_mapping = [-1]
+                            var_dist_attr.dims_mapping = [
+                                -1 for i in learning_var.shape
+                            ]
                             sub_program_dist_context.set_tensor_dist_attr_for_program(
                                 learning_var, var_dist_attr
                             )
diff --git a/python/paddle/distributed/auto_parallel/utils.py b/python/paddle/distributed/auto_parallel/utils.py
index 6b6afca969b7e..f4dfb8d9c20d5 100644
--- a/python/paddle/distributed/auto_parallel/utils.py
+++ b/python/paddle/distributed/auto_parallel/utils.py
@@ -1466,7 +1466,8 @@ def update_op_dims_mapping_by_default_dist_impl(dist_op):
                 ), "{} only the batch dimension (0-dim) can be sharded, but the dimension {} is sharded by {} part.".format(
                     op_desc.type(), idx, mapping
                 )
-        batch_dim_mappings.append(dims_mapping[0])
+        if len(dims_mapping) >= 1:
+            batch_dim_mappings.append(dims_mapping[0])
     for arg_name in op_desc.output_arg_names():
         serial_tensor = dist_op.get_serial_output(arg_name)
         if serial_tensor.is_parameter:
@@ -1480,7 +1481,8 @@ def update_op_dims_mapping_by_default_dist_impl(dist_op):
                     ), "{} only the batch dimension (0-dim) can be sharded, but the dimension {} is sharded by {} part.".format(
                         op_desc.type(), idx, mapping
                     )
-            batch_dim_mappings.append(dims_mapping[0])
+            if len(dims_mapping) >= 1:
+                batch_dim_mappings.append(dims_mapping[0])
         else:
             assert (
                 dims_mapping[0] == -1
@@ -1505,7 +1507,7 @@ def update_op_dims_mapping_by_default_dist_impl(dist_op):
         if serial_tensor.is_parameter:
             continue
         dims_mapping = op_dist_attr.get_input_dims_mapping(arg_name)
-        if compatible_dim_mapping != dims_mapping[0]:
+        if len(dims_mapping) >= 1 and compatible_dim_mapping != dims_mapping[0]:
             dims_mapping[0] = compatible_dim_mapping
             changed = True
     for arg_name in op_desc.output_arg_names():
@@ -1514,7 +1516,10 @@ def update_op_dims_mapping_by_default_dist_impl(dist_op):
             continue
         dims_mapping = op_dist_attr.get_output_dims_mapping(arg_name)
         if arg_name not in xshape_arg_names:
-            if compatible_dim_mapping != dims_mapping[0]:
+            if (
+                len(dims_mapping) >= 1
+                and compatible_dim_mapping != dims_mapping[0]
+            ):
                 dims_mapping[0] = compatible_dim_mapping
                 changed = True
         else:
diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py
index 3254bffb254c6..beb3fa0ff6875 100755
--- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py
+++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import numpy as np
 
 import paddle
 from paddle import framework
@@ -95,11 +96,10 @@ def _dygraph_clip(self, params_grads):
         # global norm of distributed FP16 params_and_grads
         if len(sum_square_dist_fp16) == 0:
             global_norm_dist_fp16 = paddle.to_tensor(
-                [0.0], dtype=paddle.float32
+                np.array(0.0), dtype=paddle.float32
             )
         else:
-            global_norm_dist_fp16 = paddle.concat(sum_square_dist_fp16)
-            global_norm_dist_fp16 = paddle.sum(global_norm_dist_fp16)
+            global_norm_dist_fp16 = paddle.add_n(sum_square_dist_fp16)
             global_norm_dist_fp16 = paddle.cast(
                 global_norm_dist_fp16, dtype=paddle.float32
             )
@@ -107,11 +107,10 @@ def _dygraph_clip(self, params_grads):
         # global norm of non-distributed FP16 params_and_grads
         if len(sum_square_not_dist_fp16) == 0:
             global_norm_not_dist_fp16 = paddle.to_tensor(
-                [0.0], dtype=paddle.float32
+                np.array(0.0), dtype=paddle.float32
             )
         else:
-            global_norm_not_dist_fp16 = paddle.concat(sum_square_not_dist_fp16)
-            global_norm_not_dist_fp16 = paddle.sum(global_norm_not_dist_fp16)
+            global_norm_not_dist_fp16 = paddle.add_n(sum_square_not_dist_fp16)
             global_norm_not_dist_fp16 = paddle.cast(
                 global_norm_not_dist_fp16, dtype=paddle.float32
             )
@@ -119,11 +118,10 @@ def _dygraph_clip(self, params_grads):
         # global norm of distributed BF16 params_and_grads
         if len(sum_square_dist_bf16) == 0:
             global_norm_dist_bf16 = paddle.to_tensor(
-                [0.0], dtype=paddle.float32
+                np.array(0.0), dtype=paddle.float32
             )
         else:
-            global_norm_dist_bf16 = paddle.concat(sum_square_dist_bf16)
-            global_norm_dist_bf16 = paddle.sum(global_norm_dist_bf16)
+            global_norm_dist_bf16 = paddle.add_n(sum_square_dist_bf16)
             global_norm_dist_bf16 = paddle.cast(
                 global_norm_dist_bf16, dtype=paddle.float32
             )
@@ -131,30 +129,29 @@ def _dygraph_clip(self, params_grads):
         # global norm of non-distributed FP16 params_and_grads
         if len(sum_square_not_dist_bf16) == 0:
             global_norm_not_dist_bf16 = paddle.to_tensor(
-                [0.0], dtype=paddle.float32
+                np.array(0.0), dtype=paddle.float32
             )
         else:
-            global_norm_not_dist_bf16 = paddle.concat(sum_square_not_dist_bf16)
-            global_norm_not_dist_bf16 = paddle.sum(global_norm_not_dist_bf16)
+            global_norm_not_dist_bf16 = paddle.add_n(sum_square_not_dist_bf16)
             global_norm_not_dist_bf16 = paddle.cast(
                 global_norm_not_dist_bf16, dtype=paddle.float32
             )
 
         # global norm of distributed FP32 params_and_grads
-        global_norm_dist_fp32 = (
-            paddle.concat(sum_square_dist_fp32)
-            if len(sum_square_dist_fp32) != 0
-            else paddle.to_tensor([0.0], dtype=paddle.float32)
-        )
-        global_norm_dist_fp32 = paddle.sum(global_norm_dist_fp32)
+        if len(sum_square_dist_fp32) == 0:
+            global_norm_dist_fp32 = paddle.to_tensor(
+                np.array(0.0), dtype=paddle.float32
+            )
+        else:
+            global_norm_dist_fp32 = paddle.add_n(sum_square_dist_fp32)
 
         # global norm of non-distributed FP32 params_and_grads
-        global_norm_not_dist_fp32 = (
-            paddle.concat(sum_square_not_dist_fp32)
-            if len(sum_square_not_dist_fp32) != 0
-            else paddle.to_tensor([0.0], dtype=paddle.float32)
-        )
-        global_norm_not_dist_fp32 = paddle.sum(global_norm_not_dist_fp32)
+        if len(sum_square_not_dist_fp32) == 0:
+            global_norm_not_dist_fp32 = paddle.to_tensor(
+                np.array(0.0), dtype=paddle.float32
+            )
+        else:
+            global_norm_not_dist_fp32 = paddle.add_n(sum_square_not_dist_fp32)
 
         global_norm_var_dist = (
             global_norm_dist_fp16
@@ -193,14 +190,14 @@ def _dygraph_clip(self, params_grads):
         )
 
         max_global_norm = paddle.full(
-            shape=[1],
+            shape=[],
             dtype=global_norm_var_fp32.dtype,
             fill_value=self.clip_norm,
         )
         clip_var = paddle.divide(
             x=max_global_norm,
             y=paddle.maximum(x=global_norm_var_fp32, y=max_global_norm)
-            + paddle.to_tensor([1.0e-6], dtype=paddle.float32),
+            + paddle.to_tensor(np.array(1.0e-6), dtype=paddle.float32),
         )
         clip_var_fp16 = paddle.cast(clip_var, paddle.float16)
 
diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py
index f64edb8b63c6a..928c93df0b40a 100644
--- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py
+++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py
@@ -94,59 +94,64 @@ def _dygraph_clip(self, params_grads):
 
         # global norm of non-distributed FP16 params_and_grads
         if len(sum_square_fp16) == 0:
-            global_norm_fp16 = paddle.to_tensor([0.0], dtype=paddle.float32)
+            global_norm_fp16 = paddle.to_tensor(
+                np.array(0.0), dtype=paddle.float32
+            )
         else:
-            global_norm_fp16 = paddle.concat(sum_square_fp16)
-            global_norm_fp16 = paddle.sum(global_norm_fp16)
+            global_norm_fp16 = paddle.add_n(sum_square_fp16)
             global_norm_fp16 = paddle.cast(
                 global_norm_fp16, dtype=paddle.float32
             )
 
         # global norm of non-distributed BFP16 params_and_grads
         if len(sum_square_bfp16) == 0:
-            global_norm_bfp16 = paddle.to_tensor([0.0], dtype=paddle.float32)
+            global_norm_bfp16 = paddle.to_tensor(
+                np.array(0.0), dtype=paddle.float32
+            )
         else:
-            global_norm_bfp16 = paddle.concat(sum_square_bfp16)
-            global_norm_bfp16 = paddle.sum(global_norm_bfp16)
+            global_norm_bfp16 = paddle.add_n(sum_square_bfp16)
             global_norm_bfp16 = paddle.cast(
                 global_norm_bfp16, dtype=paddle.float32
             )
 
         # global norm of non-distributed FP16 params_and_grads for unslice parameters
         if len(unslice_params_fp16) == 0:
-            global_unslice_fp16 = paddle.to_tensor([0.0], dtype=paddle.float32)
+            global_unslice_fp16 = paddle.to_tensor(
+                np.array(0.0), dtype=paddle.float32
+            )
         else:
-            global_unslice_fp16 = paddle.concat(unslice_params_fp16)
-            global_unslice_fp16 = paddle.sum(global_unslice_fp16)
+            global_unslice_fp16 = paddle.add_n(unslice_params_fp16)
             global_unslice_fp16 = paddle.cast(
                 global_unslice_fp16, dtype=paddle.float32
             )
 
         # global norm of non-distributed BFP16 params_and_grads for unslice parameters
         if len(unslice_params_bfp16) == 0:
-            global_unslice_bfp16 = paddle.to_tensor([0.0], dtype=paddle.float32)
+            global_unslice_bfp16 = paddle.to_tensor(
+                np.array(0.0), dtype=paddle.float32
+            )
         else:
-            global_unslice_bfp16 = paddle.concat(unslice_params_bfp16)
-            global_unslice_bfp16 = paddle.sum(global_unslice_bfp16)
+            global_unslice_bfp16 = paddle.add_n(unslice_params_bfp16)
             global_unslice_bfp16 = paddle.cast(
                 global_unslice_bfp16, dtype=paddle.float32
             )
 
         # global norm of non-distributed FP32 params_and_grads
-        global_norm_fp32 = (
-            paddle.concat(sum_square_fp32)
-            if len(sum_square_fp32) != 0
-            else paddle.to_tensor([0.0], dtype=paddle.float32)
-        )
-        global_norm_fp32 = paddle.sum(global_norm_fp32)
+        if len(sum_square_fp32) == 0:
+            global_norm_fp32 = paddle.to_tensor(
+                np.array(0.0), dtype=paddle.float32
+            )
+        else:
+            global_norm_fp32 = paddle.add_n(sum_square_fp32)
 
         # global norm of non-distributed FP32 params_and_grads for unslice parameters
-        global_unslice_fp32 = (
-            paddle.concat(unslice_params_fp32)
-            if len(unslice_params_fp32) != 0
-            else paddle.to_tensor([0.0], dtype=paddle.float32)
-        )
-        global_unslice_fp32 = paddle.sum(global_unslice_fp32)
+        if len(unslice_params_fp32) == 0:
+            global_unslice_fp32 = paddle.to_tensor(
+                np.array(0.0), dtype=paddle.float32
+            )
+        else:
+            global_unslice_fp32 = paddle.add_n(unslice_params_fp32)
+
         global_unslice_var = (
             global_unslice_fp16 + global_unslice_fp32 + global_unslice_bfp16
         )
@@ -165,7 +170,7 @@ def _dygraph_clip(self, params_grads):
 
         global_norm_var = paddle.sqrt(global_norm_var + global_unslice_var)
         max_global_norm = paddle.full(
-            shape=[1], dtype=global_norm_var.dtype, fill_value=self.clip_norm
+            shape=[], dtype=global_norm_var.dtype, fill_value=self.clip_norm
         )
 
         clip_var = paddle.divide(
diff --git a/python/paddle/distributed/fleet/metrics/metric.py b/python/paddle/distributed/fleet/metrics/metric.py
index d2f72b0c7d047..0d744d17cdd4a 100644
--- a/python/paddle/distributed/fleet/metrics/metric.py
+++ b/python/paddle/distributed/fleet/metrics/metric.py
@@ -40,7 +40,7 @@ def sum(input, scope=None, util=None):
           # in model.py
           input = paddle.cast(some_input, dtype='float32')
           cnt = paddle.sum(input)
-          global_cnt = paddle.static.create_global_var(persistable=True, dtype='float32', shape=[1], value=0)
+          global_cnt = paddle.static.create_global_var(persistable=True, dtype='float32', shape=[], value=0)
           tmp = paddle.add(cnt, global_cnt)
           paddle.assign(tmp, global_cnt)
 
@@ -80,7 +80,7 @@ def max(input, scope=None, util=None):
           # in model.py
           input = paddle.cast(some_input, dtype='float32')
           cnt = paddle.sum(input)
-          global_cnt = paddle.static.create_global_var(persistable=True, dtype='float32', shape=[1], value=0)
+          global_cnt = paddle.static.create_global_var(persistable=True, dtype='float32', shape=[], value=0)
           tmp = paddle.maximum(cnt, global_cnt)
           paddle.assign(tmp, global_cnt)
 
@@ -120,7 +120,7 @@ def min(input, scope=None, util=None):
           # in model.py
           input = paddle.cast(some_input, dtype='float32')
           cnt = paddle.sum(input)
-          global_cnt = paddle.static.create_global_var(persistable=True, dtype='float32', shape=[1], value=0)
+          global_cnt = paddle.static.create_global_var(persistable=True, dtype='float32', shape=[], value=0)
           tmp = paddle.minimum(cnt, global_cnt)
           paddle.assign(tmp, global_cnt)
 
diff --git a/python/paddle/distributed/passes/auto_parallel_amp.py b/python/paddle/distributed/passes/auto_parallel_amp.py
index 6f44c44bf3f96..def5156f811aa 100644
--- a/python/paddle/distributed/passes/auto_parallel_amp.py
+++ b/python/paddle/distributed/passes/auto_parallel_amp.py
@@ -955,7 +955,7 @@ def _cast_loss(self):
 
             loss_op._set_attr(OP_ROLE_KEY, OpRole.Forward)
             naive_set_dist_op_attr_for_program_by_mesh_and_mapping(
-                cast_op, ref_mesh, [-1], self.dist_context
+                cast_op, ref_mesh, [-1 for i in loss.shape], self.dist_context
             )
 
             # backward
@@ -970,12 +970,20 @@ def _cast_loss(self):
                 dtype=core.VarDesc.VarType.FP32,
                 persistable=loss.persistable,
             )
-            set_var_dist_attr(self.dist_context, cast_loss_grad, [-1], ref_mesh)
+            set_var_dist_attr(
+                self.dist_context,
+                cast_loss_grad,
+                [-1 for i in loss.shape],
+                ref_mesh,
+            )
 
             pre_grad_name = first_backward_op.output_arg_names[0]
             first_backward_op._rename_output(pre_grad_name, cast_loss_grad.name)
             naive_set_dist_op_attr_for_program_by_mesh_and_mapping(
-                first_backward_op, ref_mesh, [-1], self.dist_context
+                first_backward_op,
+                ref_mesh,
+                [-1 for i in loss.shape],
+                self.dist_context,
             )
             cast_grad_op = main_block._insert_op(
                 loss_op_idx + 3,
@@ -989,7 +997,10 @@ def _cast_loss(self):
                 },
             )
             naive_set_dist_op_attr_for_program_by_mesh_and_mapping(
-                cast_grad_op, ref_mesh, [-1], self.dist_context
+                cast_grad_op,
+                ref_mesh,
+                [-1 for i in loss.shape],
+                self.dist_context,
             )
             loss_op = cast_op
             loss = cast_loss
@@ -1021,7 +1032,12 @@ def _scale_loss(self):
                 dtype=loss.dtype,
                 persistable=loss.persistable,
             )
-            set_var_dist_attr(self.dist_context, scaled_loss, [-1], ref_mesh)
+            set_var_dist_attr(
+                self.dist_context,
+                scaled_loss,
+                [-1 for i in loss.shape],
+                ref_mesh,
+            )
 
             elementwise_mul_op = main_block._insert_op(
                 loss_op_idx + 1,
@@ -1034,7 +1050,10 @@ def _scale_loss(self):
             )
             loss_op._set_attr(OP_ROLE_KEY, OpRole.Forward)
             naive_set_dist_op_attr_for_program_by_mesh_and_mapping(
-                elementwise_mul_op, ref_mesh, [-1], self.dist_context
+                elementwise_mul_op,
+                ref_mesh,
+                [-1 for i in loss.shape],
+                self.dist_context,
             )
 
             # backward
@@ -1050,14 +1069,20 @@ def _scale_loss(self):
                 persistable=loss.persistable,
             )
             set_var_dist_attr(
-                self.dist_context, scaled_loss_grad, [-1], ref_mesh
+                self.dist_context,
+                scaled_loss_grad,
+                [-1 for i in loss.shape],
+                ref_mesh,
             )
             pre_grad_name = first_backward_op.output_arg_names[0]
             first_backward_op._rename_output(
                 pre_grad_name, scaled_loss_grad.name
             )
             naive_set_dist_op_attr_for_program_by_mesh_and_mapping(
-                first_backward_op, ref_mesh, [-1], self.dist_context
+                first_backward_op,
+                ref_mesh,
+                [-1 for i in loss.shape],
+                self.dist_context,
             )
             scaled_loss_grad.op = first_backward_op
             # FIXME(JZ-LIANG) a trick to insert backward op
@@ -1085,7 +1110,10 @@ def _scale_loss(self):
             elementwise_mul_grad_op = main_block.ops[loss_op_idx + 3]
             assert elementwise_mul_grad_op.type == "elementwise_mul_grad"
             naive_set_dist_op_attr_for_program_by_mesh_and_mapping(
-                elementwise_mul_grad_op, ref_mesh, [-1], self.dist_context
+                elementwise_mul_grad_op,
+                ref_mesh,
+                [-1 for i in loss.shape],
+                self.dist_context,
             )
         else:
             scaled_loss = loss
diff --git a/python/paddle/distributed/passes/auto_parallel_fp16.py b/python/paddle/distributed/passes/auto_parallel_fp16.py
index 7cf10cfbc7fbd..3216c1b408276 100644
--- a/python/paddle/distributed/passes/auto_parallel_fp16.py
+++ b/python/paddle/distributed/passes/auto_parallel_fp16.py
@@ -678,7 +678,12 @@ def _insert_memcopy(block, idx, src_var, dist_context, direction="D2H"):
         stop_gradient=src_var.stop_gradient,
     )
 
-    set_var_dist_attr(dist_context, output_var, [-1], world_process_group.ranks)
+    set_var_dist_attr(
+        dist_context,
+        output_var,
+        [-1 for i in src_var.shape],
+        world_process_group.ranks,
+    )
 
     # TODO to support CUDAPinned/NPU/XPU Places
     if direction == "D2H":
@@ -894,7 +899,7 @@ def _apply_single_impl(self, main_program, startup_program, context):
                             set_var_dist_attr(
                                 self.dist_context,
                                 found_inf,
-                                [-1],
+                                [-1 for i in found_inf.shape],
                                 world_process_group.ranks,
                             )
                             _set_op_dist_attr_with_ranks(
diff --git a/python/paddle/distributed/passes/auto_parallel_grad_clip.py b/python/paddle/distributed/passes/auto_parallel_grad_clip.py
index 91631b04ad625..481ba3b6c3113 100644
--- a/python/paddle/distributed/passes/auto_parallel_grad_clip.py
+++ b/python/paddle/distributed/passes/auto_parallel_grad_clip.py
@@ -221,7 +221,7 @@ def _init_dist_attr(self, op):
             in_var = self.block.vars[in_name]
             in_dist_attr = TensorDistAttr()
             in_dist_attr.process_mesh = ProcessMesh(self.world_ranks)
-            in_dist_attr.dims_mapping = [-1]
+            in_dist_attr.dims_mapping = [-1 for i in in_var.shape]
             self.dist_context.set_tensor_dist_attr_for_program(
                 in_var, in_dist_attr
             )
@@ -230,7 +230,7 @@ def _init_dist_attr(self, op):
             out_var = self.block.vars[out_name]
             out_dist_attr = TensorDistAttr()
             out_dist_attr.process_mesh = ProcessMesh(self.world_ranks)
-            out_dist_attr.dims_mapping = [-1]
+            out_dist_attr.dims_mapping = [-1 for i in out_var.shape]
             self.dist_context.set_tensor_dist_attr_for_program(
                 out_var, out_dist_attr
             )
diff --git a/python/paddle/distributed/passes/auto_parallel_quantization.py b/python/paddle/distributed/passes/auto_parallel_quantization.py
index f1e6261cafb03..f2f35b33728bb 100644
--- a/python/paddle/distributed/passes/auto_parallel_quantization.py
+++ b/python/paddle/distributed/passes/auto_parallel_quantization.py
@@ -300,7 +300,7 @@ def set_dist_attr_for_qat_program(
                     for slot_name in quant_op.desc.input_names():
                         in_name = quant_op.desc.input(slot_name)[0]
                         input_var = block._var_recursive(in_name)
-                        ref_dims_mapping = [-1]
+                        ref_dims_mapping = [-1 for i in input_var.shape]
                         if slot_name == "X":
                             continue
                         elif slot_name in ['Scale', 'ZeroPoint']:
@@ -333,7 +333,7 @@ def set_dist_attr_for_qat_program(
                     for slot_name in quant_op.desc.output_names():
                         output_name = quant_op.desc.output(slot_name)[0]
                         output_var = block._var_recursive(output_name)
-                        ref_dims_mapping = [-1]
+                        ref_dims_mapping = [-1 for i in output_var.shape]
                         if slot_name == "Y":
                             dist_context.set_tensor_dist_attr_for_program(
                                 output_var, consume_input_dist_attr
diff --git a/python/paddle/fluid/tests/unittests/check_nan_inf_base.py b/python/paddle/fluid/tests/unittests/check_nan_inf_base.py
index 4237987f997ea..f48df68baddeb 100644
--- a/python/paddle/fluid/tests/unittests/check_nan_inf_base.py
+++ b/python/paddle/fluid/tests/unittests/check_nan_inf_base.py
@@ -95,11 +95,7 @@ def check(use_cuda):
                     fetch_list=[y_predict.name, avg_cost.name, acc_top1.name],
                 )
                 step += 1
-                print(
-                    'iter={:.0f},cost={},acc1={}'.format(
-                        step, outs[1][0], outs[2]
-                    )
-                )
+                print(f'iter={step:.0f},cost={outs[1]},acc1={outs[2]}')
 
 
 if __name__ == '__main__':
diff --git a/python/paddle/fluid/tests/unittests/seresnext_test_base.py b/python/paddle/fluid/tests/unittests/seresnext_test_base.py
index 44fdf28caec55..bed6180caea74 100644
--- a/python/paddle/fluid/tests/unittests/seresnext_test_base.py
+++ b/python/paddle/fluid/tests/unittests/seresnext_test_base.py
@@ -49,17 +49,19 @@ def _compare_result_with_origin_model(
         )
 
         if compare_separately:
-            for loss in zip(func_1_first_loss, func_2_first_loss):
-                self.assertAlmostEqual(loss[0], loss[1], delta=1e-5)
-            for loss in zip(func_1_last_loss, func_2_last_loss):
-                self.assertAlmostEqual(loss[0], loss[1], delta=delta2)
+            self.assertAlmostEqual(
+                func_1_first_loss, func_2_first_loss, delta=1e-5
+            )
+            self.assertAlmostEqual(
+                func_1_last_loss, func_2_last_loss, delta=delta2
+            )
         else:
             np.testing.assert_allclose(
                 func_1_loss_area, func_2_loss_area, rtol=delta2
             )
             self.assertAlmostEqual(
-                np.mean(func_1_first_loss), func_2_first_loss[0], delta=1e-5
+                func_1_first_loss, func_2_first_loss, delta=1e-5
             )
             self.assertAlmostEqual(
-                np.mean(func_1_last_loss), func_2_last_loss[0], delta=delta2
+                func_1_last_loss, func_2_last_loss, delta=delta2
             )
diff --git a/python/paddle/fluid/tests/unittests/test_argsort_op.py b/python/paddle/fluid/tests/unittests/test_argsort_op.py
index b1eafb3ace41a..ec6db2f6651e9 100644
--- a/python/paddle/fluid/tests/unittests/test_argsort_op.py
+++ b/python/paddle/fluid/tests/unittests/test_argsort_op.py
@@ -24,6 +24,7 @@
 from paddle.fluid.framework import Program, grad_var_name
 
 np.random.seed(123)
+paddle.enable_static()
 
 
 class PyArgsort:
@@ -52,7 +53,7 @@ def forward(self):
         out = (
             np.array(self.indices, dtype=self.indices.dtype),
             np.array(self.sorted_x, dtype=self.sorted_x.dtype),
-            np.array([self.loss], dtype=self.loss.dtype),
+            np.array(self.loss, dtype=self.loss.dtype),
         )
         return out
 
@@ -178,7 +179,7 @@ def get_numerical_gradient(self, delta=1e-7):
 
                 f[...] = o
                 dout_dfeed = (y_pos - y_neg) / (delta * 2)
-                g[...] = dout_dfeed[0]
+                g[...] = dout_dfeed
 
         return grad_list
 
diff --git a/python/paddle/fluid/tests/unittests/test_cond.py b/python/paddle/fluid/tests/unittests/test_cond.py
index e2f2cf1a34760..c23ecd6c5d896 100644
--- a/python/paddle/fluid/tests/unittests/test_cond.py
+++ b/python/paddle/fluid/tests/unittests/test_cond.py
@@ -674,7 +674,7 @@ def backward_value_helper(self, cond_func, use_cuda):
                     },
                     fetch_list=[loss.name],
                 )
-                numerical_grad[0][j] = (loss_delta[0] - loss_value[0]) / delta
+                numerical_grad[0][j] = (loss_delta - loss_value) / delta
                 feed_img_delta[0][j] = feed_img[0][j]
             np.testing.assert_allclose(
                 img_grad, numerical_grad, rtol=0.05, atol=0.05
diff --git a/python/paddle/fluid/tests/unittests/test_cosine_embedding_loss.py b/python/paddle/fluid/tests/unittests/test_cosine_embedding_loss.py
index 1b571e176fd42..882d2f505a718 100644
--- a/python/paddle/fluid/tests/unittests/test_cosine_embedding_loss.py
+++ b/python/paddle/fluid/tests/unittests/test_cosine_embedding_loss.py
@@ -64,7 +64,7 @@ def run_dynamic(self):
             reduction='mean',
         )
         np.testing.assert_allclose(dy_result.numpy(), expected1, rtol=1e-05)
-        self.assertTrue(dy_result.shape, [1])
+        self.assertEqual(dy_result.shape, [])
 
         dy_result = paddle.nn.functional.cosine_embedding_loss(
             input1, input2, label, margin=0.5, reduction='sum'
@@ -78,7 +78,7 @@ def run_dynamic(self):
         )
 
         np.testing.assert_allclose(dy_result.numpy(), expected2, rtol=1e-05)
-        self.assertTrue(dy_result.shape, [1])
+        self.assertEqual(dy_result.shape, [])
 
         dy_result = paddle.nn.functional.cosine_embedding_loss(
             input1, input2, label, margin=0.5, reduction='none'
@@ -92,7 +92,7 @@ def run_dynamic(self):
         )
 
         np.testing.assert_allclose(dy_result.numpy(), expected3, rtol=1e-05)
-        self.assertTrue(dy_result.shape, [5])
+        self.assertEqual(dy_result.shape, [5])
 
     def run_static(self, use_gpu=False):
         input1 = static.data(name='input1', shape=[5, 3], dtype='float64')
@@ -257,7 +257,7 @@ def run_dynamic(self):
             reduction='mean',
         )
         np.testing.assert_allclose(dy_result.numpy(), expected1, rtol=1e-05)
-        self.assertTrue(dy_result.shape, [1])
+        self.assertEqual(dy_result.shape, [])
 
         input1_1D = paddle.to_tensor(self.input1_np_1D)
         input2_1D = paddle.to_tensor(self.input2_np_1D)
diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_recurrent_op.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_recurrent_op.py
index dca923b3faa71..c01b05d1e462f 100644
--- a/python/paddle/fluid/tests/unittests/test_eager_deletion_recurrent_op.py
+++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_recurrent_op.py
@@ -42,7 +42,7 @@ def step(self, step_id, x):
     def forward(self):
         for step_id in range(self.x.shape[0]):
             self.step(step_id, self.x[step_id])
-        return np.array([np.mean(self.y)])
+        return np.mean(self.y)
 
     def segment_inputs(self):
         return [self.x[i] for i in range(self.x.shape[0])]
@@ -251,7 +251,7 @@ def get_numerical_gradient(self, delta=0.005):
 
                 f[...] = o
                 dout_dfeed = (y_pos - y_neg) / (delta * 2)
-                g[...] = dout_dfeed[0]
+                g[...] = dout_dfeed
 
         return grad_list
 
diff --git a/python/paddle/fluid/tests/unittests/test_fetch_lod_tensor_array.py b/python/paddle/fluid/tests/unittests/test_fetch_lod_tensor_array.py
index b5936275dd0c2..bf0f894683d5e 100644
--- a/python/paddle/fluid/tests/unittests/test_fetch_lod_tensor_array.py
+++ b/python/paddle/fluid/tests/unittests/test_fetch_lod_tensor_array.py
@@ -69,9 +69,10 @@ def check_network(self, use_cuda=True):
             loss_v, array_v = exe.run(
                 binary, feed=feed_dict, fetch_list=[loss, array]
             )
-            self.assertEqual(np.array(loss_v).shape, (1,))
-            self.assertEqual(np.array(array_v[0]).shape, (batch_size, 784))
-            self.assertEqual(np.array(array_v[1]).shape, (batch_size, 1))
+            self.assertEqual(loss_v.shape, ())
+            self.assertEqual(array_v[0].shape, (batch_size, 784))
+            self.assertEqual(array_v[1].shape, (batch_size, 1))
+            self.assertEqual(array_v[2].shape, ())
             np.testing.assert_allclose(loss_v, array_v[2], rtol=1e-05)
 
     def test_fetch_lod_tensor_array(self):
@@ -81,4 +82,5 @@ def test_fetch_lod_tensor_array(self):
 
 
 if __name__ == '__main__':
+    paddle.enable_static()
     unittest.main()
diff --git a/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py
index eb978064a5419..379010c8e42a4 100644
--- a/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py
+++ b/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py
@@ -78,10 +78,12 @@ def compare_fuse_all_reduce_ops(
             optimizer=optimizer,
         )
 
-        for loss in zip(not_fuse_op_first_loss, fuse_op_first_loss):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-6)
-        for loss in zip(not_fuse_op_last_loss, fuse_op_last_loss):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-6)
+        self.assertAlmostEqual(
+            not_fuse_op_first_loss, fuse_op_first_loss, delta=1e-6
+        )
+        self.assertAlmostEqual(
+            not_fuse_op_last_loss, fuse_op_last_loss, delta=1e-6
+        )
 
     def optimizer(self, learning_rate=1e-3):
         optimizer = fluid.optimizer.SGD(
diff --git a/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py
index 38ac248d0760e..6df1a3209ad12 100644
--- a/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py
+++ b/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py
@@ -98,7 +98,7 @@ def check(self, place, use_cuda):
                 loss_v = exe.run(
                     binary, feed=feeder.feed(data), fetch_list=[loss]
                 )
-                loss_vals.append(loss_v[0][0])
+                loss_vals.append(loss_v[0])
 
         # open fused_bn_act_ops
         build_strategy_fused = fluid.BuildStrategy()
@@ -118,7 +118,7 @@ def check(self, place, use_cuda):
                 loss_v = exe.run(
                     binary_fused, feed=feeder.feed(data), fetch_list=[loss]
                 )
-                loss_vals_fused.append(loss_v[0][0])
+                loss_vals_fused.append(loss_v[0])
 
         # check loss
         for i in range(iters):
diff --git a/python/paddle/fluid/tests/unittests/test_fuse_bn_add_act_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_bn_add_act_pass.py
index bbbbc4a91a303..63796599278ce 100644
--- a/python/paddle/fluid/tests/unittests/test_fuse_bn_add_act_pass.py
+++ b/python/paddle/fluid/tests/unittests/test_fuse_bn_add_act_pass.py
@@ -216,7 +216,7 @@ def check(self, place, use_cuda):
                 loss_v = exe.run(
                     binary_fused, feed={"x": x, "y": y}, fetch_list=[loss]
                 )
-                loss_vals_fused.append(loss_v[0][0])
+                loss_vals_fused.append(loss_v[0])
 
         # build_origin_program: turn off fused_bn_act_ops
         build_strategy = fluid.BuildStrategy()
@@ -234,7 +234,7 @@ def check(self, place, use_cuda):
                     feed={"x": x_data[i], "y": y_data[i]},
                     fetch_list=[loss],
                 )
-                loss_vals.append(loss_v[0][0])
+                loss_vals.append(loss_v[0])
 
         # check loss
         for i in range(iters):
diff --git a/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py
index 29fc8944e2107..cce3451ed5f61 100644
--- a/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py
+++ b/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py
@@ -74,10 +74,12 @@ def _optimizer(learning_rate=1e-6):
             optimizer=_optimizer,
         )
 
-        for loss in zip(not_fuse_op_first_loss, fuse_op_first_loss):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-6)
-        for loss in zip(not_fuse_op_last_loss, fuse_op_last_loss):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-6)
+        self.assertAlmostEqual(
+            not_fuse_op_first_loss, fuse_op_first_loss, delta=1e-6
+        )
+        self.assertAlmostEqual(
+            not_fuse_op_last_loss, fuse_op_last_loss, delta=1e-6
+        )
 
     def test_simple_fc_with_fuse_op(self):
         self._compare_fuse_elewise_add_act_ops(simple_fc_net, DeviceType.CUDA)
diff --git a/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py
index 71eaf8bc5420b..4e7f1901d72e4 100644
--- a/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py
+++ b/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py
@@ -70,10 +70,12 @@ def _compare_fused_optimizer_ops(
             optimizer=optimizer,
         )
 
-        for loss in zip(not_fuse_op_first_loss, fuse_op_first_loss):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-6)
-        for loss in zip(not_fuse_op_last_loss, fuse_op_last_loss):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-6)
+        self.assertAlmostEqual(
+            not_fuse_op_first_loss, fuse_op_first_loss, delta=1e-6
+        )
+        self.assertAlmostEqual(
+            not_fuse_op_last_loss, fuse_op_last_loss, delta=1e-6
+        )
 
     def _decorate_compare_fused_optimizer_ops(
         self, model, use_device, optimizer
diff --git a/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py
index fea42b00140a9..2a48de7bedda9 100644
--- a/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py
+++ b/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py
@@ -118,10 +118,12 @@ def _optimizer(learning_rate=1e-6):
             optimizer=_optimizer,
         )
 
-        for loss in zip(not_fuse_op_first_loss, fuse_op_first_loss):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-6)
-        for loss in zip(not_fuse_op_last_loss, fuse_op_last_loss):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-6)
+        self.assertAlmostEqual(
+            not_fuse_op_first_loss, fuse_op_first_loss, delta=1e-6
+        )
+        self.assertAlmostEqual(
+            not_fuse_op_last_loss, fuse_op_last_loss, delta=1e-6
+        )
 
     def test_simple_depthwise_with_fuse_op(self):
         self._compare(simple_depthwise_net, DeviceType.CUDA)
diff --git a/python/paddle/fluid/tests/unittests/test_gradient_clip.py b/python/paddle/fluid/tests/unittests/test_gradient_clip.py
index 9a20c536bc6f3..dc84d9e70b028 100644
--- a/python/paddle/fluid/tests/unittests/test_gradient_clip.py
+++ b/python/paddle/fluid/tests/unittests/test_gradient_clip.py
@@ -152,7 +152,7 @@ def check_sparse_gradient_clip(self, place):
 
         data = next(self.train_data())
         val = exe.run(prog, feed=feeder.feed(data), fetch_list=[cost])[0]
-        self.assertEqual((1,), val.shape)
+        self.assertEqual(val.shape, ())
         self.assertFalse(np.isnan(val))
 
     def backward_and_optimize(self, cost):
diff --git a/python/paddle/fluid/tests/unittests/test_hinge_embedding_loss.py b/python/paddle/fluid/tests/unittests/test_hinge_embedding_loss.py
index 1ce6c1bc5f025..5bfeb0f0f143a 100644
--- a/python/paddle/fluid/tests/unittests/test_hinge_embedding_loss.py
+++ b/python/paddle/fluid/tests/unittests/test_hinge_embedding_loss.py
@@ -50,7 +50,7 @@ def run_dynamic_check(self, place=paddle.CPUPlace()):
         dy_result = paddle.nn.functional.hinge_embedding_loss(input, label)
         expected = calc_hinge_embedding_loss(self.input_np, self.label_np)
         np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05)
-        self.assertTrue(dy_result.shape, [1])
+        self.assertEqual(dy_result.shape, [])
 
         dy_result = paddle.nn.functional.hinge_embedding_loss(
             input, label, reduction='sum'
@@ -59,7 +59,7 @@ def run_dynamic_check(self, place=paddle.CPUPlace()):
             self.input_np, self.label_np, reduction='sum'
         )
         np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05)
-        self.assertTrue(dy_result.shape, [1])
+        self.assertEqual(dy_result.shape, [])
 
         dy_result = paddle.nn.functional.hinge_embedding_loss(
             input, label, reduction='none'
@@ -68,7 +68,7 @@ def run_dynamic_check(self, place=paddle.CPUPlace()):
             self.input_np, self.label_np, reduction='none'
         )
         np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05)
-        self.assertTrue(dy_result.shape, self.shape)
+        self.assertEqual(dy_result.shape, list(self.shape))
 
     def run_static_check(self, place=paddle.CPUPlace):
         paddle.enable_static()
@@ -129,7 +129,7 @@ def run_dynamic_check(self, place=paddle.CPUPlace()):
         dy_result = hinge_embedding_loss(input, label)
         expected = calc_hinge_embedding_loss(self.input_np, self.label_np)
         np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05)
-        self.assertTrue(dy_result.shape, [1])
+        self.assertEqual(dy_result.shape, [])
 
         hinge_embedding_loss = paddle.nn.loss.HingeEmbeddingLoss(
             reduction='sum'
@@ -139,7 +139,7 @@ def run_dynamic_check(self, place=paddle.CPUPlace()):
             self.input_np, self.label_np, reduction='sum'
         )
         np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05)
-        self.assertTrue(dy_result.shape, [1])
+        self.assertEqual(dy_result.shape, [])
 
         hinge_embedding_loss = paddle.nn.loss.HingeEmbeddingLoss(
             reduction='none'
@@ -149,7 +149,7 @@ def run_dynamic_check(self, place=paddle.CPUPlace()):
             self.input_np, self.label_np, reduction='none'
         )
         np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05)
-        self.assertTrue(dy_result.shape, self.shape)
+        self.assertTrue(dy_result.shape, list(self.shape))
 
     def run_static_check(self, place=paddle.CPUPlace):
         paddle.enable_static()
diff --git a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_pass.py b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_pass.py
index 046ad9a89cb12..4fff201519ca2 100644
--- a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_pass.py
+++ b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_pass.py
@@ -80,10 +80,9 @@ def _compare_ir_memory_optimize(self, model, use_device):
             use_device=use_device,
             use_ir_memory_optimize=True,
         )
-        for loss in zip(first_loss0, first_loss1):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-6)
-        for loss in zip(last_loss0, last_loss1):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-6)
+
+        self.assertAlmostEqual(first_loss0, first_loss1, delta=1e-6)
+        self.assertAlmostEqual(last_loss0, last_loss1, delta=1e-6)
 
     def test_simple_fc_net(self):
         self._compare_ir_memory_optimize(simple_fc_net, DeviceType.CPU)
diff --git a/python/paddle/fluid/tests/unittests/test_l1_loss.py b/python/paddle/fluid/tests/unittests/test_l1_loss.py
index ce5d4127fbc7e..20295d1a9a92b 100644
--- a/python/paddle/fluid/tests/unittests/test_l1_loss.py
+++ b/python/paddle/fluid/tests/unittests/test_l1_loss.py
@@ -36,7 +36,7 @@ def run_imperative(self):
         dy_result = paddle.nn.functional.l1_loss(input, label, reduction='sum')
         expected = np.sum(np.abs(self.input_np - self.label_np))
         np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05)
-        self.assertEqual(dy_result.shape, [1])
+        self.assertEqual(dy_result.shape, [])
 
         dy_result = paddle.nn.functional.l1_loss(input, label, reduction='none')
         expected = np.abs(self.input_np - self.label_np)
@@ -125,7 +125,7 @@ def run_imperative(self):
         dy_result = l1_loss(input, label)
         expected = np.sum(np.abs(self.input_np - self.label_np))
         np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05)
-        self.assertEqual(dy_result.shape, [1])
+        self.assertEqual(dy_result.shape, [])
 
         l1_loss = paddle.nn.loss.L1Loss(reduction='none')
         dy_result = l1_loss(input, label)
diff --git a/python/paddle/fluid/tests/unittests/test_mse_loss.py b/python/paddle/fluid/tests/unittests/test_mse_loss.py
index ba680f91458dc..37fb0a6a95883 100644
--- a/python/paddle/fluid/tests/unittests/test_mse_loss.py
+++ b/python/paddle/fluid/tests/unittests/test_mse_loss.py
@@ -118,7 +118,7 @@ def test_NNMseLoss_mean(self):
             np.testing.assert_allclose(static_result, expected, rtol=1e-05)
             np.testing.assert_allclose(static_result, dy_result, rtol=1e-05)
             np.testing.assert_allclose(dy_result, expected, rtol=1e-05)
-            self.assertTrue(dy_result.shape, [1])
+            self.assertEqual(dy_result.shape, ())
 
     def test_NNMseLoss_sum(self):
         for dim in [[10, 10], [2, 10, 10], [3, 3, 10, 10]]:
@@ -164,7 +164,7 @@ def test_NNMseLoss_sum(self):
             np.testing.assert_allclose(static_result, expected, rtol=1e-05)
             np.testing.assert_allclose(static_result, dy_result, rtol=1e-05)
             np.testing.assert_allclose(dy_result, expected, rtol=1e-05)
-            self.assertTrue(dy_result.shape, [1])
+            self.assertEqual(dy_result.shape, ())
 
     def test_NNMseLoss_none(self):
         for dim in [[10, 10], [2, 10, 10], [3, 3, 10, 10]]:
@@ -210,7 +210,7 @@ def test_NNMseLoss_none(self):
             np.testing.assert_allclose(static_result, expected, rtol=1e-05)
             np.testing.assert_allclose(static_result, dy_result, rtol=1e-05)
             np.testing.assert_allclose(dy_result, expected, rtol=1e-05)
-            self.assertTrue(dy_result.shape, [1])
+            self.assertEqual(dy_result.shape, tuple(dim))
 
 
 class TestNNFunctionalMseLoss(unittest.TestCase):
@@ -254,7 +254,7 @@ def test_NNFunctionalMseLoss_mean(self):
             np.testing.assert_allclose(static_result, expected, rtol=1e-05)
             np.testing.assert_allclose(static_result, dy_result, rtol=1e-05)
             np.testing.assert_allclose(dy_result, expected, rtol=1e-05)
-            self.assertTrue(dy_result.shape, [1])
+            self.assertEqual(dy_result.shape, ())
 
     def test_NNFunctionalMseLoss_sum(self):
         for dim in [[10, 10], [2, 10, 10], [3, 3, 10, 10]]:
@@ -296,7 +296,7 @@ def test_NNFunctionalMseLoss_sum(self):
             np.testing.assert_allclose(static_result, expected, rtol=1e-05)
             np.testing.assert_allclose(static_result, dy_result, rtol=1e-05)
             np.testing.assert_allclose(dy_result, expected, rtol=1e-05)
-            self.assertTrue(dy_result.shape, [1])
+            self.assertEqual(dy_result.shape, ())
 
     def test_NNFunctionalMseLoss_none(self):
         for dim in [[10, 10], [2, 10, 10], [3, 3, 10, 10]]:
@@ -338,7 +338,7 @@ def test_NNFunctionalMseLoss_none(self):
             np.testing.assert_allclose(static_result, expected, rtol=1e-05)
             np.testing.assert_allclose(static_result, dy_result, rtol=1e-05)
             np.testing.assert_allclose(dy_result, expected, rtol=1e-05)
-            self.assertTrue(dy_result.shape, [1])
+            self.assertEqual(dy_result.shape, tuple(dim))
 
 
 if __name__ == "__main__":
diff --git a/python/paddle/fluid/tests/unittests/test_nan_inf.py b/python/paddle/fluid/tests/unittests/test_nan_inf.py
index 5e233e26a6283..4a60902768fd4 100644
--- a/python/paddle/fluid/tests/unittests/test_nan_inf.py
+++ b/python/paddle/fluid/tests/unittests/test_nan_inf.py
@@ -42,10 +42,6 @@ def check_nan_inf(self):
 
         out, err = proc.communicate()
         returncode = proc.returncode
-
-        print(out)
-        print(err)
-
         # in python3, type(out+err) is 'bytes', need use encode
         assert (out + err).find(b'There are NAN or INF') != -1
 
diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_run_cinn.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_run_cinn.py
index 79cbabee262c7..c598073f434a8 100644
--- a/python/paddle/fluid/tests/unittests/test_parallel_executor_run_cinn.py
+++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_run_cinn.py
@@ -110,7 +110,7 @@ def train(dot_save_dir, prefix, seed=1234):
     loss_values = []
     for step in range(iters):
         loss_v = exe.run(compiled_program, feed=feed[step], fetch_list=[loss])
-        loss_values.append(loss_v[0][0])
+        loss_values.append(loss_v[0])
     return loss_values
 
 
diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_cpu.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_cpu.py
index fa1abbed09505..d1e3d9f6d0c9a 100644
--- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_cpu.py
+++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_cpu.py
@@ -48,10 +48,14 @@ def _compare_reduce_and_allreduce(self, use_device, delta2=1e-5):
             optimizer=seresnext_net.optimizer,
         )
 
-        for loss in zip(all_reduce_first_loss, reduce_first_loss):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-5)
-        for loss in zip(all_reduce_last_loss, reduce_last_loss):
-            self.assertAlmostEqual(loss[0], loss[1], delta=loss[0] * delta2)
+        self.assertAlmostEqual(
+            all_reduce_first_loss, reduce_first_loss, delta=1e-5
+        )
+        self.assertAlmostEqual(
+            all_reduce_last_loss,
+            reduce_last_loss,
+            delta=all_reduce_last_loss * delta2,
+        )
 
         if not use_device:
             return
@@ -86,20 +90,32 @@ def _compare_reduce_and_allreduce(self, use_device, delta2=1e-5):
             enable_sequential_execution=True,
         )
 
-        for loss in zip(all_reduce_first_loss, all_reduce_first_loss_seq):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-5)
-        for loss in zip(all_reduce_last_loss, all_reduce_last_loss_seq):
-            self.assertAlmostEqual(loss[0], loss[1], delta=loss[0] * delta2)
+        self.assertAlmostEqual(
+            all_reduce_first_loss, all_reduce_first_loss_seq, delta=1e-5
+        )
+        self.assertAlmostEqual(
+            all_reduce_last_loss,
+            all_reduce_last_loss_seq,
+            delta=all_reduce_last_loss * delta2,
+        )
 
-        for loss in zip(reduce_first_loss, reduce_first_loss_seq):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-5)
-        for loss in zip(reduce_last_loss, reduce_last_loss_seq):
-            self.assertAlmostEqual(loss[0], loss[1], delta=loss[0] * delta2)
+        self.assertAlmostEqual(
+            reduce_first_loss, reduce_first_loss_seq, delta=1e-5
+        )
+        self.assertAlmostEqual(
+            reduce_last_loss,
+            reduce_last_loss_seq,
+            delta=reduce_last_loss * delta2,
+        )
 
-        for loss in zip(all_reduce_first_loss_seq, reduce_first_loss_seq):
-            self.assertAlmostEqual(loss[0], loss[1], delta=1e-5)
-        for loss in zip(all_reduce_last_loss_seq, reduce_last_loss_seq):
-            self.assertAlmostEqual(loss[0], loss[1], delta=loss[0] * delta2)
+        self.assertAlmostEqual(
+            all_reduce_first_loss_seq, reduce_first_loss_seq, delta=1e-5
+        )
+        self.assertAlmostEqual(
+            all_reduce_last_loss_seq,
+            reduce_last_loss_seq,
+            delta=all_reduce_last_loss_seq * delta2,
+        )
 
 
 class TestResnetWithReduceCPU(TestResnetWithReduceBase):
diff --git a/python/paddle/fluid/tests/unittests/test_recurrent_op.py b/python/paddle/fluid/tests/unittests/test_recurrent_op.py
index 5a468d8e3ef3a..8874d955c06ab 100644
--- a/python/paddle/fluid/tests/unittests/test_recurrent_op.py
+++ b/python/paddle/fluid/tests/unittests/test_recurrent_op.py
@@ -37,7 +37,7 @@ def step(self, step_id, x):
     def forward(self):
         for step_id in range(self.x.shape[0]):
             self.step(step_id, self.x[step_id])
-        return np.array([np.mean(self.y)])
+        return np.mean(self.y)
 
     def segment_inputs(self):
         return [self.x[i] for i in range(self.x.shape[0])]
@@ -239,7 +239,7 @@ def get_numerical_gradient(self, delta=0.005):
 
                 f[...] = o
                 dout_dfeed = (y_pos - y_neg) / (delta * 2)
-                g[...] = dout_dfeed[0]
+                g[...] = dout_dfeed
 
         return grad_list
 
diff --git a/python/paddle/fluid/tests/unittests/test_resnet50_with_cinn.py b/python/paddle/fluid/tests/unittests/test_resnet50_with_cinn.py
index 85517ffeba2d0..d262319eee8e3 100644
--- a/python/paddle/fluid/tests/unittests/test_resnet50_with_cinn.py
+++ b/python/paddle/fluid/tests/unittests/test_resnet50_with_cinn.py
@@ -103,7 +103,7 @@ def train(self, place, iters, feed, use_cinn=False, seed=1234):
                     fetch_list=[loss],
                     return_numpy=True,
                 )
-                loss_vals.append(loss_v[0][0])
+                loss_vals.append(loss_v[0])
         return loss_vals
 
     def test_check_resnet50_accuracy(self):
diff --git a/python/paddle/fluid/tests/unittests/test_run_program_op.py b/python/paddle/fluid/tests/unittests/test_run_program_op.py
index 73eb152de43ac..0f381b5bdda5e 100644
--- a/python/paddle/fluid/tests/unittests/test_run_program_op.py
+++ b/python/paddle/fluid/tests/unittests/test_run_program_op.py
@@ -514,7 +514,7 @@ def test_stop_gradient(self):
 
         dy_loss = self.train(to_static=False)
         st_loss = self.train(to_static=True)
-        self.assertEqual(dy_loss[0], st_loss[0])
+        self.assertEqual(dy_loss, st_loss)
 
         paddle.enable_static()
 
diff --git a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
index bc8a7b82c5bc8..9482f252f876b 100644
--- a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
+++ b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
@@ -220,14 +220,6 @@ def test_dygraph_reduce(self):
                 self.assertEqual(x.grad.shape, [])
                 np.testing.assert_allclose(x.grad.numpy(), np.array(3.0))
 
-            if api in [
-                paddle.sum,
-                paddle.mean,
-                paddle.nanmean,
-                paddle.nansum,
-            ]:
-                return
-
             # 2) x is ND, reduce to 0D
             if api in [paddle.all, paddle.any]:
                 x = paddle.randint(0, 2, [3, 5]).astype('bool')
@@ -302,20 +294,11 @@ def test_static_reduce(self):
                     np.testing.assert_allclose(res[2], np.array(1.0))
                     np.testing.assert_allclose(res[3], np.array(1.0))
 
-                if api in [
-                    paddle.sum,
-                    paddle.mean,
-                    paddle.nanmean,
-                    paddle.nansum,
-                ]:
-                    return
-
                 # 2) x is ND, reduce to 0D
                 if api in [paddle.all, paddle.any]:
                     x = paddle.randint(0, 2, [3, 5]).astype('bool')
                 else:
                     x = paddle.rand([3, 5])
-                x = paddle.rand([3, 5])
                 x.stop_gradient = False
                 out = api(x, None)
                 paddle.static.append_backward(out)
@@ -1365,6 +1348,7 @@ def test_metric_accuracy(self):
         self.assertEqual(out.shape, [])
 
     def test_std(self):
+        # 1) x is 0D
         x = paddle.rand([])
         x.stop_gradient = False
         out1 = paddle.std(x)
@@ -1372,18 +1356,24 @@ def test_std(self):
         out1.backward()
         out2.backward()
 
-        # checkout shape of out
         self.assertEqual(out1.shape, [])
         self.assertEqual(out2.shape, [])
-
-        # checkout value of out
         self.assertEqual(out1, 0)
         self.assertEqual(out2, 0)
 
-        # checkout backward
         self.assertEqual(x.grad.shape, [])
 
+        # 2) x is ND
+        x = paddle.rand([3, 5])
+        x.stop_gradient = False
+        out = paddle.std(x)
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        self.assertEqual(x.grad.shape, [3, 5])
+
     def test_var(self):
+        # 1) x is 0D
         x = paddle.rand([])
         x.stop_gradient = False
         out1 = paddle.var(x)
@@ -1391,18 +1381,23 @@ def test_var(self):
         out1.backward()
         out2.backward()
 
-        # checkout shape of out
         self.assertEqual(out1.shape, [])
         self.assertEqual(out2.shape, [])
-
-        # checkout value of out
         self.assertEqual(out1, 0)
         self.assertEqual(out2, 0)
 
-        # checkout backward
         self.assertEqual(x.grad.shape, [])
         np.testing.assert_allclose(x.grad, 0)
 
+        # 2) x is ND
+        x = paddle.rand([3, 5])
+        x.stop_gradient = False
+        out = paddle.std(x)
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        self.assertEqual(x.grad.shape, [3, 5])
+
     def test_quantile(self):
         # 1) x is 0D
         x = paddle.rand([])
@@ -1598,7 +1593,6 @@ def test_clip(self):
         out = paddle.clip(x, -5, 5)
         out.retain_grads()
         out.backward()
-
         self.assertEqual(out.shape, [])
         self.assertEqual(out.grad.shape, [])
         self.assertEqual(x.grad.shape, [])
@@ -1608,7 +1602,6 @@ def test_clip(self):
         out1 = paddle.clip(x1, paddle.full([], 5.0), paddle.full([], 5.0))
         out1.retain_grads()
         out1.backward()
-
         self.assertEqual(out1.shape, [])
         self.assertEqual(out1.grad.shape, [])
         self.assertEqual(x1.grad.shape, [])
@@ -5643,8 +5636,7 @@ def test_Categorical(self):
         self.assertEqual(
             d.log_prob(paddle.full([], 2, dtype='int64')).shape, []
         )
-        # because use paddle.sum
-        # self.assertEqual(d.entropy().shape, [])
+        self.assertEqual(d.entropy().shape, [])
 
     def test_Normal(self):
         normal = paddle.distribution.Normal(0.0, 3.0)
@@ -5687,10 +5679,9 @@ def test_Beta(self):
         self.assertEqual(beta.sample([]).shape, [])
         self.assertEqual(beta.mean.shape, [])
         self.assertEqual(beta.variance.shape, [])
-        # because use paddle.sum
-        # self.assertEqual(beta.prob(self.x).shape, [])
-        # self.assertEqual(beta.log_prob(self.x).shape, [])
-        # self.assertEqual(beta.entropy().shape, [])
+        self.assertEqual(beta.prob(self.x).shape, [])
+        self.assertEqual(beta.log_prob(self.x).shape, [])
+        self.assertEqual(beta.entropy().shape, [])
 
     def test_kl_divergence(self):
         p = paddle.distribution.Beta(alpha=0.5, beta=0.5)
@@ -5749,10 +5740,9 @@ def test_Multinomial(self):
         d = paddle.distribution.Multinomial(
             10, paddle.to_tensor([0.2, 0.3, 0.5])
         )
-        # because use paddle.sum
-        # self.assertEqual(d.prob(self.x).shape, [])
-        # self.assertEqual(d.log_prob(self.x).shape, [])
-        # self.assertEqual(d.entropy().shape, [])
+        self.assertEqual(d.prob(self.x).shape, [])
+        self.assertEqual(d.log_prob(self.x).shape, [])
+        self.assertEqual(d.entropy().shape, [])
 
 
 class TestLossAPI(unittest.TestCase):
@@ -5770,10 +5760,10 @@ def test_sigmoid_focal_loss(self):
         fg_num_1 = paddle.full([1], 2.0)
 
         out0 = F.sigmoid_focal_loss(
-            logit, label, normalizer=fg_num_0, reduction='mean'
+            logit, label, normalizer=fg_num_0, reduction='sum'
         )
         out1 = F.sigmoid_focal_loss(
-            logit, label, normalizer=fg_num_1, reduction='mean'
+            logit, label, normalizer=fg_num_1, reduction='sum'
         )
         out0.retain_grads()
 
@@ -5788,6 +5778,28 @@ def test_sigmoid_focal_loss(self):
         self.assertEqual(out0.grad.shape, [])
         self.assertEqual(logit.grad.shape, [2, 3])
 
+    def test_cross_entropy(self):
+        input = paddle.rand([3, 5])
+        input.stop_gradient = False
+        label = paddle.randint(0, 5, shape=[3])
+
+        loss = paddle.nn.functional.cross_entropy(input, label, reduction='sum')
+        loss.backward()
+
+        self.assertEqual(loss.shape, [])
+        self.assertEqual(input.grad.shape, [3, 5])
+
+    def test_l1_loss(self):
+        input = paddle.rand([3, 5])
+        input.stop_gradient = False
+        label = paddle.rand([3, 5])
+
+        loss = paddle.nn.functional.l1_loss(input, label, reduction='mean')
+        loss.backward()
+
+        self.assertEqual(loss.shape, [])
+        self.assertEqual(input.grad.shape, [3, 5])
+
 
 class TestLossAPIStatic(unittest.TestCase):
     def setUp(self):
@@ -5818,12 +5830,42 @@ def test_sigmoid_focal_loss(self):
             prog, fetch_list=[out0, out1, out0.grad_name, logit.grad_name]
         )
         np.testing.assert_allclose(res[0], res[1])
-        # because static use paddle.mean
-        # self.assertEqual(res[0].shape, ())
-        # self.assertEqual(res[1].shape, ())
-        # self.assertEqual(res[2].shape, ())
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, ())
+        self.assertEqual(res[2].shape, ())
         self.assertEqual(res[3].shape, (2, 3))
 
+    @prog_scope()
+    def test_cross_entropy(self):
+        input = paddle.rand([3, 5])
+        input.stop_gradient = False
+        label = paddle.randint(0, 5, shape=[3])
+        label.stop_gradient = False
+
+        loss = paddle.nn.functional.cross_entropy(
+            input, label, reduction='mean'
+        )
+        paddle.static.append_backward(loss)
+
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[loss, input.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (3, 5))
+
+    @prog_scope()
+    def test_l1_loss(self):
+        input = paddle.rand([3, 5])
+        input.stop_gradient = False
+        label = paddle.rand([3, 5])
+
+        loss = paddle.nn.functional.l1_loss(input, label, reduction='sum')
+        paddle.static.append_backward(loss)
+
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[loss, input.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (3, 5))
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/python/paddle/incubate/autograd/composite_rules.py b/python/paddle/incubate/autograd/composite_rules.py
index 0a47715c6062c..8a1c71395a775 100644
--- a/python/paddle/incubate/autograd/composite_rules.py
+++ b/python/paddle/incubate/autograd/composite_rules.py
@@ -252,7 +252,7 @@ def mean_composite(x, axis, keepdim):
         operator.mul, [x.shape[axis] for axis in axes]
     )
     norm = fill_constant(
-        shape=x.shape if len(x.shape) == 0 else [1],
+        shape=[],
         value=value_to_fill,
         dtype=sum_x.dtype,
     )
diff --git a/python/paddle/incubate/distributed/models/moe/grad_clip.py b/python/paddle/incubate/distributed/models/moe/grad_clip.py
index a386347ce2770..59ba6bb8f9454 100644
--- a/python/paddle/incubate/distributed/models/moe/grad_clip.py
+++ b/python/paddle/incubate/distributed/models/moe/grad_clip.py
@@ -142,22 +142,18 @@ def get_l2_norm_pow(params_grads, sum_dtype=None):
 
         global_norm_var = []
         if len(sum_square_list_fp16) > 0:
-            global_norm_var_fp16 = paddle.concat(sum_square_list_fp16)
-            global_norm_var_fp16 = paddle.sum(global_norm_var_fp16)
+            global_norm_var_fp16 = paddle.add_n(sum_square_list_fp16)
             global_norm_var.append(global_norm_var_fp16.astype(sum_dtype))
         if len(sum_square_list_fp32) > 0:
-            global_norm_var_fp32 = paddle.concat(sum_square_list_fp32)
-            global_norm_var_fp32 = paddle.sum(global_norm_var_fp32)
+            global_norm_var_fp32 = paddle.add_n(sum_square_list_fp32)
             if sum_dtype == 'float32':
                 global_norm_var.append(global_norm_var_fp32)
             else:
                 global_norm_var.append(global_norm_var_fp32.astype(sum_dtype))
         if len(sum_square_list) > 0:
-            global_norm_var_fp64 = paddle.concat(sum_square_list)
-            global_norm_var_fp64 = paddle.sum(global_norm_var_fp64)
+            global_norm_var_fp64 = paddle.add_n(sum_square_list)
             global_norm_var.append(global_norm_var_fp64)
-        global_norm_var = paddle.concat(global_norm_var)
-        global_norm_var = paddle.sum(global_norm_var)
+        global_norm_var = paddle.add_n(global_norm_var)
         return global_norm_var, sum_dtype
 
     @no_grad()
diff --git a/test/auto_parallel/test_while_op_partition.py b/test/auto_parallel/test_while_op_partition.py
index fdbcee8255a2b..6dc02d6834f5b 100644
--- a/test/auto_parallel/test_while_op_partition.py
+++ b/test/auto_parallel/test_while_op_partition.py
@@ -206,7 +206,7 @@ def get_program():
         auto.shard_tensor(error_cost, _g_process_mesh, [None, None, None])
 
         loss = paddle.mean(error_cost)
-        auto.shard_tensor(loss, _g_process_mesh, [None])
+        auto.shard_tensor(loss, _g_process_mesh, [])
 
     return train_program, start_program, dataloader, i, loss
 
diff --git a/test/autograd/test_autograd_functional_static.py b/test/autograd/test_autograd_functional_static.py
index 19ab630651b99..38c27c5e2384e 100644
--- a/test/autograd/test_autograd_functional_static.py
+++ b/test/autograd/test_autograd_functional_static.py
@@ -41,14 +41,14 @@
             'v_not_none',
             utils.reduce,
             np.random.rand(2, 3),
-            np.random.rand(1),
+            np.array(np.random.rand()),
             False,
         ),
         (
             'xs_stop_gradient',
             utils.reduce,
             np.random.rand(2, 3),
-            np.random.rand(1),
+            np.array(np.random.rand()),
             True,
         ),
         (
diff --git a/test/contrib/test_multi_precision_fp16_train.py b/test/contrib/test_multi_precision_fp16_train.py
index 218cfcd542da4..3672cd2427a86 100644
--- a/test/contrib/test_multi_precision_fp16_train.py
+++ b/test/contrib/test_multi_precision_fp16_train.py
@@ -178,7 +178,7 @@ def train_loop():
                 (loss,) = exe.run(
                     train_program, feed=feeder.feed(data), fetch_list=[sum_cost]
                 )
-                loss_v = loss[0] if isinstance(loss, np.ndarray) else loss
+                loss_v = float(loss) if isinstance(loss, np.ndarray) else loss
                 print(
                     'PassID {:1}, Train Batch ID {:04}, train loss {:2.4}'.format(
                         pass_id, batch_id + 1, float(loss_v)
diff --git a/test/distribution/test_distribution_transform.py b/test/distribution/test_distribution_transform.py
index 63ecd99d77baa..640391b472d7a 100644
--- a/test/distribution/test_distribution_transform.py
+++ b/test/distribution/test_distribution_transform.py
@@ -1205,7 +1205,7 @@ def test_inverse_shape(self, shape, expected_shape):
     @param.param_func(((np.random.random(10),),))
     def test_forward_log_det_jacobian(self, x):
         self.assertEqual(
-            self._t.forward_log_det_jacobian(paddle.to_tensor(x)).shape, [1]
+            self._t.forward_log_det_jacobian(paddle.to_tensor(x)).shape, []
         )
 
 
diff --git a/test/legacy_test/test_async_read_write.py b/test/legacy_test/test_async_read_write.py
index 4fc20039881b9..98aee8d905669 100644
--- a/test/legacy_test/test_async_read_write.py
+++ b/test/legacy_test/test_async_read_write.py
@@ -65,7 +65,7 @@ def func_test_async_read_success(self):
             )
         # index data
         index_array1 = paddle.gather(self.src, self.index)
-        count_numel = paddle.sum(count).numpy()[0]
+        count_numel = paddle.sum(count).item()
         index_array2 = self.dst[count_numel : count_numel + len(self.index)]
         np.testing.assert_allclose(
             index_array1.numpy(), index_array2.numpy(), rtol=1e-05
diff --git a/test/prim/prim/vjp/eager/test_comp_eager_sum_grad.py b/test/prim/prim/vjp/eager/test_comp_eager_sum_grad.py
index bed88e395210b..9cade52686fe1 100644
--- a/test/prim/prim/vjp/eager/test_comp_eager_sum_grad.py
+++ b/test/prim/prim/vjp/eager/test_comp_eager_sum_grad.py
@@ -41,7 +41,7 @@ def desired(primal, cotangent, axis, keep_dim):
 class TestSumGradComp(unittest.TestCase):
     def test_sum_grad_comp_1(self):
         self.primal = np.random.rand(10, 10)
-        self.cotangent = np.random.rand(1)
+        self.cotangent = np.array(np.random.rand())
         paddle.disable_static()
 
         np.testing.assert_allclose(
diff --git a/test/standalone_executor/test_standalone_cuda_graph_multi_stream.py b/test/standalone_executor/test_standalone_cuda_graph_multi_stream.py
index f6deb87d1c602..bee898653ca07 100644
--- a/test/standalone_executor/test_standalone_cuda_graph_multi_stream.py
+++ b/test/standalone_executor/test_standalone_cuda_graph_multi_stream.py
@@ -126,7 +126,7 @@ def test_result(self):
 
         for out in outs:
             for baseline, result in zip(outs[0], out):
-                self.assertEqual(baseline[0], result[0])
+                self.assertEqual(baseline, result)
 
 
 if __name__ == "__main__":
diff --git a/test/xpu/test_zero_dim_tensor_xpu.py b/test/xpu/test_zero_dim_tensor_xpu.py
index 9ecce0af8305d..9369a9b0ed3ff 100644
--- a/test/xpu/test_zero_dim_tensor_xpu.py
+++ b/test/xpu/test_zero_dim_tensor_xpu.py
@@ -1269,10 +1269,11 @@ def test_sigmoid_focal_loss(self):
             out0.numpy(),
             out1.numpy(),
         )
+        self.assertEqual(out0.shape, [])
 
         out0.retain_grads()
         out0.backward()
-        self.assertEqual(out0.grad.shape, [1])
+        self.assertEqual(out0.grad.shape, [])
         self.assertEqual(logit.grad.shape, [2, 3])
 
     def test_allclose(self):

From afa3f9b3c76573b7bddc4ce9963e96a701a7ac45 Mon Sep 17 00:00:00 2001
From: Aurelius84 <zhangliujie@baidu.com>
Date: Thu, 4 May 2023 09:41:19 +0800
Subject: [PATCH 150/405] [Perf]Removed usless assign op in while_loop (#53452)

* [Perf]Removed usless assign op in while_loop

* refine assign
---
 python/paddle/static/nn/control_flow.py | 4 ++++
 python/paddle/tensor/creation.py        | 5 ++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/python/paddle/static/nn/control_flow.py b/python/paddle/static/nn/control_flow.py
index bc5f1d2d5d6f5..79cc848804bdc 100644
--- a/python/paddle/static/nn/control_flow.py
+++ b/python/paddle/static/nn/control_flow.py
@@ -395,6 +395,10 @@ def has_shape_diff(x_var, y_var):
                     input.shape, output.shape
                 )
             )
+        # NOTE(dev): Avoid assign if input is output in Variable level which means
+        # input is not generated in While sub block and modified by in-place and only
+        # belong to inplace ops in constructing program process, because in-place pass
+        # is only available in Graph level.
         paddle.assign(input, output)
 
 
diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py
index c57fceeeb8525..db1006b0b1af0 100644
--- a/python/paddle/tensor/creation.py
+++ b/python/paddle/tensor/creation.py
@@ -2029,6 +2029,10 @@ def assign(x, output=None):
             result2 = paddle.assign(data)  # result2 = [[2.5, 2.5], [2.5, 2.5], [2.5, 2.5]]
             result3 = paddle.assign(np.array([[2.5, 2.5], [2.5, 2.5], [2.5, 2.5]], dtype='float32')) # result3 = [[2.5, 2.5], [2.5, 2.5], [2.5, 2.5]]
     """
+    # speed up
+    if x is output and isinstance(x, Variable):
+        return x
+
     input = x
     helper = LayerHelper('assign', **locals())
     check_type(
@@ -2037,7 +2041,6 @@ def assign(x, output=None):
         (Variable, np.ndarray, list, tuple, float, int, bool),
         'assign',
     )
-    is_inplace = True if output is not None else False
 
     if np.isscalar(input) and not isinstance(input, str):
         input = np.array([input])

From 6f5a4dfe2bb72118b3b2de179180eb2a2e08db89 Mon Sep 17 00:00:00 2001
From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com>
Date: Thu, 4 May 2023 10:28:41 +0800
Subject: [PATCH 151/405] add rules (#53425)

---
 tools/check_file_diff_approvals.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tools/check_file_diff_approvals.sh b/tools/check_file_diff_approvals.sh
index 55efe2d47c395..3215f9d507222 100644
--- a/tools/check_file_diff_approvals.sh
+++ b/tools/check_file_diff_approvals.sh
@@ -85,7 +85,6 @@ API_FILES=("CMakeLists.txt"
            "paddle/fluid/prim/api/api.yaml"
            "python/paddle/incubate/autograd/composite_rules.py"
 	   "python/paddle/incubate/autograd/primitives.py"
-
            )
 
 approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000`
@@ -208,7 +207,7 @@ for API_FILE in ${API_FILES[*]}; do
       elif [ "${API_FILE}" == "paddle/phi/api/include/tensor.h" ] || [ "${API_FILE}" == "paddle/phi/core/tensor_base.h" ] || [ "${API_FILE}" == "paddle/phi/core/dense_tensor.h" ] || [ "${API_FILE}" == "paddle/phi/core/meta_tensor.h" ] || [ "${API_FILE}" == "paddle/phi/core/tensor_meta.h" ] || [ "${API_FILE}" == "paddle/phi/core/attribute.h" ] || [ "${API_FILE}" == "paddle/phi/core/device_context.h" ] || [ "${API_FILE}" == "paddle/phi/core/kernel_utils.h" ] || [ "${API_FILE}" == "paddle/phi/core/kernel_registry.h" ] || [ "${API_FILE}" == "paddle/phi/core/kernel_factory.h" ] || [ "${API_FILE}" == "paddle/phi/core/kernel_context.h" ] || [ "${API_FILE}" == "paddle/phi/core/infermeta_utils.h" ]; then
             echo_line="You must have one RD (chenwhql, phlrain, zyfncg, YuanRisheng) approval for changing ${API_FILE} , which manages the underlying code for PaddlePaddle PHI Library.\n"
             check_approval chenwhql phlrain zyfncg YuanRisheng
-      elif [ "${API_FILE}" == "paddle/fluid/prim/api/composite_backward/composite_backward_api.h" ] || [ "${API_FILE}" == "paddle/fluid/prim/api/manual_prim/prim_manual_api.h" ] || ["${API_FILE}" == "paddle/fluid/prim/api/api.yaml"]; then
+      elif [ "${API_FILE}" == "paddle/fluid/prim/api/composite_backward/composite_backward_api.h" ] || [ "${API_FILE}" == "paddle/fluid/prim/api/manual_prim/prim_manual_api.h" ] || [ "${API_FILE}" == "paddle/fluid/prim/api/api.yaml" ]; then
             echo_line="You must have one RD (JiabinYang, cxxly(chenxiaoxu) , xiaoguoguo626807(wangruting)) approval for changing ${API_FILE} , which manages the code for PaddlePaddle Composite Bacward Prim API.\n"
             check_approval 1 JiabinYang cxxly xiaoguoguo626807
       elif [ "${API_FILE}" == "python/paddle/incubate/autograd/primitives.py" ] || [ "${API_FILE}" == "python/paddle/incubate/autograd/composite_rules.py" ]; then

From 72e235d02037a5460a0114111ebd8ba902ec4644 Mon Sep 17 00:00:00 2001
From: Yuanle Liu <yuanlehome@163.com>
Date: Thu, 4 May 2023 11:17:09 +0800
Subject: [PATCH 152/405] tensor should be defined (#53449)

---
 paddle/fluid/inference/api/analysis_predictor.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc
index 4ef15fbbc15dc..e46d4c600dc74 100644
--- a/paddle/fluid/inference/api/analysis_predictor.cc
+++ b/paddle/fluid/inference/api/analysis_predictor.cc
@@ -1151,10 +1151,15 @@ bool AnalysisPredictor::SetFeed(const std::vector<paddle::Tensor> &inputs,
                         feeds_.size(),
                         inputs.size()));
   for (size_t i = 0; i < inputs.size(); ++i) {
-    PADDLE_ENFORCE_EQ(inputs[i].initialized(),
+    PADDLE_ENFORCE_EQ(inputs[i].defined(),
                       true,
                       paddle::platform::errors::InvalidArgument(
-                          "The input Tensor expected to be initialized."));
+                          "The input Tensor expected to be defined."));
+    PADDLE_ENFORCE_EQ(
+        inputs[i].is_dense_tensor(),
+        true,
+        paddle::platform::errors::InvalidArgument(
+            "The input Tensor expected to be type of dense tensor."));
   }
 
   if (std::all_of(inputs.cbegin(), inputs.cend(), [&](const paddle::Tensor &t) {

From d71f68a0e1abdb403bd66200f3a3cce8603a3b6d Mon Sep 17 00:00:00 2001
From: hua-zi <83271073+hua-zi@users.noreply.github.com>
Date: Thu, 4 May 2023 14:32:04 +0800
Subject: [PATCH 153/405] =?UTF-8?q?=E3=80=90=E8=8B=B1=E6=96=87=E6=96=87?=
 =?UTF-8?q?=E6=A1=A3=E4=BF=AE=E5=A4=8D=E3=80=91updata=20Adam.py=20(#53424)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 python/paddle/optimizer/adam.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/paddle/optimizer/adam.py b/python/paddle/optimizer/adam.py
index 2934700154a70..82b100da7573b 100644
--- a/python/paddle/optimizer/adam.py
+++ b/python/paddle/optimizer/adam.py
@@ -105,7 +105,7 @@ class Adam(Optimizer):
             loss = paddle.mean(out)
             adam = paddle.optimizer.Adam(learning_rate=0.1,
                     parameters=linear.parameters())
-            out.backward()
+            loss.backward()
             adam.step()
             adam.clear_grad()
 
@@ -127,7 +127,7 @@ class Adam(Optimizer):
                     beta1=beta1,
                     beta2=beta2,
                     weight_decay=0.01)
-            out.backward()
+            loss.backward()
             adam.step()
             adam.clear_grad()
 
@@ -150,7 +150,7 @@ class Adam(Optimizer):
                 }],
                 weight_decay=0.01,
                 beta1=0.9)
-            out.backward()
+            loss.backward()
             adam.step()
             adam.clear_grad()
 

From a06be6380e3571b1e5f398798fbbbacade20e49b Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Thu, 4 May 2023 14:33:26 +0800
Subject: [PATCH 154/405] =?UTF-8?q?=E3=80=90Hackathon=20No.64=E3=80=91matm?=
 =?UTF-8?q?ul=5Fwith=5Fflatten=20=E7=AE=97=E5=AD=90FP16/BF16=E5=8D=95?=
 =?UTF-8?q?=E6=B5=8B=E5=AE=8C=E5=96=84=20(#52953)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 paddle/phi/kernels/gpu/matmul_grad_kernel.cu  |   1 +
 .../fluid/tests/unittests/test_mul_op.py      | 126 ++++++++++++++++--
 2 files changed, 116 insertions(+), 11 deletions(-)

diff --git a/paddle/phi/kernels/gpu/matmul_grad_kernel.cu b/paddle/phi/kernels/gpu/matmul_grad_kernel.cu
index 4b48e2eb17322..e27d5251a9144 100644
--- a/paddle/phi/kernels/gpu/matmul_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/matmul_grad_kernel.cu
@@ -64,4 +64,5 @@ PD_REGISTER_KERNEL(matmul_with_flatten_double_grad,
                    phi::MatmulWithFlattenDoubleGradKernel,
                    float,
                    double,
+                   phi::dtype::bfloat16,
                    phi::dtype::float16) {}
diff --git a/python/paddle/fluid/tests/unittests/test_mul_op.py b/python/paddle/fluid/tests/unittests/test_mul_op.py
index 1e2b909d90dab..9ce95d354d7af 100644
--- a/python/paddle/fluid/tests/unittests/test_mul_op.py
+++ b/python/paddle/fluid/tests/unittests/test_mul_op.py
@@ -20,7 +20,7 @@
 from paddle.fluid import core
 
 sys.path.append("..")
-from eager_op_test import OpTest
+from eager_op_test import OpTest, convert_float_to_uint16
 
 
 class TestMulOp(OpTest):
@@ -114,14 +114,14 @@ def test_check_grad_ignore_y(self):
 @unittest.skipIf(
     not core.is_compiled_with_cuda(), "core is not compiled with CUDA"
 )
-class TestFP16MulOp1(TestMulOp):
+class TestMulFP16Op1(TestMulOp):
     def init_dtype_type(self):
         self.dtype = np.float16
 
     def test_check_output(self):
         place = core.CUDAPlace(0)
         if core.is_float16_supported(place):
-            self.check_output_with_place(place, atol=1e-1, check_dygraph=False)
+            self.check_output_with_place(place, check_dygraph=False)
 
     def test_check_grad_normal(self):
         place = core.CUDAPlace(0)
@@ -130,7 +130,6 @@ def test_check_grad_normal(self):
                 place,
                 ['X', 'Y'],
                 'Out',
-                max_relative_error=0.5,
                 check_dygraph=False,
             )
 
@@ -141,7 +140,6 @@ def test_check_grad_ingore_x(self):
                 place,
                 ['Y'],
                 'Out',
-                max_relative_error=0.5,
                 no_grad_set=set("X"),
                 check_dygraph=False,
             )
@@ -153,7 +151,6 @@ def test_check_grad_ingore_y(self):
                 place,
                 ['X'],
                 'Out',
-                max_relative_error=0.5,
                 no_grad_set=set('Y'),
                 check_dygraph=False,
             )
@@ -162,14 +159,14 @@ def test_check_grad_ingore_y(self):
 @unittest.skipIf(
     not core.is_compiled_with_cuda(), "core is not compiled with CUDA"
 )
-class TestFP16MulOp2(TestMulOp2):
+class TestMulFP16Op2(TestMulOp2):
     def init_dtype_type(self):
         self.dtype = np.float16
 
     def test_check_output(self):
         place = core.CUDAPlace(0)
         if core.is_float16_supported(place):
-            self.check_output_with_place(place, atol=2e-1, check_dygraph=False)
+            self.check_output_with_place(place, check_dygraph=False)
 
     def test_check_grad_normal(self):
         place = core.CUDAPlace(0)
@@ -178,7 +175,6 @@ def test_check_grad_normal(self):
                 place,
                 ['X', 'Y'],
                 'Out',
-                max_relative_error=0.9,
                 check_dygraph=False,
             )
 
@@ -189,7 +185,6 @@ def test_check_grad_ingore_x(self):
                 place,
                 ['Y'],
                 'Out',
-                max_relative_error=0.5,
                 no_grad_set=set("X"),
                 check_dygraph=False,
             )
@@ -201,11 +196,120 @@ def test_check_grad_ingore_y(self):
                 place,
                 ['X'],
                 'Out',
-                max_relative_error=0.9,
                 no_grad_set=set('Y'),
                 check_dygraph=False,
             )
 
 
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support bfloat16",
+)
+class TestMulBF16Op1(OpTest):
+    def setUp(self):
+        self.op_type = "mul"
+        self.init_dtype_type()
+        self.inputs = {
+            'X': np.random.random((20, 5)).astype(self.np_dtype),
+            'Y': np.random.random((5, 21)).astype(self.np_dtype),
+        }
+        self.outputs = {'Out': np.dot(self.inputs['X'], self.inputs['Y'])}
+
+        self.inputs['X'] = convert_float_to_uint16(self.inputs['X'])
+        self.inputs['Y'] = convert_float_to_uint16(self.inputs['Y'])
+        self.outputs['Out'] = convert_float_to_uint16(self.outputs['Out'])
+        self.place = core.CUDAPlace(0)
+
+    def init_dtype_type(self):
+        self.dtype = np.uint16
+        self.np_dtype = np.float32
+
+    def test_check_output(self):
+        self.check_output_with_place(self.place, check_dygraph=False)
+
+    def test_check_grad_normal(self):
+        self.check_grad_with_place(
+            self.place, ['X', 'Y'], 'Out', check_dygraph=False
+        )
+
+    def test_check_grad_ingore_x(self):
+        self.check_grad_with_place(
+            self.place,
+            ['Y'],
+            'Out',
+            no_grad_set=set("X"),
+            check_dygraph=False,
+        )
+
+    def test_check_grad_ingore_y(self):
+        self.check_grad_with_place(
+            self.place,
+            ['X'],
+            'Out',
+            no_grad_set=set('Y'),
+            check_dygraph=False,
+        )
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support bfloat16",
+)
+class TestMulBF16Op2(TestMulBF16Op1):
+    def setUp(self):
+        self.op_type = "mul"
+        self.init_dtype_type()
+        self.inputs = {
+            'X': np.random.random((3, 4, 2, 9)).astype(self.np_dtype),
+            'Y': np.random.random((3, 6, 1, 2, 3)).astype(self.np_dtype),
+        }
+        self.attrs = {
+            'x_num_col_dims': 2,
+            'y_num_col_dims': 2,
+        }
+        result = np.dot(
+            self.inputs['X'].reshape(3 * 4, 2 * 9),
+            self.inputs['Y'].reshape(3 * 6, 1 * 2 * 3),
+        )
+        result = result.reshape(3, 4, 1, 2, 3)
+        self.outputs = {'Out': result}
+
+        self.inputs['X'] = convert_float_to_uint16(self.inputs['X'])
+        self.inputs['Y'] = convert_float_to_uint16(self.inputs['Y'])
+        self.outputs['Out'] = convert_float_to_uint16(self.outputs['Out'])
+        self.place = core.CUDAPlace(0)
+
+    def test_check_grad_normal(self):
+        self.check_grad_with_place(
+            self.place,
+            ['X', 'Y'],
+            'Out',
+            numeric_grad_delta=0.02,
+            check_dygraph=False,
+        )
+
+    def test_check_grad_ingore_x(self):
+        self.check_grad_with_place(
+            self.place,
+            ['Y'],
+            'Out',
+            numeric_grad_delta=0.02,
+            no_grad_set=set("X"),
+            check_dygraph=False,
+        )
+
+    def test_check_grad_ingore_y(self):
+        self.check_grad_with_place(
+            self.place,
+            ['X'],
+            'Out',
+            numeric_grad_delta=0.02,
+            no_grad_set=set('Y'),
+            check_dygraph=False,
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

From 3a340cc736cb87567582e7bf60caab80b5b8feb3 Mon Sep 17 00:00:00 2001
From: tianshuo78520a <707759223@qq.com>
Date: Thu, 4 May 2023 14:38:00 +0800
Subject: [PATCH 155/405] Mv prim test (#53466)

---
 .../fluid/tests/unittests => test}/prim/test_comp_dispensable.py  | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename {python/paddle/fluid/tests/unittests => test}/prim/test_comp_dispensable.py (100%)

diff --git a/python/paddle/fluid/tests/unittests/prim/test_comp_dispensable.py b/test/prim/test_comp_dispensable.py
similarity index 100%
rename from python/paddle/fluid/tests/unittests/prim/test_comp_dispensable.py
rename to test/prim/test_comp_dispensable.py

From 49d7bc5cb5a065fd8dd55368a6c75a248dfd7bab Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Thu, 4 May 2023 15:20:22 +0800
Subject: [PATCH 156/405] [test]mv fluid op [reader, prim_ops, nccl,
 reduce_ops, lite] to test (#53429)

* [test]mv fluid reader to test/

* [test]mv fluid op prim_ops to test/cpp/fluid/prim_ops

* [test]mv fluid op nccl to /test/cpp/fluid/nccl/

* [test]mv fluid op reduce_ops to test/cpp/fluid/reduce_ops

* [test]mv fluid op lite to test/cpp/fluid/lite

* [test]fix lite

* [test]fix prim op path

* [fluid]clean prim ops cmakelists
---
 paddle/fluid/operators/lite/CMakeLists.txt    |  2 -
 paddle/fluid/operators/nccl/CMakeLists.txt    | 14 -------
 .../fluid/operators/prim_ops/CMakeLists.txt   | 39 -------------------
 paddle/fluid/operators/reader/CMakeLists.txt  |  1 -
 .../fluid/operators/reduce_ops/CMakeLists.txt | 21 ----------
 test/cpp/fluid/CMakeLists.txt                 | 13 ++++---
 test/cpp/fluid/lite/CMakeLists.txt            |  2 +
 .../cpp/fluid}/lite/lite_engine_op_test.cc    |  0
 test/cpp/fluid/nccl/CMakeLists.txt            | 13 +++++++
 .../cpp/fluid}/nccl/nccl_op_test.cu.cc        |  0
 test/cpp/fluid/prim_ops/CMakeLists.txt        | 39 +++++++++++++++++++
 .../cpp/fluid}/prim_ops/prim_op_test.cc       |  0
 test/cpp/fluid/reader/CMakeLists.txt          |  1 +
 .../reader/reader_blocking_queue_test.cc      |  0
 test/cpp/fluid/reduce_ops/CMakeLists.txt      | 20 ++++++++++
 .../reduce_ops/check_reduce_rank_test.cu      |  0
 16 files changed, 82 insertions(+), 83 deletions(-)
 create mode 100644 test/cpp/fluid/lite/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/lite/lite_engine_op_test.cc (100%)
 create mode 100644 test/cpp/fluid/nccl/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/nccl/nccl_op_test.cu.cc (100%)
 create mode 100644 test/cpp/fluid/prim_ops/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/prim_ops/prim_op_test.cc (100%)
 create mode 100644 test/cpp/fluid/reader/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/reader/reader_blocking_queue_test.cc (100%)
 create mode 100644 test/cpp/fluid/reduce_ops/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/reduce_ops/check_reduce_rank_test.cu (100%)

diff --git a/paddle/fluid/operators/lite/CMakeLists.txt b/paddle/fluid/operators/lite/CMakeLists.txt
index c272e8bc94c50..ca3b62648378b 100644
--- a/paddle/fluid/operators/lite/CMakeLists.txt
+++ b/paddle/fluid/operators/lite/CMakeLists.txt
@@ -1,3 +1 @@
 op_library(lite_engine_op DEPS lite_engine lite_tensor_utils)
-cc_test_old(test_lite_engine_op SRCS lite_engine_op_test.cc DEPS lite_engine_op
-            analysis)
diff --git a/paddle/fluid/operators/nccl/CMakeLists.txt b/paddle/fluid/operators/nccl/CMakeLists.txt
index 218d53aa6303a..629b41b4b582b 100644
--- a/paddle/fluid/operators/nccl/CMakeLists.txt
+++ b/paddle/fluid/operators/nccl/CMakeLists.txt
@@ -22,17 +22,3 @@ if(WITH_GPU OR WITH_ROCM)
       ${OPERATOR_DEPS} nccl_common
       PARENT_SCOPE)
 endif()
-
-if(WITH_GPU AND NOT WIN32)
-  nv_test(
-    nccl_op_test
-    SRCS nccl_op_test.cu.cc
-    DEPS nccl_op gpu_info device_context)
-endif()
-
-if(WITH_ROCM AND NOT WIN32)
-  hip_test(
-    nccl_op_test
-    SRCS nccl_op_test.cu.cc
-    DEPS nccl_op gpu_info device_context)
-endif()
diff --git a/paddle/fluid/operators/prim_ops/CMakeLists.txt b/paddle/fluid/operators/prim_ops/CMakeLists.txt
index 85741b2f5f3a8..7a1278219bb6d 100644
--- a/paddle/fluid/operators/prim_ops/CMakeLists.txt
+++ b/paddle/fluid/operators/prim_ops/CMakeLists.txt
@@ -4,42 +4,3 @@ if(WITH_UNITY_BUILD)
   include(unity_build_rule.cmake)
 endif()
 register_operators()
-
-set(PRIM_OP_SRCS
-    reshape_p_op.cc
-    broadcast_p_op.cc
-    reduce_sum_p_op.cc
-    transpose_p_op.cc
-    split_p_op.cc
-    concat_p_op.cc
-    slice_select_p_op.cc
-    slice_assign_p_op.cc
-    gather_p_op.cc
-    scatter_add_p_op.cc
-    add_p_op.cc
-    sub_p_op.cc
-    mul_p_op.cc
-    div_p_op.cc
-    sqrt_p_op.cc
-    tanh_p_op.cc
-    sin_p_op.cc
-    cos_p_op.cc
-    exp_p_op.cc
-    matmul_p_op.cc
-    fill_constant_p_op.cc
-    log_p_op.cc
-    select_p_op.cc
-    eq_p_op.cc
-    gt_p_op.cc
-    ge_p_op.cc
-    ne_p_op.cc
-    pow_p_op.cc
-    max_p_op.cc
-    erf_p_op.cc
-    bernoulli_p_op.cc
-    abs_p_op.cc
-    cast_p_op.cc
-    rsqrt_p_op.cc
-    uniform_random_p_op.cc)
-
-cc_test_old(prim_op_test SRCS prim_op_test.cc ${PRIM_OP_SRCS} DEPS op_registry)
diff --git a/paddle/fluid/operators/reader/CMakeLists.txt b/paddle/fluid/operators/reader/CMakeLists.txt
index 9dd59de98d553..61622f516c451 100644
--- a/paddle/fluid/operators/reader/CMakeLists.txt
+++ b/paddle/fluid/operators/reader/CMakeLists.txt
@@ -35,6 +35,5 @@ reader_library(create_py_reader_op SRCS create_py_reader_op.cc DEPS py_reader)
 
 op_library(read_op DEPS py_reader buffered_reader)
 
-cc_test(reader_blocking_queue_test SRCS reader_blocking_queue_test.cc)
 # Export local libraries to parent
 # set(READER_LIBRARY ${LOCAL_READER_LIBS} PARENT_SCOPE)
diff --git a/paddle/fluid/operators/reduce_ops/CMakeLists.txt b/paddle/fluid/operators/reduce_ops/CMakeLists.txt
index 27a2ff68d3aad..070c9f7262a45 100644
--- a/paddle/fluid/operators/reduce_ops/CMakeLists.txt
+++ b/paddle/fluid/operators/reduce_ops/CMakeLists.txt
@@ -12,24 +12,3 @@ if(WITH_GPU)
 else()
   register_operators()
 endif()
-
-if(WITH_GPU)
-  if(${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0)
-    nv_test(
-      check_reduce_rank_test
-      SRCS check_reduce_rank_test.cu
-      DEPS tensor cub)
-  else()
-    nv_test(
-      check_reduce_rank_test
-      SRCS check_reduce_rank_test.cu
-      DEPS tensor)
-  endif()
-endif()
-
-if(WITH_ROCM)
-  hip_test(
-    check_reduce_rank_test
-    SRCS check_reduce_rank_test.cu
-    DEPS tensor)
-endif()
diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt
index cabad612e0d07..31ba506aa9155 100644
--- a/test/cpp/fluid/CMakeLists.txt
+++ b/test/cpp/fluid/CMakeLists.txt
@@ -7,17 +7,18 @@ endif()
 # add_subdirectory(dlnne)
 # add_subdirectory(elementwise)
 # add_subdirectory(fused)
-# if(WITH_LITE)
-#   add_subdirectory(lite)
-# endif()
+if(WITH_LITE)
+  add_subdirectory(lite)
+endif()
 # add_subdirectory(math)
 # if(WITH_MKLDNN)
 #   add_subdirectory(mkldnn)
 # endif()
-# add_subdirectory(nccl)
+add_subdirectory(nccl)
 # if(WITH_PSCORE)
 #   add_subdirectory(pscore)
 # endif()
-# add_subdirectory(prim_ops)
-# add_subdirectory(reduce_ops)
+add_subdirectory(prim_ops)
+add_subdirectory(reader)
+add_subdirectory(reduce_ops)
 # add_subdirectory(tensorrt)
diff --git a/test/cpp/fluid/lite/CMakeLists.txt b/test/cpp/fluid/lite/CMakeLists.txt
new file mode 100644
index 0000000000000..c9dbdb8665eba
--- /dev/null
+++ b/test/cpp/fluid/lite/CMakeLists.txt
@@ -0,0 +1,2 @@
+cc_test_old(test_lite_engine_op SRCS lite_engine_op_test.cc DEPS lite_engine_op
+            analysis)
diff --git a/paddle/fluid/operators/lite/lite_engine_op_test.cc b/test/cpp/fluid/lite/lite_engine_op_test.cc
similarity index 100%
rename from paddle/fluid/operators/lite/lite_engine_op_test.cc
rename to test/cpp/fluid/lite/lite_engine_op_test.cc
diff --git a/test/cpp/fluid/nccl/CMakeLists.txt b/test/cpp/fluid/nccl/CMakeLists.txt
new file mode 100644
index 0000000000000..d0482bee421ea
--- /dev/null
+++ b/test/cpp/fluid/nccl/CMakeLists.txt
@@ -0,0 +1,13 @@
+if(WITH_GPU AND NOT WIN32)
+  nv_test(
+    nccl_op_test
+    SRCS nccl_op_test.cu.cc
+    DEPS nccl_op gpu_info device_context)
+endif()
+
+if(WITH_ROCM AND NOT WIN32)
+  hip_test(
+    nccl_op_test
+    SRCS nccl_op_test.cu.cc
+    DEPS nccl_op gpu_info device_context)
+endif()
diff --git a/paddle/fluid/operators/nccl/nccl_op_test.cu.cc b/test/cpp/fluid/nccl/nccl_op_test.cu.cc
similarity index 100%
rename from paddle/fluid/operators/nccl/nccl_op_test.cu.cc
rename to test/cpp/fluid/nccl/nccl_op_test.cu.cc
diff --git a/test/cpp/fluid/prim_ops/CMakeLists.txt b/test/cpp/fluid/prim_ops/CMakeLists.txt
new file mode 100644
index 0000000000000..578284c261165
--- /dev/null
+++ b/test/cpp/fluid/prim_ops/CMakeLists.txt
@@ -0,0 +1,39 @@
+set(PRIM_OP_PATH ${PADDLE_SOURCE_DIR}/paddle/fluid/operators/prim_ops/)
+
+set(PRIM_OP_SRCS
+    ${PRIM_OP_PATH}reshape_p_op.cc
+    ${PRIM_OP_PATH}broadcast_p_op.cc
+    ${PRIM_OP_PATH}reduce_sum_p_op.cc
+    ${PRIM_OP_PATH}transpose_p_op.cc
+    ${PRIM_OP_PATH}split_p_op.cc
+    ${PRIM_OP_PATH}concat_p_op.cc
+    ${PRIM_OP_PATH}slice_select_p_op.cc
+    ${PRIM_OP_PATH}slice_assign_p_op.cc
+    ${PRIM_OP_PATH}gather_p_op.cc
+    ${PRIM_OP_PATH}scatter_add_p_op.cc
+    ${PRIM_OP_PATH}add_p_op.cc
+    ${PRIM_OP_PATH}sub_p_op.cc
+    ${PRIM_OP_PATH}mul_p_op.cc
+    ${PRIM_OP_PATH}div_p_op.cc
+    ${PRIM_OP_PATH}sqrt_p_op.cc
+    ${PRIM_OP_PATH}tanh_p_op.cc
+    ${PRIM_OP_PATH}sin_p_op.cc
+    ${PRIM_OP_PATH}cos_p_op.cc
+    ${PRIM_OP_PATH}exp_p_op.cc
+    ${PRIM_OP_PATH}matmul_p_op.cc
+    ${PRIM_OP_PATH}fill_constant_p_op.cc
+    ${PRIM_OP_PATH}log_p_op.cc
+    ${PRIM_OP_PATH}select_p_op.cc
+    ${PRIM_OP_PATH}eq_p_op.cc
+    ${PRIM_OP_PATH}gt_p_op.cc
+    ${PRIM_OP_PATH}ge_p_op.cc
+    ${PRIM_OP_PATH}ne_p_op.cc
+    ${PRIM_OP_PATH}pow_p_op.cc
+    ${PRIM_OP_PATH}max_p_op.cc
+    ${PRIM_OP_PATH}erf_p_op.cc
+    ${PRIM_OP_PATH}bernoulli_p_op.cc
+    ${PRIM_OP_PATH}abs_p_op.cc
+    ${PRIM_OP_PATH}cast_p_op.cc
+    ${PRIM_OP_PATH}rsqrt_p_op.cc
+    ${PRIM_OP_PATH}uniform_random_p_op.cc)
+cc_test_old(prim_op_test SRCS prim_op_test.cc ${PRIM_OP_SRCS} DEPS op_registry)
diff --git a/paddle/fluid/operators/prim_ops/prim_op_test.cc b/test/cpp/fluid/prim_ops/prim_op_test.cc
similarity index 100%
rename from paddle/fluid/operators/prim_ops/prim_op_test.cc
rename to test/cpp/fluid/prim_ops/prim_op_test.cc
diff --git a/test/cpp/fluid/reader/CMakeLists.txt b/test/cpp/fluid/reader/CMakeLists.txt
new file mode 100644
index 0000000000000..1bd2ad7cdd0c1
--- /dev/null
+++ b/test/cpp/fluid/reader/CMakeLists.txt
@@ -0,0 +1 @@
+cc_test(reader_blocking_queue_test SRCS reader_blocking_queue_test.cc)
diff --git a/paddle/fluid/operators/reader/reader_blocking_queue_test.cc b/test/cpp/fluid/reader/reader_blocking_queue_test.cc
similarity index 100%
rename from paddle/fluid/operators/reader/reader_blocking_queue_test.cc
rename to test/cpp/fluid/reader/reader_blocking_queue_test.cc
diff --git a/test/cpp/fluid/reduce_ops/CMakeLists.txt b/test/cpp/fluid/reduce_ops/CMakeLists.txt
new file mode 100644
index 0000000000000..8c04b94138ba8
--- /dev/null
+++ b/test/cpp/fluid/reduce_ops/CMakeLists.txt
@@ -0,0 +1,20 @@
+if(WITH_GPU)
+  if(${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0)
+    nv_test(
+      check_reduce_rank_test
+      SRCS check_reduce_rank_test.cu
+      DEPS tensor cub)
+  else()
+    nv_test(
+      check_reduce_rank_test
+      SRCS check_reduce_rank_test.cu
+      DEPS tensor)
+  endif()
+endif()
+
+if(WITH_ROCM)
+  hip_test(
+    check_reduce_rank_test
+    SRCS check_reduce_rank_test.cu
+    DEPS tensor)
+endif()
diff --git a/paddle/fluid/operators/reduce_ops/check_reduce_rank_test.cu b/test/cpp/fluid/reduce_ops/check_reduce_rank_test.cu
similarity index 100%
rename from paddle/fluid/operators/reduce_ops/check_reduce_rank_test.cu
rename to test/cpp/fluid/reduce_ops/check_reduce_rank_test.cu

From ace61b8ba538c77882bcaef78cf25fbf13c4e8ba Mon Sep 17 00:00:00 2001
From: weishengying <63448337+weishengying@users.noreply.github.com>
Date: Thu, 4 May 2023 16:30:45 +0800
Subject: [PATCH 157/405] Fix a bug in constant folding pass (#53456)

---
 paddle/fluid/framework/ir/constant_folding_pass.cc | 2 +-
 paddle/fluid/inference/api/paddle_pass_builder.cc  | 2 +-
 paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc      | 4 +++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/paddle/fluid/framework/ir/constant_folding_pass.cc b/paddle/fluid/framework/ir/constant_folding_pass.cc
index 74d8e4a29a73c..9e3d1d5c08c72 100644
--- a/paddle/fluid/framework/ir/constant_folding_pass.cc
+++ b/paddle/fluid/framework/ir/constant_folding_pass.cc
@@ -64,7 +64,7 @@ void ConstantFoldingPass::ApplyImpl(ir::Graph *graph) const {
       platform::errors::Fatal(
           "scope must not be null when applying constant floding."));
 
-  std::vector<std::string> blacklist{"feed", "matrix_multiply"};
+  std::vector<std::string> blacklist{"feed", "matrix_multiply", "save"};
 
   auto op_node_sorted = framework::ir::TopologyVarientSort(
       *graph, static_cast<framework::ir::SortKind>(0));
diff --git a/paddle/fluid/inference/api/paddle_pass_builder.cc b/paddle/fluid/inference/api/paddle_pass_builder.cc
index e7c24272b81c5..134d56180b617 100644
--- a/paddle/fluid/inference/api/paddle_pass_builder.cc
+++ b/paddle/fluid/inference/api/paddle_pass_builder.cc
@@ -420,6 +420,7 @@ void CpuPassStrategy::EnableMkldnnInt8() {
     passes_.push_back("simplify_with_basic_ops_pass");
     passes_.push_back("quant_dequant_mkldnn_pass");
     passes_.push_back("mkldnn_placement_pass");
+    passes_.push_back("constant_folding_pass");
     passes_.push_back("squeeze2_transpose2_onednn_fuse_pass");
     passes_.push_back("layer_norm_fuse_pass");
     passes_.push_back("attention_lstm_fuse_pass");
@@ -474,7 +475,6 @@ void CpuPassStrategy::EnableMkldnnInt8() {
     passes_.push_back("quant_transpose2_dequant_onednn_fuse_pass");
     passes_.push_back("int8_scale_calculation_mkldnn_pass");
     passes_.push_back("params_quantization_mkldnn_pass");
-    passes_.push_back("constant_folding_pass");
   }
   use_mkldnn_int8_ = true;
 #else
diff --git a/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
index a5274c5f7ae7c..d408514e83921 100644
--- a/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
@@ -192,7 +192,9 @@ class FCMKLDNNHandler
     } else {
       auto scale_in_data = ctx.Attr<float>("Scale_in");
       auto scale_weights_data = ctx.Attr<std::vector<float>>("Scale_weights");
-      bool has_activation = !ctx.Attr<std::string>("activation_type").empty();
+      bool has_activation = !ctx.Attr<std::string>("activation_type").empty() ||
+                            (ctx.HasAttr("fuse_activation") &&
+                             !ctx.Attr<std::string>("fuse_activation").empty());
       bool force_fp32_output = ctx.Attr<bool>("force_fp32_output");
       bool fuse_residual_conn = ctx.HasAttr("fuse_residual_connection") &&
                                 ctx.Attr<bool>("fuse_residual_connection");

From 19950e65e2cd1c20b7e1a0699d4898862e3bb11e Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Thu, 4 May 2023 16:57:02 +0800
Subject: [PATCH 158/405] mv fluid op elementwise to test/cpp/fluid/elementwise
 (#53448)

---
 paddle/fluid/operators/elementwise/CMakeLists.txt   | 13 -------------
 test/cpp/fluid/CMakeLists.txt                       |  2 +-
 test/cpp/fluid/elementwise/CMakeLists.txt           | 12 ++++++++++++
 .../elementwise/test_elementwise_add_grad_grad.cc   |  2 +-
 .../elementwise/test_elementwise_add_op_inplace.cc  |  0
 .../elementwise/test_elementwise_div_grad_grad.cc   |  2 +-
 .../elementwise/test_elementwise_op_grad_grad.h     |  0
 7 files changed, 15 insertions(+), 16 deletions(-)
 create mode 100644 test/cpp/fluid/elementwise/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/elementwise/test_elementwise_add_grad_grad.cc (97%)
 rename {paddle/fluid/operators => test/cpp/fluid}/elementwise/test_elementwise_add_op_inplace.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/elementwise/test_elementwise_div_grad_grad.cc (97%)
 rename {paddle/fluid/operators => test/cpp/fluid}/elementwise/test_elementwise_op_grad_grad.h (100%)

diff --git a/paddle/fluid/operators/elementwise/CMakeLists.txt b/paddle/fluid/operators/elementwise/CMakeLists.txt
index e0714041dfabe..2276d82320473 100644
--- a/paddle/fluid/operators/elementwise/CMakeLists.txt
+++ b/paddle/fluid/operators/elementwise/CMakeLists.txt
@@ -4,16 +4,3 @@ if(WITH_UNITY_BUILD)
   include(unity_build_rule.cmake)
 endif()
 register_operators(DEPS op_version_registry)
-
-cc_test(
-  test_elementwise_add_op_inplace
-  SRCS test_elementwise_add_op_inplace.cc
-  DEPS op_registry elementwise_add_op scope device_context enforce executor)
-cc_test(
-  test_elementwise_div_grad_grad
-  SRCS test_elementwise_div_grad_grad.cc
-  DEPS op_registry elementwise_div_op scope device_context enforce executor)
-cc_test(
-  test_elementwise_add_grad_grad
-  SRCS test_elementwise_add_grad_grad.cc
-  DEPS op_registry elementwise_add_op scope device_context enforce executor)
diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt
index 31ba506aa9155..45134d4b38a03 100644
--- a/test/cpp/fluid/CMakeLists.txt
+++ b/test/cpp/fluid/CMakeLists.txt
@@ -5,7 +5,7 @@ endif()
 # add_subdirectory(controlflow)
 # add_subdirectory(detection)
 # add_subdirectory(dlnne)
-# add_subdirectory(elementwise)
+add_subdirectory(elementwise)
 # add_subdirectory(fused)
 if(WITH_LITE)
   add_subdirectory(lite)
diff --git a/test/cpp/fluid/elementwise/CMakeLists.txt b/test/cpp/fluid/elementwise/CMakeLists.txt
new file mode 100644
index 0000000000000..25fffee90b93a
--- /dev/null
+++ b/test/cpp/fluid/elementwise/CMakeLists.txt
@@ -0,0 +1,12 @@
+cc_test(
+  test_elementwise_add_op_inplace
+  SRCS test_elementwise_add_op_inplace.cc
+  DEPS op_registry elementwise_add_op scope device_context enforce executor)
+cc_test(
+  test_elementwise_div_grad_grad
+  SRCS test_elementwise_div_grad_grad.cc
+  DEPS op_registry elementwise_div_op scope device_context enforce executor)
+cc_test(
+  test_elementwise_add_grad_grad
+  SRCS test_elementwise_add_grad_grad.cc
+  DEPS op_registry elementwise_add_op scope device_context enforce executor)
diff --git a/paddle/fluid/operators/elementwise/test_elementwise_add_grad_grad.cc b/test/cpp/fluid/elementwise/test_elementwise_add_grad_grad.cc
similarity index 97%
rename from paddle/fluid/operators/elementwise/test_elementwise_add_grad_grad.cc
rename to test/cpp/fluid/elementwise/test_elementwise_add_grad_grad.cc
index 8afb311494182..4fefd8864c5e9 100644
--- a/paddle/fluid/operators/elementwise/test_elementwise_add_grad_grad.cc
+++ b/test/cpp/fluid/elementwise/test_elementwise_add_grad_grad.cc
@@ -14,10 +14,10 @@
 
 #include "gtest/gtest.h"
 #include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/operators/elementwise/test_elementwise_op_grad_grad.h"
 #include "paddle/fluid/platform/place.h"
 #include "paddle/phi/core/ddim.h"
 #include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/fluid/elementwise/test_elementwise_op_grad_grad.h"
 
 USE_OP_ITSELF(elementwise_add);
 PD_DECLARE_KERNEL(add_double_grad, CPU, ALL_LAYOUT);
diff --git a/paddle/fluid/operators/elementwise/test_elementwise_add_op_inplace.cc b/test/cpp/fluid/elementwise/test_elementwise_add_op_inplace.cc
similarity index 100%
rename from paddle/fluid/operators/elementwise/test_elementwise_add_op_inplace.cc
rename to test/cpp/fluid/elementwise/test_elementwise_add_op_inplace.cc
diff --git a/paddle/fluid/operators/elementwise/test_elementwise_div_grad_grad.cc b/test/cpp/fluid/elementwise/test_elementwise_div_grad_grad.cc
similarity index 97%
rename from paddle/fluid/operators/elementwise/test_elementwise_div_grad_grad.cc
rename to test/cpp/fluid/elementwise/test_elementwise_div_grad_grad.cc
index df0e6a1391e06..e74da1c884f67 100644
--- a/paddle/fluid/operators/elementwise/test_elementwise_div_grad_grad.cc
+++ b/test/cpp/fluid/elementwise/test_elementwise_div_grad_grad.cc
@@ -24,11 +24,11 @@
 #include "paddle/fluid/framework/op_registry.h"
 #include "paddle/fluid/framework/operator.h"
 #include "paddle/fluid/framework/scope.h"
-#include "paddle/fluid/operators/elementwise/test_elementwise_op_grad_grad.h"
 #include "paddle/fluid/platform/device_context.h"
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/platform/place.h"
 #include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/fluid/elementwise/test_elementwise_op_grad_grad.h"
 
 USE_OP_ITSELF(elementwise_div);
 
diff --git a/paddle/fluid/operators/elementwise/test_elementwise_op_grad_grad.h b/test/cpp/fluid/elementwise/test_elementwise_op_grad_grad.h
similarity index 100%
rename from paddle/fluid/operators/elementwise/test_elementwise_op_grad_grad.h
rename to test/cpp/fluid/elementwise/test_elementwise_op_grad_grad.h

From eda0c588f2a17fb2d935ee4ad1ed640e7ccd7f71 Mon Sep 17 00:00:00 2001
From: Roc <30228238+sljlp@users.noreply.github.com>
Date: Thu, 4 May 2023 19:08:53 +0800
Subject: [PATCH 159/405] revert p2p communication for xpu (#53496)

---
 .../pp_utils/p2p_communication.py             | 26 -------------------
 1 file changed, 26 deletions(-)

diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
index 2c30dc8bedfe0..7d7a02d3e25a7 100644
--- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
+++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
@@ -24,26 +24,6 @@
 _use_cache = False
 _enable_partial_send_recv = True
 
-_xpu_comm_group_started = False
-
-
-def _xpu_comm_group_start():
-    if not paddle.is_compiled_with_xpu():
-        return
-    global _xpu_comm_group_started
-    assert not _xpu_comm_group_started
-    framework.core.ProcessGroupBKCL.group_start()
-    _xpu_comm_group_started = True
-
-
-def _xpu_comm_group_end():
-    if not paddle.is_compiled_with_xpu():
-        return
-    global _xpu_comm_group_started
-    if _xpu_comm_group_started:
-        framework.core.ProcessGroupBKCL.group_end()
-        _xpu_comm_group_started = False
-
 
 def initialize_p2p_groups(hcg, use_cache=True, enable_partial_send_recv=True):
     global _hcg, _use_cache, _enable_partial_send_recv
@@ -371,7 +351,6 @@ def _p2p_helper(
     # TODO(Yuang Liu): use batch_isend_irecv replace all these comm ops
     tasks = []
     # start to p2p communicate
-    _xpu_comm_group_start()
     if tensor_send_prev is not None:
         if isinstance(tensor_send_prev, tuple):
             for d in tensor_send_prev:
@@ -407,7 +386,6 @@ def _p2p_helper(
                     use_calc_stream=sync_recv,
                 )
                 if sync_recv:
-                    _xpu_comm_group_end()
                     allgather_partial(
                         d,
                         nranks=mp_degree,
@@ -428,7 +406,6 @@ def _p2p_helper(
             )
 
             if sync_recv:
-                _xpu_comm_group_end()
                 allgather_partial(
                     tensor_recv_prev,
                     nranks=mp_degree,
@@ -475,7 +452,6 @@ def _p2p_helper(
                 )
 
                 if sync_recv:
-                    _xpu_comm_group_end()
                     allgather_partial(
                         d,
                         nranks=mp_degree,
@@ -496,7 +472,6 @@ def _p2p_helper(
                 use_calc_stream=sync_recv,
             )
             if sync_recv:
-                _xpu_comm_group_end()
                 allgather_partial(
                     tensor_recv_next,
                     nranks=mp_degree,
@@ -506,7 +481,6 @@ def _p2p_helper(
                 )
             else:
                 tasks.append(task)
-    _xpu_comm_group_end()
     if not sync_recv:
         if framework.in_dygraph_mode():
             # wait irecv tasks in eager dygraph mode with new comm library

From 74074a8d23574fb04d8d57b770a7a094050d0f63 Mon Sep 17 00:00:00 2001
From: zhupengyang <zhu_py@qq.com>
Date: Fri, 5 May 2023 09:54:36 +0800
Subject: [PATCH 160/405] fix xpu max size (#53505)

---
 paddle/phi/infermeta/fusion.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/paddle/phi/infermeta/fusion.cc b/paddle/phi/infermeta/fusion.cc
index f775cedce8c11..fb7ede4c9e742 100644
--- a/paddle/phi/infermeta/fusion.cc
+++ b/paddle/phi/infermeta/fusion.cc
@@ -158,7 +158,7 @@ void Conv2dXPUInferMeta(const MetaTensor& x,
   }
   // set output and output max dims
   out->set_dims(DDim(out_shape.data(), out_shape.size()));
-  out_max->set_dims(phi::make_ddim({4}));
+  out_max->set_dims(phi::make_ddim({6}));
 }
 
 void EmbeddingWithEltwiseAddXPUInferMeta(
@@ -204,7 +204,7 @@ void FcXPUInferMeta(const MetaTensor& x,
   out->set_dims(DDim(out_shape.data(), out_shape.size()));
   out->set_dtype(x.dtype());
   out->set_layout(x.layout());
-  out_max->set_dims(w_max.dims());
+  out_max->set_dims(phi::make_ddim({6}));
   out_max->set_dtype(x.dtype());
   out_max->set_layout(x.layout());
 }

From 841efcd4de5afcd2822307e099e1de59d014321c Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Fri, 5 May 2023 10:18:51 +0800
Subject: [PATCH 161/405] =?UTF-8?q?=E3=80=90Hackathon=20No.59=E3=80=91addm?=
 =?UTF-8?q?m=20=E7=AE=97=E5=AD=90FP16/BF16=E5=8D=95=E6=B5=8B=E5=AE=8C?=
 =?UTF-8?q?=E5=96=84=20(#53111)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add addmm tests

* Fix code
---
 paddle/phi/kernels/funcs/blas/blas_impl.cu.h  |  68 ++++++++++++
 paddle/phi/kernels/funcs/blas/blas_impl.hip.h |  66 ++++++++++-
 paddle/phi/kernels/gpu/addmm_grad_kernel.cu   |  10 +-
 paddle/phi/kernels/gpu/addmm_kernel.cu        |   9 +-
 .../phi/kernels/impl/addmm_grad_kernel_impl.h | 104 ++++++++++++++++--
 paddle/phi/kernels/impl/addmm_kernel_impl.h   |   6 +-
 .../fluid/tests/unittests/test_addmm_op.py    |  63 ++++++++++-
 python/paddle/tensor/math.py                  |  10 +-
 8 files changed, 313 insertions(+), 23 deletions(-)

diff --git a/paddle/phi/kernels/funcs/blas/blas_impl.cu.h b/paddle/phi/kernels/funcs/blas/blas_impl.cu.h
index 35769d0e6e338..e39c860bc8096 100644
--- a/paddle/phi/kernels/funcs/blas/blas_impl.cu.h
+++ b/paddle/phi/kernels/funcs/blas/blas_impl.cu.h
@@ -1316,6 +1316,74 @@ inline void Blas<phi::GPUContext>::GEMM(bool transA,
   });
 }
 
+template <>
+template <>
+inline void Blas<phi::GPUContext>::GEMM(bool transA,
+                                        bool transB,
+                                        int M,
+                                        int N,
+                                        int K,
+                                        phi::dtype::bfloat16 alpha,
+                                        const phi::dtype::bfloat16 *A,
+                                        int lda,
+                                        const phi::dtype::bfloat16 *B,
+                                        int ldb,
+                                        phi::dtype::bfloat16 beta,
+                                        phi::dtype::bfloat16 *C,
+                                        int ldc) const {
+#if CUDA_VERSION >= 11000
+  // Note that cublas follows fortran order, so the order is different from
+  // the cblas convention.
+  cublasOperation_t cuTransA = transA ? CUBLAS_OP_T : CUBLAS_OP_N;
+  cublasOperation_t cuTransB = transB ? CUBLAS_OP_T : CUBLAS_OP_N;
+
+  PADDLE_ENFORCE_GE(
+      context_.GetComputeCapability(),
+      80,
+      phi::errors::InvalidArgument(
+          "cublas bf16 gemm requires GPU compute capability >= 80,"
+          "but received %d",
+          context_.GetComputeCapability()));
+
+  float h_alpha = static_cast<float>(alpha);
+  float h_beta = static_cast<float>(beta);
+
+  cublasGemmAlgo_t algo = CUBLAS_GEMM_DEFAULT;
+  bool use_tensor_op_math = context_.tensor_core_available();
+  if (use_tensor_op_math) {
+    algo = CUBLAS_GEMM_DFALT_TENSOR_OP;
+  }
+  VLOG(5) << "use_tensor_op_math: " << (use_tensor_op_math ? "True" : "False");
+
+  context_.TensorCoreCublasCallIfAvailable([&](cublasHandle_t handle) {
+    PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cublasGemmEx(handle,
+                                                          cuTransB,
+                                                          cuTransA,
+                                                          N,
+                                                          M,
+                                                          K,
+                                                          &h_alpha,
+                                                          B,
+                                                          CUDA_R_16BF,
+                                                          ldb,
+                                                          A,
+                                                          CUDA_R_16BF,
+                                                          lda,
+                                                          &h_beta,
+                                                          C,
+                                                          CUDA_R_16BF,
+                                                          ldc,
+                                                          CUDA_R_32F,
+                                                          algo));
+  });
+#else
+  // raise error
+  PADDLE_THROW(phi::errors::Unimplemented(
+      "cublasGemmEx with bfloat16 is not supported on cuda <= 11"));
+
+#endif  // CUDA_VERSION >= 11000
+}
+
 template <>
 template <typename T>
 void Blas<phi::GPUContext>::AXPY(int n, T alpha, const T *x, T *y) const {
diff --git a/paddle/phi/kernels/funcs/blas/blas_impl.hip.h b/paddle/phi/kernels/funcs/blas/blas_impl.hip.h
index 5edfe3a602c7b..bb02242e2db72 100644
--- a/paddle/phi/kernels/funcs/blas/blas_impl.hip.h
+++ b/paddle/phi/kernels/funcs/blas/blas_impl.hip.h
@@ -751,7 +751,7 @@ inline void Blas<phi::GPUContext>::GEMM(CBLAS_TRANSPOSE transA,
       context_.GetComputeCapability(),
       80,
       phi::errors::InvalidArgument(
-          "rocblas fp16 gemm requires GPU compute capability >= 80,"
+          "rocblas bf16 gemm requires GPU compute capability >= 80,"
           "but received %d",
           context_.GetComputeCapability()));
 
@@ -982,6 +982,70 @@ inline void Blas<phi::GPUContext>::GEMM(bool transA,
   });
 }
 
+template <>
+template <>
+inline void Blas<phi::GPUContext>::GEMM(bool transA,
+                                        bool transB,
+                                        int M,
+                                        int N,
+                                        int K,
+                                        phi::dtype::bfloat16 alpha,
+                                        const phi::dtype::bfloat16 *A,
+                                        int lda,
+                                        const phi::dtype::bfloat16 *B,
+                                        int ldb,
+                                        phi::dtype::bfloat16 beta,
+                                        phi::dtype::bfloat16 *C,
+                                        int ldc) const {
+  // Note that cublas follows fortran order, so the order is different from
+  // the cblas convention.
+  rocblas_operation cuTransA = (transA == CblasNoTrans)
+                                   ? rocblas_operation_none
+                                   : rocblas_operation_transpose;
+  rocblas_operation cuTransB = (transB == CblasNoTrans)
+                                   ? rocblas_operation_none
+                                   : rocblas_operation_transpose;
+  PADDLE_ENFORCE_GE(
+      context_.GetComputeCapability(),
+      80,
+      phi::errors::InvalidArgument(
+          "rocblas bf16 gemm requires GPU compute capability >= 80,"
+          "but received %d",
+          context_.GetComputeCapability()));
+
+  float h_alpha = static_cast<float>(alpha);
+  float h_beta = static_cast<float>(beta);
+  rocblas_gemm_algo algo = rocblas_gemm_algo_standard;
+
+  context_.TensorCoreCublasCallIfAvailable([&](rocblas_handle handle) {
+    PADDLE_ENFORCE_GPU_SUCCESS(
+        phi::dynload::rocblas_gemm_ex(handle,
+                                      cuTransB,
+                                      cuTransA,
+                                      N,
+                                      M,
+                                      K,
+                                      &h_alpha,
+                                      B,
+                                      rocblas_datatype_bf16_r,
+                                      ldb,
+                                      A,
+                                      rocblas_datatype_bf16_r,
+                                      lda,
+                                      &h_beta,
+                                      C,
+                                      rocblas_datatype_bf16_r,
+                                      ldc,
+                                      C,
+                                      rocblas_datatype_bf16_r,
+                                      ldc,
+                                      rocblas_datatype_f32_r,
+                                      algo,
+                                      0,
+                                      0));
+  });
+}
+
 template <>
 template <typename T>
 void Blas<phi::GPUContext>::AXPY(int n, T alpha, const T *x, T *y) const {
diff --git a/paddle/phi/kernels/gpu/addmm_grad_kernel.cu b/paddle/phi/kernels/gpu/addmm_grad_kernel.cu
index 65978da1374e4..9d915af9170f6 100644
--- a/paddle/phi/kernels/gpu/addmm_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/addmm_grad_kernel.cu
@@ -18,5 +18,11 @@ limitations under the License. */
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/impl/addmm_grad_kernel_impl.h"
 
-PD_REGISTER_KERNEL(
-    addmm_grad, GPU, ALL_LAYOUT, phi::AddmmGradKernel, float, double) {}
+PD_REGISTER_KERNEL(addmm_grad,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::AddmmGradKernel,
+                   float,
+                   double,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/paddle/phi/kernels/gpu/addmm_kernel.cu b/paddle/phi/kernels/gpu/addmm_kernel.cu
index 7b589ce20acca..563b137040ac7 100644
--- a/paddle/phi/kernels/gpu/addmm_kernel.cu
+++ b/paddle/phi/kernels/gpu/addmm_kernel.cu
@@ -18,4 +18,11 @@ limitations under the License. */
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/impl/addmm_kernel_impl.h"
 
-PD_REGISTER_KERNEL(addmm, GPU, ALL_LAYOUT, phi::AddmmKernel, float, double) {}
+PD_REGISTER_KERNEL(addmm,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::AddmmKernel,
+                   float,
+                   double,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/paddle/phi/kernels/impl/addmm_grad_kernel_impl.h b/paddle/phi/kernels/impl/addmm_grad_kernel_impl.h
index a72fb6062ceff..7b05b08eef5fb 100644
--- a/paddle/phi/kernels/impl/addmm_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/addmm_grad_kernel_impl.h
@@ -18,13 +18,34 @@ limitations under the License. */
 
 #include "glog/logging.h"
 
+#include "paddle/phi/common/amp_type_traits.h"
 #include "paddle/phi/kernels/addmm_grad_kernel.h"
 #include "paddle/phi/kernels/funcs/blas/blas.h"
 #include "paddle/phi/kernels/funcs/eigen/common.h"
 #include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
 
 namespace phi {
 
+template <typename T>
+struct CopyOrScaleFunctor {
+  CopyOrScaleFunctor(const float scale, const T* x, T* output, int64_t numel)
+      : scale_(scale), x_(x), output_(output), numel_(numel) {}
+
+  HOSTDEVICE void operator()(int64_t idx) const {
+    using MPType = typename phi::dtype::MPTypeTrait<T>::Type;
+    const MPType mp_scale = static_cast<MPType>(scale_);
+    const MPType mp_x = static_cast<MPType>(x_[idx]);
+    output_[idx] = static_cast<T>(mp_scale * mp_x);
+  }
+
+ private:
+  const float scale_;
+  const T* x_;
+  T* output_;
+  int64_t numel_;
+};
+
 template <typename T,
           size_t D,
           int MajorType = Eigen::RowMajor,
@@ -45,6 +66,13 @@ void AddmmGradKernel(const Context& dev_ctx,
                      DenseTensor* input_grad,
                      DenseTensor* x_grad,
                      DenseTensor* y_grad) {
+  using MPType = typename phi::dtype::MPTypeTrait<T>::Type;
+  bool is_float16_or_bfloat16 = false;
+  if (std::is_same<T, phi::dtype::float16>::value ||
+      std::is_same<T, phi::dtype::bfloat16>::value) {
+    is_float16_or_bfloat16 = true;
+  }
+
   auto in_dims = input.dims();
   if (input.dims().size() == 1) {
     in_dims = {1, input.dims()[0]};
@@ -65,6 +93,7 @@ void AddmmGradKernel(const Context& dev_ctx,
   }
 
   auto blas = funcs::GetBlas<Context, T>(dev_ctx);
+  auto mt_blas = funcs::GetBlas<Context, MPType>(dev_ctx);
   if (input_grad) {
     dev_ctx.template Alloc<T>(input_grad);
     total_elems = in_dims[0] * in_dims[1];
@@ -78,19 +107,60 @@ void AddmmGradKernel(const Context& dev_ctx,
         Array2(input_grad->dims()[0], input_grad->dims()[1]);
 
     if (row_compress && col_compress) {
-      eigen_dinput.device(place) =
-          eigen_dout.sum().eval().reshape(eigen_dinput_shape);
+      if (!is_float16_or_bfloat16) {
+        eigen_dinput.device(place) =
+            eigen_dout.sum().eval().reshape(eigen_dinput_shape);
+      } else {
+        eigen_dinput.device(place) = eigen_dout.template cast<MPType>()
+                                         .sum()
+                                         .eval()
+                                         .reshape(eigen_dinput_shape)
+                                         .template cast<T>();
+      }
     } else if (row_compress) {
-      eigen_dinput.device(place) =
-          eigen_dout.sum(Array1(0)).eval().reshape(eigen_dinput_shape);
+      if (!is_float16_or_bfloat16) {
+        eigen_dinput.device(place) =
+            eigen_dout.sum(Array1(0)).eval().reshape(eigen_dinput_shape);
+      } else {
+        eigen_dinput.device(place) = eigen_dout.template cast<MPType>()
+                                         .sum(Array1(0))
+                                         .eval()
+                                         .reshape(eigen_dinput_shape)
+                                         .template cast<T>();
+      }
     } else if (col_compress) {
-      eigen_dinput.device(place) =
-          eigen_dout.sum(Array1(1)).eval().reshape(eigen_dinput_shape);
+      if (!is_float16_or_bfloat16) {
+        eigen_dinput.device(place) =
+            eigen_dout.sum(Array1(1)).eval().reshape(eigen_dinput_shape);
+      } else {
+        eigen_dinput.device(place) = eigen_dout.template cast<MPType>()
+                                         .sum(Array1(1))
+                                         .eval()
+                                         .reshape(eigen_dinput_shape)
+                                         .template cast<T>();
+      }
     } else {
-      blas.VCOPY(total_elems, out_grad.data<T>(), input_grad->data<T>());
+      // The VCOPY does not support the float16, bfloat16
+      if (!is_float16_or_bfloat16) {
+        mt_blas.VCOPY(
+            total_elems, out_grad.data<MPType>(), input_grad->data<MPType>());
+      } else {
+        phi::funcs::ForRange<Context> for_range(dev_ctx, total_elems);
+        CopyOrScaleFunctor<T> functor(
+            1, out_grad.data<T>(), input_grad->data<T>(), total_elems);
+        for_range(functor);
+      }
     }
 
-    blas.SCAL(total_elems, beta, input_grad->data<T>());
+    // The SCAL does not support the float16, bfloat16
+    if (!is_float16_or_bfloat16) {
+      mt_blas.SCAL(total_elems, beta, input_grad->data<MPType>());
+    } else {
+      phi::funcs::ForRange<Context> for_range(dev_ctx, total_elems);
+      CopyOrScaleFunctor<T> functor(
+          beta, input_grad->data<T>(), input_grad->data<T>(), total_elems);
+      for_range(functor);
+    }
 
     if (input.dims().size() == 1) {
       input_grad->Resize(input.dims());
@@ -101,14 +171,28 @@ void AddmmGradKernel(const Context& dev_ctx,
     total_elems = x.dims()[0] * x.dims()[1];
     // x_grad = out_grad * y'. x_grad: M x K, out_grad : M x N, y : K x N
     blas.MatMul(out_grad, false, y, true, x_grad);
-    blas.SCAL(total_elems, alpha, x_grad->data<T>());
+    if (!is_float16_or_bfloat16) {
+      mt_blas.SCAL(total_elems, alpha, x_grad->data<MPType>());
+    } else {
+      phi::funcs::ForRange<Context> for_range(dev_ctx, total_elems);
+      CopyOrScaleFunctor<T> functor(
+          alpha, x_grad->data<T>(), x_grad->data<T>(), total_elems);
+      for_range(functor);
+    }
   }
   if (y_grad) {
     dev_ctx.template Alloc<T>(y_grad);
     total_elems = x.dims()[1] * y.dims()[1];
     // y_grad = x' * out_grad. y_grad K x N, out_grad : M x N, x : M x K
     blas.MatMul(x, true, out_grad, false, y_grad);
-    blas.SCAL(total_elems, alpha, y_grad->data<T>());
+    if (!is_float16_or_bfloat16) {
+      mt_blas.SCAL(total_elems, alpha, y_grad->data<MPType>());
+    } else {
+      phi::funcs::ForRange<Context> for_range(dev_ctx, total_elems);
+      CopyOrScaleFunctor<T> functor(
+          alpha, y_grad->data<T>(), y_grad->data<T>(), total_elems);
+      for_range(functor);
+    }
   }
 }
 
diff --git a/paddle/phi/kernels/impl/addmm_kernel_impl.h b/paddle/phi/kernels/impl/addmm_kernel_impl.h
index c86cea80e47e8..957e02de6af62 100644
--- a/paddle/phi/kernels/impl/addmm_kernel_impl.h
+++ b/paddle/phi/kernels/impl/addmm_kernel_impl.h
@@ -112,17 +112,19 @@ void AddmmKernel(const Context& dev_ctx,
   funcs::EigenBroadcast<std::decay_t<decltype(place)>, T, 2>::Eval(
       place, eigen_out, eigen_input, bcast_dims);
 
+  T t_alpha = static_cast<T>(alpha);
+  T t_beta = static_cast<T>(beta);
   blas.GEMM(false,
             false,
             x_dims[0],
             y_dims[1],
             x_dims[1],
-            alpha,
+            t_alpha,
             x.data<T>(),
             x_dims[1],
             y.data<T>(),
             y_dims[1],
-            beta,
+            t_beta,
             out->data<T>(),
             y_dims[1]);
 }
diff --git a/python/paddle/fluid/tests/unittests/test_addmm_op.py b/python/paddle/fluid/tests/unittests/test_addmm_op.py
index 3041841cdf84f..66a86961e885d 100644
--- a/python/paddle/fluid/tests/unittests/test_addmm_op.py
+++ b/python/paddle/fluid/tests/unittests/test_addmm_op.py
@@ -15,11 +15,11 @@
 import unittest
 
 import numpy as np
-from eager_op_test import OpTest
+from eager_op_test import OpTest, convert_float_to_uint16
 
 import paddle
 from paddle import fluid
-from paddle.fluid import Program, program_guard
+from paddle.fluid import Program, core, program_guard
 
 
 class TestAddMMOp(OpTest):
@@ -27,7 +27,6 @@ class TestAddMMOp(OpTest):
     def setUp(self):
         self.op_type = "addmm"
         self.python_api = paddle.addmm
-        self.dtype = np.float64
         self.init_dtype_type()
         self.inputs = {
             'Input': np.random.random((100, 1)).astype(self.dtype),
@@ -40,7 +39,7 @@ def setUp(self):
         }
 
     def init_dtype_type(self):
-        pass
+        self.dtype = np.float64
 
     def test_check_output(self):
         self.check_output()
@@ -58,6 +57,62 @@ def test_check_grad_input(self):
         self.check_grad(['Input'], 'Out', no_grad_set=None)
 
 
+class TestAddMMFP16Op(TestAddMMOp):
+    def init_dtype_type(self):
+        self.dtype = np.float16
+
+    def test_check_output(self):
+        self.check_output(atol=1e-2)
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support bfloat16",
+)
+class TestAddMMBF16Op(OpTest):
+    def setUp(self):
+        self.op_type = "addmm"
+        self.python_api = paddle.addmm
+        self.init_dtype_type()
+        self.inputs = {
+            'Input': np.random.random((100, 1)).astype(self.np_dtype),
+            'X': np.random.random((100, 10)).astype(self.np_dtype),
+            'Y': np.random.random((10, 20)).astype(self.np_dtype),
+        }
+        self.outputs = {
+            'Out': self.inputs['Input']
+            + np.dot(self.inputs['X'], self.inputs['Y'])
+        }
+
+        self.inputs['Input'] = convert_float_to_uint16(self.inputs['Input'])
+        self.inputs['X'] = convert_float_to_uint16(self.inputs['X'])
+        self.inputs['Y'] = convert_float_to_uint16(self.inputs['Y'])
+        self.outputs['Out'] = convert_float_to_uint16(self.outputs['Out'])
+        self.place = core.CUDAPlace(0)
+
+    def init_dtype_type(self):
+        self.dtype = np.uint16
+        self.np_dtype = np.float32
+
+    def test_check_output(self):
+        self.check_output_with_place(self.place)
+
+    def test_check_grad_normal(self):
+        self.check_grad_with_place(self.place, ['Input', 'X', 'Y'], 'Out')
+
+    def test_check_grad_x(self):
+        self.check_grad_with_place(self.place, ['X'], 'Out', no_grad_set=None)
+
+    def test_check_grad_y(self):
+        self.check_grad_with_place(self.place, ['Y'], 'Out', no_grad_set=None)
+
+    def test_check_grad_input(self):
+        self.check_grad_with_place(
+            self.place, ['Input'], 'Out', no_grad_set=None
+        )
+
+
 class TestAddMMOpError(unittest.TestCase):
     # test error
     def test_errors(self):
diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py
index 37519996a2acd..4a257822e5b0d 100644
--- a/python/paddle/tensor/math.py
+++ b/python/paddle/tensor/math.py
@@ -1959,10 +1959,14 @@ def addmm(input, x, y, beta=1.0, alpha=1.0, name=None):
 
         helper = LayerHelper("addmm", **locals())
         check_variable_and_dtype(
-            input, 'Input', ['float32', 'float64'], 'addmm'
+            input, 'Input', ['float16', 'float32', 'float64', 'uint16'], 'addmm'
+        )
+        check_variable_and_dtype(
+            x, 'X', ['float16', 'float32', 'float64', 'uint16'], 'addmm'
+        )
+        check_variable_and_dtype(
+            y, 'Y', ['float16', 'float32', 'float64', 'uint16'], 'addmm'
         )
-        check_variable_and_dtype(x, 'X', ['float32', 'float64'], 'addmm')
-        check_variable_and_dtype(y, 'Y', ['float32', 'float64'], 'addmm')
         out = helper.create_variable_for_type_inference(dtype=x.dtype)
 
         helper.append_op(

From decc4c3870dcdc6b64931eea0718a06edc996937 Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Fri, 5 May 2023 10:21:50 +0800
Subject: [PATCH 162/405] [test]mv fluid op math to test/cpp/fluid/math
 (#53446)

---
 paddle/fluid/operators/math/CMakeLists.txt    | 37 -------------------
 test/cpp/fluid/CMakeLists.txt                 |  2 +-
 test/cpp/fluid/math/CMakeLists.txt            | 36 ++++++++++++++++++
 .../cpp/fluid}/math/beam_search_test.cc       |  0
 .../cpp/fluid}/math/concat_test.cc            |  0
 .../cpp/fluid}/math/im2col_test.cc            |  0
 .../fluid}/math/selected_rows_functor_test.cc |  0
 .../math/selected_rows_functor_test.cu.cc     |  0
 .../cpp/fluid}/math/vol2col_test.cc           |  0
 9 files changed, 37 insertions(+), 38 deletions(-)
 create mode 100644 test/cpp/fluid/math/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/math/beam_search_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/math/concat_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/math/im2col_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/math/selected_rows_functor_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/math/selected_rows_functor_test.cu.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/math/vol2col_test.cc (100%)

diff --git a/paddle/fluid/operators/math/CMakeLists.txt b/paddle/fluid/operators/math/CMakeLists.txt
index c67703ba52814..c439ace9714b0 100644
--- a/paddle/fluid/operators/math/CMakeLists.txt
+++ b/paddle/fluid/operators/math/CMakeLists.txt
@@ -24,40 +24,3 @@ math_library(unpooling)
 math_library(prelu)
 math_library(bert_encoder_functor)
 math_library(tree2col DEPS math_function)
-
-cc_test(
-  selected_rows_functor_test
-  SRCS selected_rows_functor_test.cc
-  DEPS allocator selected_rows_functor)
-cc_test(
-  im2col_test
-  SRCS im2col_test.cc
-  DEPS im2col)
-cc_test(
-  vol2col_test
-  SRCS vol2col_test.cc
-  DEPS vol2col)
-cc_test(
-  beam_search_test
-  SRCS beam_search_test.cc
-  DEPS beam_search)
-if(WITH_GPU)
-  nv_test(
-    selected_rows_functor_gpu_test
-    SRCS selected_rows_functor_test.cu.cc
-    DEPS selected_rows_functor math_function)
-endif()
-if(WITH_ROCM)
-  hip_test(
-    selected_rows_functor_gpu_test
-    SRCS selected_rows_functor_test.cu.cc
-    DEPS selected_rows_functor math_function)
-endif()
-cc_test(
-  concat_test
-  SRCS concat_test.cc
-  DEPS concat_and_split)
-
-if(WITH_TESTING AND TEST im2col_test)
-  set_tests_properties(im2col_test PROPERTIES TIMEOUT 120)
-endif()
diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt
index 45134d4b38a03..872e7c71d2520 100644
--- a/test/cpp/fluid/CMakeLists.txt
+++ b/test/cpp/fluid/CMakeLists.txt
@@ -10,7 +10,7 @@ add_subdirectory(elementwise)
 if(WITH_LITE)
   add_subdirectory(lite)
 endif()
-# add_subdirectory(math)
+add_subdirectory(math)
 # if(WITH_MKLDNN)
 #   add_subdirectory(mkldnn)
 # endif()
diff --git a/test/cpp/fluid/math/CMakeLists.txt b/test/cpp/fluid/math/CMakeLists.txt
new file mode 100644
index 0000000000000..cbe53b0828cea
--- /dev/null
+++ b/test/cpp/fluid/math/CMakeLists.txt
@@ -0,0 +1,36 @@
+cc_test(
+  selected_rows_functor_test
+  SRCS selected_rows_functor_test.cc
+  DEPS allocator selected_rows_functor)
+cc_test(
+  im2col_test
+  SRCS im2col_test.cc
+  DEPS im2col)
+cc_test(
+  vol2col_test
+  SRCS vol2col_test.cc
+  DEPS vol2col)
+cc_test(
+  beam_search_test
+  SRCS beam_search_test.cc
+  DEPS beam_search)
+if(WITH_GPU)
+  nv_test(
+    selected_rows_functor_gpu_test
+    SRCS selected_rows_functor_test.cu.cc
+    DEPS selected_rows_functor math_function)
+endif()
+if(WITH_ROCM)
+  hip_test(
+    selected_rows_functor_gpu_test
+    SRCS selected_rows_functor_test.cu.cc
+    DEPS selected_rows_functor math_function)
+endif()
+cc_test(
+  concat_test
+  SRCS concat_test.cc
+  DEPS concat_and_split)
+
+if(WITH_TESTING AND TEST im2col_test)
+  set_tests_properties(im2col_test PROPERTIES TIMEOUT 120)
+endif()
diff --git a/paddle/fluid/operators/math/beam_search_test.cc b/test/cpp/fluid/math/beam_search_test.cc
similarity index 100%
rename from paddle/fluid/operators/math/beam_search_test.cc
rename to test/cpp/fluid/math/beam_search_test.cc
diff --git a/paddle/fluid/operators/math/concat_test.cc b/test/cpp/fluid/math/concat_test.cc
similarity index 100%
rename from paddle/fluid/operators/math/concat_test.cc
rename to test/cpp/fluid/math/concat_test.cc
diff --git a/paddle/fluid/operators/math/im2col_test.cc b/test/cpp/fluid/math/im2col_test.cc
similarity index 100%
rename from paddle/fluid/operators/math/im2col_test.cc
rename to test/cpp/fluid/math/im2col_test.cc
diff --git a/paddle/fluid/operators/math/selected_rows_functor_test.cc b/test/cpp/fluid/math/selected_rows_functor_test.cc
similarity index 100%
rename from paddle/fluid/operators/math/selected_rows_functor_test.cc
rename to test/cpp/fluid/math/selected_rows_functor_test.cc
diff --git a/paddle/fluid/operators/math/selected_rows_functor_test.cu.cc b/test/cpp/fluid/math/selected_rows_functor_test.cu.cc
similarity index 100%
rename from paddle/fluid/operators/math/selected_rows_functor_test.cu.cc
rename to test/cpp/fluid/math/selected_rows_functor_test.cu.cc
diff --git a/paddle/fluid/operators/math/vol2col_test.cc b/test/cpp/fluid/math/vol2col_test.cc
similarity index 100%
rename from paddle/fluid/operators/math/vol2col_test.cc
rename to test/cpp/fluid/math/vol2col_test.cc

From 7f9ab2bdbdf515aacbf0dfd4ea945eb50b7a0048 Mon Sep 17 00:00:00 2001
From: zhangkaihuo <zhangkaihuo@baidu.com>
Date: Fri, 5 May 2023 10:22:38 +0800
Subject: [PATCH 163/405] Add scaled_dot_product_attention (#53113)

---
 .../tests/unittests/test_flash_attention.py   |  56 ++++-
 python/paddle/nn/functional/__init__.py       |   2 +
 .../paddle/nn/functional/flash_attention.py   | 224 ++++++++++++++----
 3 files changed, 233 insertions(+), 49 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/test_flash_attention.py b/python/paddle/fluid/tests/unittests/test_flash_attention.py
index b96c1e3cabd4f..8d6af72c3f4e3 100644
--- a/python/paddle/fluid/tests/unittests/test_flash_attention.py
+++ b/python/paddle/fluid/tests/unittests/test_flash_attention.py
@@ -68,6 +68,7 @@ def setUp(self):
         self.dropout = 0.0
         self.causal = False
         self.return_softmax = False
+        self.use_sdp_kernel = False
 
     def test_unpadded(self):
         print(
@@ -189,9 +190,19 @@ def test_all(self):
             value, place=self.place, dtype=self.dtype, stop_gradient=False
         )
 
-        out, _ = flash_attention(
-            q, k, v, self.dropout, self.causal, self.return_softmax
-        )
+        if self.use_sdp_kernel:
+            with paddle.nn.functional.sdp_kernel(
+                enable_math=self.enable_math,
+                enable_flash=self.enable_flash,
+                enable_mem_efficient=self.enable_mem_efficient,
+            ):
+                out, _ = flash_attention(
+                    q, k, v, self.dropout, self.causal, self.return_softmax
+                )
+        else:
+            out, _ = flash_attention(
+                q, k, v, self.dropout, self.causal, self.return_softmax
+            )
         out_ = attention_naive(q_, k_, v_, self.causal)
 
         out.backward()
@@ -220,9 +231,24 @@ def test_all(self):
                 name="v", shape=self.shape, dtype=self.dtype
             )
 
-            outs, softmax = flash_attention(
-                qs, ks, vs, self.dropout, self.causal, self.return_softmax
-            )
+            if self.use_sdp_kernel:
+                with paddle.nn.functional.sdp_kernel(
+                    enable_math=self.enable_math,
+                    enable_flash=self.enable_flash,
+                    enable_mem_efficient=self.enable_mem_efficient,
+                ):
+                    outs, softmax = flash_attention(
+                        qs,
+                        ks,
+                        vs,
+                        self.dropout,
+                        self.causal,
+                        self.return_softmax,
+                    )
+            else:
+                outs, softmax = flash_attention(
+                    qs, ks, vs, self.dropout, self.causal, self.return_softmax
+                )
 
             exe = fluid.Executor(self.place)
             fetches_result = exe.run(
@@ -247,6 +273,7 @@ def setUp(self):
         self.dropout = 0.0
         self.causal = False
         self.return_softmax = False
+        self.use_sdp_kernel = False
 
 
 class TestFlashAttentionAPITest2(TestFlashAttentionAPI):
@@ -257,6 +284,7 @@ def setUp(self):
         self.dropout = 0.0
         self.causal = False
         self.return_softmax = True
+        self.use_sdp_kernel = False
 
 
 class TestFlashAttentionAPITest3(TestFlashAttentionAPI):
@@ -267,6 +295,7 @@ def setUp(self):
         self.dropout = 0.0
         self.causal = True
         self.return_softmax = False
+        self.use_sdp_kernel = False
 
 
 class TestFlashAttentionAPITest4(TestFlashAttentionAPI):
@@ -277,6 +306,21 @@ def setUp(self):
         self.dropout = 0.0
         self.causal = False
         self.return_softmax = False
+        self.use_sdp_kernel = False
+
+
+class TestMathAttentionAPITest(TestFlashAttentionAPI):
+    def setUp(self):
+        self.place = paddle.CUDAPlace(0)
+        self.shape = (8, 1024, 16, 128)
+        self.dtype = paddle.float16
+        self.dropout = 0.0
+        self.causal = False
+        self.return_softmax = False
+        self.use_sdp_kernel = True
+        self.enable_math = True
+        self.enable_flash = False
+        self.enable_mem_efficient = False
 
 
 if __name__ == '__main__':
diff --git a/python/paddle/nn/functional/__init__.py b/python/paddle/nn/functional/__init__.py
index 2eabd18d394b5..448a533b82d5c 100644
--- a/python/paddle/nn/functional/__init__.py
+++ b/python/paddle/nn/functional/__init__.py
@@ -134,6 +134,8 @@
 from .extension import temporal_shift  # noqa: F401
 
 from .sparse_attention import sparse_attention
+from .flash_attention import scaled_dot_product_attention
+from .flash_attention import sdp_kernel
 
 __all__ = [  # noqa
     'celu',
diff --git a/python/paddle/nn/functional/flash_attention.py b/python/paddle/nn/functional/flash_attention.py
index 78c2dd6e7618a..50e79c1ad17f5 100644
--- a/python/paddle/nn/functional/flash_attention.py
+++ b/python/paddle/nn/functional/flash_attention.py
@@ -13,8 +13,113 @@
 # limitations under the License.
 
 import paddle
+import paddle.nn.functional as F
 from paddle import _C_ops, in_dynamic_mode
 from paddle.fluid.layer_helper import LayerHelper
+from paddle.fluid.wrapped_decorator import signature_safe_contextmanager
+
+g_enable_math = None
+g_enable_flash = None
+g_enable_mem_efficient = None
+
+
+@signature_safe_contextmanager
+def sdp_kernel(enable_math=False, enable_flash=True, enable_mem_efficient=True):
+    r"""
+    With the sdp_kernel context manager, different algorithm implementations can
+    be selected for scaled_dot_product_attention.
+    """
+    global g_enable_math, g_enable_flash, g_enable_mem_efficient
+    original_enable_math = g_enable_math
+    original_enable_flash = g_enable_math
+    original_enable_mem_efficient = g_enable_mem_efficient
+
+    g_enable_math = enable_math
+    g_enable_flash = enable_flash
+    g_enable_mem_efficient = enable_mem_efficient
+    try:
+        yield
+    finally:
+        g_enable_math = original_enable_math
+        g_enable_flash = original_enable_flash
+        g_enable_mem_efficient = original_enable_mem_efficient
+
+
+def _math_attention(
+    query,
+    key,
+    value,
+    dropout_rate=0.0,
+    causal=False,
+    return_softmax=False,
+    training=True,
+):
+    r"""
+    This is a basic implementation of scaled dot product attention composed of
+    combinations of fundamental components.
+    """
+    head_dim = query.shape[-1]
+    query = paddle.transpose(query, [0, 2, 1, 3])
+    key = paddle.transpose(key, [0, 2, 1, 3])
+    value = paddle.transpose(value, [0, 2, 1, 3])
+    product = paddle.matmul(
+        x=query * (head_dim**-0.5), y=key, transpose_y=True
+    )
+    weights = (
+        paddle.incubate.softmax_mask_fuse_upper_triangle(product)
+        if causal
+        else F.softmax(product)
+    )
+    if dropout_rate > 0.0:
+        weights = F.dropout(
+            weights, dropout_rate, training=training, mode="upscale_in_train"
+        )
+
+    out = paddle.matmul(weights, value)
+    out = paddle.transpose(out, [0, 2, 1, 3])
+    return out, weights if return_softmax else None
+
+
+def _select_sdp_cuda(head_dim):
+    if head_dim < 128:
+        return "flash_attn"
+    else:
+        return "mem_efficient"
+
+
+def _select_sdp(head_dim):
+    r"""
+    There are currently three different implementation options available for
+    scaled dot product attention, and the chosen approach depends on whether it
+    is determined by the sdp_kernel configuration or specified through input values.
+    """
+    place = paddle.get_device()
+    # not use sdp_kernel
+    if g_enable_flash is None:
+        if "gpu" not in place:
+            return "math"
+        else:
+            return _select_sdp_cuda(head_dim)
+
+    if (
+        g_enable_math is False
+        and g_enable_flash is False
+        and g_enable_mem_efficient is False
+    ):
+        raise AssertionError(
+            "No available backend for scaled_dot_product_attention was found."
+        )
+
+    if g_enable_math is True:
+        if g_enable_flash is False and g_enable_mem_efficient is False:
+            return "math"
+        if "gpu" not in place:
+            return "math"
+    if g_enable_flash is True and g_enable_mem_efficient is True:
+        return _select_sdp_cuda(head_dim)
+    if g_enable_flash is True:
+        return "flash_attn"
+    return "mem_efficient"
 
 
 def flash_attention(
@@ -84,51 +189,81 @@ def flash_attention(
             output = paddle.nn.functional.flash_attention(q, q, q, 0.9, False, False)
             print(output)
     """
-    if in_dynamic_mode():
-        (result_attention, result_softmax,) = _C_ops.flash_attn(
-            query,
-            key,
-            value,
-            fixed_seed_offset,
-            dropout,
-            causal,
-            return_softmax,
-            not training,
-            rng_name,
+    head_dim = query.shape[3]
+    sdp_func_name = _select_sdp(head_dim)
+
+    if sdp_func_name == "flash_attn":
+        if in_dynamic_mode():
+            (result_attention, result_softmax,) = _C_ops.flash_attn(
+                query,
+                key,
+                value,
+                fixed_seed_offset,
+                dropout,
+                causal,
+                return_softmax,
+                not training,
+                rng_name,
+            )
+            return result_attention, result_softmax if return_softmax else None
+
+        helper = LayerHelper('flash_attn', **locals())
+        dtype = helper.input_dtype(input_param_name='q')
+        out = helper.create_variable_for_type_inference(dtype)
+        softmax = helper.create_variable_for_type_inference(dtype)
+        softmax_lse = helper.create_variable_for_type_inference(paddle.float32)
+        seed_offset = helper.create_variable_for_type_inference(paddle.int64)
+        inputs = {
+            'q': query,
+            'k': key,
+            'v': value,
+            'fixed_seed_offset': fixed_seed_offset,
+        }
+        outputs = {
+            'out': out,
+            'softmax': softmax,
+            'softmax_lse': softmax_lse,
+            'seed_offset': seed_offset,
+        }
+        helper.append_op(
+            type='flash_attn',
+            inputs=inputs,
+            outputs=outputs,
+            attrs={
+                'dropout': dropout,
+                'causal': causal,
+                'return_softmax': return_softmax,
+                'is_test': not training,
+                'rng_name': rng_name,
+            },
         )
-        return result_attention, result_softmax if return_softmax else None
+        return out, softmax if return_softmax else None
+    else:
+        if sdp_func_name == "mem_efficient":
+            from paddle.incubate.nn.memory_efficient_attention import (
+                memory_efficient_attention,
+            )
 
-    helper = LayerHelper('flash_attn', **locals())
-    dtype = helper.input_dtype(input_param_name='q')
-    out = helper.create_variable_for_type_inference(dtype)
-    softmax = helper.create_variable_for_type_inference(dtype)
-    softmax_lse = helper.create_variable_for_type_inference(paddle.float32)
-    seed_offset = helper.create_variable_for_type_inference(paddle.int64)
-    inputs = {
-        'q': query,
-        'k': key,
-        'v': value,
-        'fixed_seed_offset': fixed_seed_offset,
-    }
-    outputs = {
-        'out': out,
-        'softmax': softmax,
-        'softmax_lse': softmax_lse,
-        'seed_offset': seed_offset,
-    }
-    helper.append_op(
-        type='flash_attn',
-        inputs=inputs,
-        outputs=outputs,
-        attrs={
-            'dropout': dropout,
-            'causal': causal,
-            'return_softmax': return_softmax,
-            'is_test': not training,
-            'rng_name': rng_name,
-        },
-    )
-    return out, softmax if return_softmax else None
+            output = memory_efficient_attention(
+                query,
+                key,
+                value,
+                attn_bias=None,
+                p=dropout,
+                scale=None,
+                training=training,
+            )
+            return output, None
+        else:
+            return _math_attention(
+                query,
+                key,
+                value,
+                dropout_rate=dropout,
+                causal=causal,
+                return_softmax=return_softmax,
+                training=training,
+            )
 
 
 def flash_attn_unpadded(
@@ -264,3 +399,6 @@ def flash_attn_unpadded(
         },
     )
     return out, softmax if return_softmax else None
+
+
+scaled_dot_product_attention = flash_attention

From 903c5638d37615873a0ad5f2342286c9ed1dc1a0 Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Fri, 5 May 2023 10:22:55 +0800
Subject: [PATCH 164/405] [test]mv fluid op fused to test/cpp/fluid/fused
 (#53434)

---
 paddle/fluid/operators/fused/CMakeLists.txt   | 59 ----------------
 test/cpp/fluid/CMakeLists.txt                 |  2 +-
 test/cpp/fluid/fused/CMakeLists.txt           | 68 +++++++++++++++++++
 .../fluid}/fused/cudnn_bn_add_relu_test.cc    |  0
 .../cpp/fluid}/fused/cudnn_norm_conv_test.cc  |  0
 .../fused/fused_dropout_act_bias_test.cu      |  2 +-
 .../cpp/fluid}/fused/fused_dropout_test.h     |  0
 ...ed_layernorm_residual_dropout_bias_test.cu |  2 +-
 .../fused/fused_residual_dropout_bias_test.cu |  2 +-
 .../cpp/fluid}/fused/fusion_group_op_test.cc  |  0
 10 files changed, 72 insertions(+), 63 deletions(-)
 create mode 100644 test/cpp/fluid/fused/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/fused/cudnn_bn_add_relu_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/fused/cudnn_norm_conv_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/fused/fused_dropout_act_bias_test.cu (99%)
 rename {paddle/fluid/operators => test/cpp/fluid}/fused/fused_dropout_test.h (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/fused/fused_layernorm_residual_dropout_bias_test.cu (99%)
 rename {paddle/fluid/operators => test/cpp/fluid}/fused/fused_residual_dropout_bias_test.cu (99%)
 rename {paddle/fluid/operators => test/cpp/fluid}/fused/fusion_group_op_test.cc (100%)

diff --git a/paddle/fluid/operators/fused/CMakeLists.txt b/paddle/fluid/operators/fused/CMakeLists.txt
index becef73569b29..261d1905246c8 100755
--- a/paddle/fluid/operators/fused/CMakeLists.txt
+++ b/paddle/fluid/operators/fused/CMakeLists.txt
@@ -74,10 +74,6 @@ if(WITH_GPU OR WITH_ROCM)
   # fusion_group
   if(NOT APPLE AND NOT WIN32)
     op_library(fusion_group_op DEPS device_code)
-    cc_test(
-      test_fusion_group_op
-      SRCS fusion_group_op_test.cc
-      DEPS fusion_group_op)
   endif()
   # fused_bn_add_activation
   # HIP not support bn act fuse in MIOPEN
@@ -87,37 +83,6 @@ if(WITH_GPU OR WITH_ROCM)
   # fused_dropout
   # only support CUDA
   if(NOT WITH_ROCM)
-    nv_test(
-      test_fused_residual_dropout_bias
-      SRCS fused_residual_dropout_bias_test.cu
-      DEPS tensor
-           op_registry
-           dropout_op
-           layer_norm_op
-           device_context
-           generator
-           memory)
-    nv_test(
-      test_fused_dropout_act_bias
-      SRCS fused_dropout_act_bias_test.cu
-      DEPS tensor
-           op_registry
-           dropout_op
-           layer_norm_op
-           device_context
-           generator
-           memory)
-    nv_test(
-      test_fused_layernorm_residual_dropout_bias
-      SRCS fused_layernorm_residual_dropout_bias_test.cu
-      DEPS tensor
-           op_registry
-           dropout_op
-           layer_norm_op
-           device_context
-           generator
-           memory)
-
     op_library(fused_feedforward_op)
     # fused_attention_op
     op_library(fused_attention_op)
@@ -128,30 +93,6 @@ if(WITH_GPU OR WITH_ROCM)
   # resnet_unit needs cudnn 8.0 above
   if((NOT WITH_ROCM) AND (NOT ${CUDNN_VERSION} VERSION_LESS 8000))
     op_library(resnet_unit_op)
-    cc_test(
-      test_cudnn_norm_conv
-      SRCS cudnn_norm_conv_test.cc
-      DEPS conv_op
-           blas
-           im2col
-           vol2col
-           depthwise_conv
-           eigen_function
-           tensor
-           op_registry
-           device_context
-           generator
-           memory)
-    cc_test(
-      test_cudnn_bn_add_relu
-      SRCS cudnn_bn_add_relu_test.cc
-      DEPS batch_norm_op
-           fused_bn_add_activation_op
-           tensor
-           op_registry
-           device_context
-           generator
-           memory)
   endif()
 
   if(CUDA_VERSION GREATER_EQUAL 11.6)
diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt
index 872e7c71d2520..c1bf46dce291e 100644
--- a/test/cpp/fluid/CMakeLists.txt
+++ b/test/cpp/fluid/CMakeLists.txt
@@ -6,7 +6,7 @@ endif()
 # add_subdirectory(detection)
 # add_subdirectory(dlnne)
 add_subdirectory(elementwise)
-# add_subdirectory(fused)
+add_subdirectory(fused)
 if(WITH_LITE)
   add_subdirectory(lite)
 endif()
diff --git a/test/cpp/fluid/fused/CMakeLists.txt b/test/cpp/fluid/fused/CMakeLists.txt
new file mode 100644
index 0000000000000..4bbcfbb93d2f8
--- /dev/null
+++ b/test/cpp/fluid/fused/CMakeLists.txt
@@ -0,0 +1,68 @@
+if(WITH_GPU OR WITH_ROCM)
+  # fusion_group
+  if(NOT APPLE AND NOT WIN32)
+    cc_test(
+      test_fusion_group_op
+      SRCS fusion_group_op_test.cc
+      DEPS fusion_group_op)
+  endif()
+  if(NOT WITH_ROCM)
+    nv_test(
+      test_fused_residual_dropout_bias
+      SRCS fused_residual_dropout_bias_test.cu
+      DEPS tensor
+           op_registry
+           dropout_op
+           layer_norm_op
+           device_context
+           generator
+           memory)
+    nv_test(
+      test_fused_dropout_act_bias
+      SRCS fused_dropout_act_bias_test.cu
+      DEPS tensor
+           op_registry
+           dropout_op
+           layer_norm_op
+           device_context
+           generator
+           memory)
+    nv_test(
+      test_fused_layernorm_residual_dropout_bias
+      SRCS fused_layernorm_residual_dropout_bias_test.cu
+      DEPS tensor
+           op_registry
+           dropout_op
+           layer_norm_op
+           device_context
+           generator
+           memory)
+  endif()
+  # resnet_unit needs cudnn 8.0 above
+  if((NOT WITH_ROCM) AND (NOT ${CUDNN_VERSION} VERSION_LESS 8000))
+    cc_test(
+      test_cudnn_norm_conv
+      SRCS cudnn_norm_conv_test.cc
+      DEPS conv_op
+           blas
+           im2col
+           vol2col
+           depthwise_conv
+           eigen_function
+           tensor
+           op_registry
+           device_context
+           generator
+           memory)
+    cc_test(
+      test_cudnn_bn_add_relu
+      SRCS cudnn_bn_add_relu_test.cc
+      DEPS batch_norm_op
+           fused_bn_add_activation_op
+           tensor
+           op_registry
+           device_context
+           generator
+           memory)
+  endif()
+endif()
diff --git a/paddle/fluid/operators/fused/cudnn_bn_add_relu_test.cc b/test/cpp/fluid/fused/cudnn_bn_add_relu_test.cc
similarity index 100%
rename from paddle/fluid/operators/fused/cudnn_bn_add_relu_test.cc
rename to test/cpp/fluid/fused/cudnn_bn_add_relu_test.cc
diff --git a/paddle/fluid/operators/fused/cudnn_norm_conv_test.cc b/test/cpp/fluid/fused/cudnn_norm_conv_test.cc
similarity index 100%
rename from paddle/fluid/operators/fused/cudnn_norm_conv_test.cc
rename to test/cpp/fluid/fused/cudnn_norm_conv_test.cc
diff --git a/paddle/fluid/operators/fused/fused_dropout_act_bias_test.cu b/test/cpp/fluid/fused/fused_dropout_act_bias_test.cu
similarity index 99%
rename from paddle/fluid/operators/fused/fused_dropout_act_bias_test.cu
rename to test/cpp/fluid/fused/fused_dropout_act_bias_test.cu
index 56f150c2dce42..ddcc3f513d510 100644
--- a/paddle/fluid/operators/fused/fused_dropout_act_bias_test.cu
+++ b/test/cpp/fluid/fused/fused_dropout_act_bias_test.cu
@@ -19,9 +19,9 @@ limitations under the License. */
 
 #include "paddle/fluid/operators/amp/fp16_type_traits.h"
 #include "paddle/fluid/operators/fused/fused_dropout_act_bias.h"
-#include "paddle/fluid/operators/fused/fused_dropout_test.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/funcs/functors.h"
+#include "test/cpp/fluid/fused/fused_dropout_test.h"
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 PD_DECLARE_KERNEL(dropout, GPU, ALL_LAYOUT);
diff --git a/paddle/fluid/operators/fused/fused_dropout_test.h b/test/cpp/fluid/fused/fused_dropout_test.h
similarity index 100%
rename from paddle/fluid/operators/fused/fused_dropout_test.h
rename to test/cpp/fluid/fused/fused_dropout_test.h
diff --git a/paddle/fluid/operators/fused/fused_layernorm_residual_dropout_bias_test.cu b/test/cpp/fluid/fused/fused_layernorm_residual_dropout_bias_test.cu
similarity index 99%
rename from paddle/fluid/operators/fused/fused_layernorm_residual_dropout_bias_test.cu
rename to test/cpp/fluid/fused/fused_layernorm_residual_dropout_bias_test.cu
index e57f39ec01e6d..60cb67c837a12 100644
--- a/paddle/fluid/operators/fused/fused_layernorm_residual_dropout_bias_test.cu
+++ b/test/cpp/fluid/fused/fused_layernorm_residual_dropout_bias_test.cu
@@ -17,9 +17,9 @@ limitations under the License. */
 #include <random>
 #include <vector>
 
-#include "paddle/fluid/operators/fused/fused_dropout_test.h"
 #include "paddle/fluid/operators/fused/fused_layernorm_residual_dropout_bias.h"
 #include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/fluid/fused/fused_dropout_test.h"
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 PD_DECLARE_KERNEL(dropout, GPU, ALL_LAYOUT);
diff --git a/paddle/fluid/operators/fused/fused_residual_dropout_bias_test.cu b/test/cpp/fluid/fused/fused_residual_dropout_bias_test.cu
similarity index 99%
rename from paddle/fluid/operators/fused/fused_residual_dropout_bias_test.cu
rename to test/cpp/fluid/fused/fused_residual_dropout_bias_test.cu
index d74bd9bfe1750..823eb2ce7c607 100644
--- a/paddle/fluid/operators/fused/fused_residual_dropout_bias_test.cu
+++ b/test/cpp/fluid/fused/fused_residual_dropout_bias_test.cu
@@ -17,9 +17,9 @@ limitations under the License. */
 #include <random>
 #include <vector>
 
-#include "paddle/fluid/operators/fused/fused_dropout_test.h"
 #include "paddle/fluid/operators/fused/fused_residual_dropout_bias.h"
 #include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/fluid/fused/fused_dropout_test.h"
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 PD_DECLARE_KERNEL(dropout, GPU, ALL_LAYOUT);
diff --git a/paddle/fluid/operators/fused/fusion_group_op_test.cc b/test/cpp/fluid/fused/fusion_group_op_test.cc
similarity index 100%
rename from paddle/fluid/operators/fused/fusion_group_op_test.cc
rename to test/cpp/fluid/fused/fusion_group_op_test.cc

From 5cc6a512efd3345a4db9b52616d88f2c2bad1dba Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Fri, 5 May 2023 10:24:39 +0800
Subject: [PATCH 165/405] [test]mv fluid op pscore to test (#53460)

* [test]mv fluid op pscore to test/cpp/fluid/pscore

* [test]add -faligned-new

* [test] fix brpc link
---
 paddle/fluid/operators/pscore/CMakeLists.txt  |  85 -----------
 test/cpp/fluid/CMakeLists.txt                 |   6 +-
 test/cpp/fluid/pscore/CMakeLists.txt          | 141 ++++++++++++++++++
 .../pscore/heter_cloud_comm_cpu_test.cc       |   0
 .../pscore/heter_listen_and_server_test.cc    |   0
 .../cpp/fluid}/pscore/heter_server_test.cc    |   0
 .../pscore/send_and_recv_op_cpu_test.cc       |   0
 .../pscore/send_and_recv_op_gpu_test.cc       |   0
 .../cpp/fluid}/pscore/switch_server_test.cc   |   0
 9 files changed, 144 insertions(+), 88 deletions(-)
 create mode 100644 test/cpp/fluid/pscore/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/pscore/heter_cloud_comm_cpu_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/pscore/heter_listen_and_server_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/pscore/heter_server_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/pscore/send_and_recv_op_cpu_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/pscore/send_and_recv_op_gpu_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/pscore/switch_server_test.cc (100%)

diff --git a/paddle/fluid/operators/pscore/CMakeLists.txt b/paddle/fluid/operators/pscore/CMakeLists.txt
index 89b33d9a144c0..d2aa95c2fd379 100755
--- a/paddle/fluid/operators/pscore/CMakeLists.txt
+++ b/paddle/fluid/operators/pscore/CMakeLists.txt
@@ -73,88 +73,3 @@ register_operators(DEPS ${DISTRIBUTE_DEPS})
 set(OPERATOR_DEPS
     ${OPERATOR_DEPS} ${DISTRIBUTE_DEPS}
     PARENT_SCOPE)
-
-set_source_files_properties(
-  heter_server_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
-cc_test_old(
-  heter_server_test
-  SRCS
-  heter_server_test.cc
-  DEPS
-  ${RPC_DEPS}
-  ${DISTRIBUTE_DEPS}
-  executor
-  scope
-  proto_desc
-  generated_op
-  eigen_function)
-
-set_source_files_properties(
-  send_and_recv_op_cpu_test.cc PROPERTIES COMPILE_FLAGS
-                                          ${DISTRIBUTE_COMPILE_FLAGS})
-cc_test_old(
-  send_and_recv_cpu_test
-  SRCS
-  send_and_recv_op_cpu_test.cc
-  DEPS
-  executor
-  scope
-  proto_desc
-  generated_op
-  send_and_recv_op
-  ${RPC_DEPS}
-  ${DISTRIBUTE_DEPS}
-  eigen_function)
-
-set_source_files_properties(
-  send_and_recv_op_gpu_test.cc PROPERTIES COMPILE_FLAGS
-                                          ${DISTRIBUTE_COMPILE_FLAGS})
-cc_test_old(
-  send_and_recv_gpu_test
-  SRCS
-  send_and_recv_op_gpu_test.cc
-  DEPS
-  executor
-  scope
-  proto_desc
-  generated_op
-  send_and_recv_op
-  ${RPC_DEPS}
-  ${DISTRIBUTE_DEPS}
-  eigen_function)
-
-set_source_files_properties(
-  heter_listen_and_server_test.cc PROPERTIES COMPILE_FLAGS
-                                             ${DISTRIBUTE_COMPILE_FLAGS})
-cc_test_old(
-  heter_listen_and_server_test
-  SRCS
-  heter_listen_and_server_test.cc
-  DEPS
-  executor
-  scope
-  proto_desc
-  generated_op
-  heter_listen_and_serv_op
-  ${RPC_DEPS}
-  ${DISTRIBUTE_DEPS}
-  eigen_function)
-
-#set_source_files_properties(heter_cloud_comm_cpu_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
-#cc_test(heter_cloud_comm_cpu_test SRCS heter_cloud_comm_cpu_test.cc DEPS executor scope proto_desc generated_static_op heter_listen_and_serv_op ${RPC_DEPS} ${DISTRIBUTE_DEPS} eigen_function)
-
-set_source_files_properties(
-  switch_server_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
-cc_binary(
-  switch_server_test
-  SRCS
-  switch_server_test.cc
-  DEPS
-  executor
-  scope
-  proto_desc
-  generated_op
-  heter_listen_and_serv_op
-  ${RPC_DEPS}
-  ${DISTRIBUTE_DEPS}
-  eigen_function)
diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt
index c1bf46dce291e..ad9160b975062 100644
--- a/test/cpp/fluid/CMakeLists.txt
+++ b/test/cpp/fluid/CMakeLists.txt
@@ -15,9 +15,9 @@ add_subdirectory(math)
 #   add_subdirectory(mkldnn)
 # endif()
 add_subdirectory(nccl)
-# if(WITH_PSCORE)
-#   add_subdirectory(pscore)
-# endif()
+if(WITH_PSCORE)
+  add_subdirectory(pscore)
+endif()
 add_subdirectory(prim_ops)
 add_subdirectory(reader)
 add_subdirectory(reduce_ops)
diff --git a/test/cpp/fluid/pscore/CMakeLists.txt b/test/cpp/fluid/pscore/CMakeLists.txt
new file mode 100644
index 0000000000000..c195c6c797535
--- /dev/null
+++ b/test/cpp/fluid/pscore/CMakeLists.txt
@@ -0,0 +1,141 @@
+if(WITH_PSLIB)
+  return()
+endif()
+
+set(DISTRIBUTE_DEPS "")
+
+if(WITH_ARM_BRPC)
+  list(
+    APPEND
+    DISTRIBUTE_DEPS
+    executor
+    fleet
+    ps_service
+    brpc_utils
+    heter_server
+    heter_client
+    ps_framework_proto
+    framework_proto
+    sendrecv_rpc
+    arm_brpc
+    gflags
+    glog
+    snappy
+    device_context)
+else()
+  list(
+    APPEND
+    DISTRIBUTE_DEPS
+    executor
+    fleet
+    ps_service
+    brpc_utils
+    heter_server
+    heter_client
+    ps_framework_proto
+    framework_proto
+    sendrecv_rpc
+    brpc
+    leveldb
+    ssl
+    crypto
+    protobuf
+    gflags
+    glog
+    zlib
+    snappy
+    device_context)
+endif()
+
+set(DISTRIBUTE_COMPILE_FLAGS
+    "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=parentheses"
+)
+
+if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
+  set(DISTRIBUTE_COMPILE_FLAGS "${DISTRIBUTE_COMPILE_FLAGS} -faligned-new")
+endif()
+
+set_source_files_properties(
+  heter_server_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
+cc_test_old(
+  heter_server_test
+  SRCS
+  heter_server_test.cc
+  DEPS
+  ${RPC_DEPS}
+  ${DISTRIBUTE_DEPS}
+  executor
+  scope
+  proto_desc
+  generated_op
+  eigen_function)
+
+set_source_files_properties(
+  send_and_recv_op_cpu_test.cc PROPERTIES COMPILE_FLAGS
+                                          ${DISTRIBUTE_COMPILE_FLAGS})
+cc_test_old(
+  send_and_recv_cpu_test
+  SRCS
+  send_and_recv_op_cpu_test.cc
+  DEPS
+  executor
+  scope
+  proto_desc
+  generated_op
+  send_and_recv_op
+  ${RPC_DEPS}
+  ${DISTRIBUTE_DEPS}
+  eigen_function)
+
+set_source_files_properties(
+  send_and_recv_op_gpu_test.cc PROPERTIES COMPILE_FLAGS
+                                          ${DISTRIBUTE_COMPILE_FLAGS})
+cc_test_old(
+  send_and_recv_gpu_test
+  SRCS
+  send_and_recv_op_gpu_test.cc
+  DEPS
+  executor
+  scope
+  proto_desc
+  generated_op
+  send_and_recv_op
+  ${RPC_DEPS}
+  ${DISTRIBUTE_DEPS}
+  eigen_function)
+
+set_source_files_properties(
+  heter_listen_and_server_test.cc PROPERTIES COMPILE_FLAGS
+                                             ${DISTRIBUTE_COMPILE_FLAGS})
+cc_test_old(
+  heter_listen_and_server_test
+  SRCS
+  heter_listen_and_server_test.cc
+  DEPS
+  executor
+  scope
+  proto_desc
+  generated_op
+  heter_listen_and_serv_op
+  ${RPC_DEPS}
+  ${DISTRIBUTE_DEPS}
+  eigen_function)
+
+#set_source_files_properties(heter_cloud_comm_cpu_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
+#cc_test(heter_cloud_comm_cpu_test SRCS heter_cloud_comm_cpu_test.cc DEPS executor scope proto_desc generated_static_op heter_listen_and_serv_op ${RPC_DEPS} ${DISTRIBUTE_DEPS} eigen_function)
+
+set_source_files_properties(
+  switch_server_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
+cc_binary(
+  switch_server_test
+  SRCS
+  switch_server_test.cc
+  DEPS
+  executor
+  scope
+  proto_desc
+  generated_op
+  heter_listen_and_serv_op
+  ${RPC_DEPS}
+  ${DISTRIBUTE_DEPS}
+  eigen_function)
diff --git a/paddle/fluid/operators/pscore/heter_cloud_comm_cpu_test.cc b/test/cpp/fluid/pscore/heter_cloud_comm_cpu_test.cc
similarity index 100%
rename from paddle/fluid/operators/pscore/heter_cloud_comm_cpu_test.cc
rename to test/cpp/fluid/pscore/heter_cloud_comm_cpu_test.cc
diff --git a/paddle/fluid/operators/pscore/heter_listen_and_server_test.cc b/test/cpp/fluid/pscore/heter_listen_and_server_test.cc
similarity index 100%
rename from paddle/fluid/operators/pscore/heter_listen_and_server_test.cc
rename to test/cpp/fluid/pscore/heter_listen_and_server_test.cc
diff --git a/paddle/fluid/operators/pscore/heter_server_test.cc b/test/cpp/fluid/pscore/heter_server_test.cc
similarity index 100%
rename from paddle/fluid/operators/pscore/heter_server_test.cc
rename to test/cpp/fluid/pscore/heter_server_test.cc
diff --git a/paddle/fluid/operators/pscore/send_and_recv_op_cpu_test.cc b/test/cpp/fluid/pscore/send_and_recv_op_cpu_test.cc
similarity index 100%
rename from paddle/fluid/operators/pscore/send_and_recv_op_cpu_test.cc
rename to test/cpp/fluid/pscore/send_and_recv_op_cpu_test.cc
diff --git a/paddle/fluid/operators/pscore/send_and_recv_op_gpu_test.cc b/test/cpp/fluid/pscore/send_and_recv_op_gpu_test.cc
similarity index 100%
rename from paddle/fluid/operators/pscore/send_and_recv_op_gpu_test.cc
rename to test/cpp/fluid/pscore/send_and_recv_op_gpu_test.cc
diff --git a/paddle/fluid/operators/pscore/switch_server_test.cc b/test/cpp/fluid/pscore/switch_server_test.cc
similarity index 100%
rename from paddle/fluid/operators/pscore/switch_server_test.cc
rename to test/cpp/fluid/pscore/switch_server_test.cc

From 0ded3f0457eb564620ca776bac81f79eb0dd0b5c Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Fri, 5 May 2023 10:26:27 +0800
Subject: [PATCH 166/405] [test]mv fluid *test* to test (#53472)

* [test]mv fluid *test* to test/cpp/fluid

* [phi] fix link error
---
 paddle/fluid/operators/CMakeLists.txt         | 27 ------
 test/cpp/fluid/CMakeLists.txt                 | 85 +++++++++++++++++++
 .../cpp/fluid}/assign_op_test.cc              |  0
 .../cpp/fluid}/beam_search_decode_op_test.cc  |  0
 .../fluid}/beam_search_decode_op_xpu_test.cc  |  0
 .../cpp/fluid}/copy_cross_scope_test.cc       |  0
 .../cpp/fluid}/dropout_op_test.cc             |  0
 .../cpp/fluid}/feed_forward_test.cu           |  0
 .../cpp/fluid}/gather_test.cc                 |  0
 .../cpp/fluid}/op_debug_string_test.cc        |  0
 .../cpp/fluid}/save_load_combine_op_test.cc   |  0
 .../cpp/fluid}/save_load_op_test.cc           |  0
 .../cpp/fluid}/scatter_test.cc                |  0
 .../cpp/fluid}/share_buffer_op_test.cc        |  0
 .../test_common_infer_shape_functions.cc      |  0
 .../test_leaky_relu_grad_grad_functor.cc      |  2 +-
 .../test_leaky_relu_grad_grad_functor.cu      |  2 +-
 .../test_leaky_relu_grad_grad_functor.h       |  0
 18 files changed, 87 insertions(+), 29 deletions(-)
 rename {paddle/fluid/operators => test/cpp/fluid}/assign_op_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/beam_search_decode_op_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/beam_search_decode_op_xpu_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/copy_cross_scope_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/dropout_op_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/feed_forward_test.cu (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/gather_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/op_debug_string_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/save_load_combine_op_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/save_load_op_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/scatter_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/share_buffer_op_test.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/test_common_infer_shape_functions.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/test_leaky_relu_grad_grad_functor.cc (93%)
 rename {paddle/fluid/operators => test/cpp/fluid}/test_leaky_relu_grad_grad_functor.cu (93%)
 rename {paddle/fluid/operators => test/cpp/fluid}/test_leaky_relu_grad_grad_functor.h (100%)

diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt
index 984dacfa8bf88..a0d89394ece6a 100644
--- a/paddle/fluid/operators/CMakeLists.txt
+++ b/paddle/fluid/operators/CMakeLists.txt
@@ -151,7 +151,6 @@ if (WITH_GPU OR WITH_ROCM)
   set(COMMON_OP_DEPS ${COMMON_OP_DEPS} depthwise_conv prelu bert_encoder_functor)
 endif()
 if(WITH_XPU)
-  cc_test(beam_search_decode_op_xpu_test SRCS beam_search_decode_op_xpu_test.cc DEPS lod_tensor)
   set(COMMON_OP_DEPS ${COMMON_OP_DEPS} xpulib)
 endif()
 set(COMMON_OP_DEPS ${COMMON_OP_DEPS} layer)
@@ -169,25 +168,6 @@ set(COMMON_OP_DEPS ${COMMON_OP_DEPS} op_version_registry)
 set(OPERATOR_DEPS ${OPERATOR_DEPS} ${COMMON_OP_DEPS})
 set(GLOB_OPERATOR_DEPS ${OPERATOR_DEPS} CACHE INTERNAL "Global Op dependencies")
 
-cc_test(test_common_infer_shape_functions SRCS test_common_infer_shape_functions.cc DEPS common_infer_shape_functions ${COMMON_OP_DEPS} activation_op elementwise_add_op softmax_op softmax)
-cc_test(gather_test SRCS gather_test.cc DEPS tensor)
-cc_test(assign_op_test SRCS assign_op_test.cc DEPS generated_static_op)
-cc_test(scatter_test SRCS scatter_test.cc DEPS tensor math_function)
-cc_test(beam_search_decode_op_test SRCS beam_search_decode_op_test.cc DEPS lod_tensor)
-cc_test(save_load_op_test SRCS save_load_op_test.cc DEPS save_op load_op)
-cc_test(save_load_combine_op_test SRCS save_load_combine_op_test.cc DEPS save_combine_op load_combine_op)
-if (WITH_GPU)
-    nv_test(dropout_op_test SRCS dropout_op_test.cc DEPS dropout_op tensor generator)
-    nv_test(test_leaky_relu_grad_grad_functor SRCS test_leaky_relu_grad_grad_functor.cc test_leaky_relu_grad_grad_functor.cu DEPS tensor device_context eigen3)
-    nv_test(feed_forward_test SRCS feed_forward_test.cu DEPS elementwise_add_op matmul_op tensor generator)
-elseif(WITH_ROCM)
-    hip_test(dropout_op_test SRCS dropout_op_test.cc DEPS dropout_op tensor generator)
-    hip_test(test_leaky_relu_grad_grad_functor SRCS test_leaky_relu_grad_grad_functor.cc test_leaky_relu_grad_grad_functor.cu DEPS tensor device_context eigen3)
-else()
-    cc_test(test_leaky_relu_grad_grad_functor SRCS test_leaky_relu_grad_grad_functor.cc DEPS tensor device_context eigen3)
-endif()
-cc_test(share_buffer_op_cpp_test SRCS share_buffer_op_test.cc DEPS lod_tensor device_context generated_static_op)
-
 cc_library(tensor_formatter SRCS tensor_formatter.cc DEPS ${OP_HEADER_DEPS})
 if (WITH_PYTHON)
   cc_library(py_func_op SRCS py_func_op.cc DEPS op_registry python pybind)
@@ -195,9 +175,6 @@ endif()
 
 set(GLOB_OP_LIB ${OP_LIBRARY} CACHE INTERNAL "Global OP library")
 
-cc_test_old(op_debug_string_test SRCS op_debug_string_test.cc DEPS elementwise_add_op ${COMMON_OP_DEPS})
-
-
 if(WITH_MKLDNN)
 include(mkldnn/inplace_op_tests.cmake)
 include(mkldnn/caching_tests.cmake)
@@ -211,10 +188,6 @@ if(WITH_UNITY_BUILD)
     target_link_libraries(paddle_operators_unity ${OP_HEADER_DEPS} ${COMMON_OP_DEPS})
 endif()
 
-if (WITH_GPU)
-cc_test(copy_cross_scope_test SRCS copy_cross_scope_test.cc DEPS op_registry copy_cross_scope_op scope device_context enforce executor)
-endif()
-
 copy_if_different(${pybind_file} ${pybind_file_final})
 
 if (WITH_CUSTOM_DEVICE)
diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt
index ad9160b975062..029cf3db9053d 100644
--- a/test/cpp/fluid/CMakeLists.txt
+++ b/test/cpp/fluid/CMakeLists.txt
@@ -22,3 +22,88 @@ add_subdirectory(prim_ops)
 add_subdirectory(reader)
 add_subdirectory(reduce_ops)
 # add_subdirectory(tensorrt)
+
+set(COMMON_OP_DEPS ${COMMON_OP_DEPS} executor)
+
+if(WITH_XPU)
+  cc_test(
+    beam_search_decode_op_xpu_test
+    SRCS beam_search_decode_op_xpu_test.cc
+    DEPS lod_tensor)
+  set(COMMON_OP_DEPS ${COMMON_OP_DEPS} xpulib)
+endif()
+
+cc_test(
+  test_common_infer_shape_functions
+  SRCS test_common_infer_shape_functions.cc
+  DEPS common_infer_shape_functions ${COMMON_OP_DEPS} activation_op
+       elementwise_add_op softmax_op softmax)
+cc_test(
+  gather_test
+  SRCS gather_test.cc
+  DEPS tensor)
+cc_test(
+  assign_op_test
+  SRCS assign_op_test.cc
+  DEPS generated_static_op)
+cc_test(
+  scatter_test
+  SRCS scatter_test.cc
+  DEPS tensor math_function)
+cc_test(
+  beam_search_decode_op_test
+  SRCS beam_search_decode_op_test.cc
+  DEPS lod_tensor)
+cc_test(
+  save_load_op_test
+  SRCS save_load_op_test.cc
+  DEPS save_op load_op)
+cc_test(
+  save_load_combine_op_test
+  SRCS save_load_combine_op_test.cc
+  DEPS save_combine_op load_combine_op)
+
+if(WITH_GPU)
+  nv_test(
+    dropout_op_test
+    SRCS dropout_op_test.cc
+    DEPS dropout_op tensor generator)
+  nv_test(
+    test_leaky_relu_grad_grad_functor
+    SRCS test_leaky_relu_grad_grad_functor.cc
+         test_leaky_relu_grad_grad_functor.cu
+    DEPS tensor device_context eigen3)
+  nv_test(
+    feed_forward_test
+    SRCS feed_forward_test.cu
+    DEPS elementwise_add_op matmul_op tensor generator)
+elseif(WITH_ROCM)
+  hip_test(
+    dropout_op_test
+    SRCS dropout_op_test.cc
+    DEPS dropout_op tensor generator)
+  hip_test(
+    test_leaky_relu_grad_grad_functor
+    SRCS test_leaky_relu_grad_grad_functor.cc
+         test_leaky_relu_grad_grad_functor.cu
+    DEPS tensor device_context eigen3)
+else()
+  cc_test(
+    test_leaky_relu_grad_grad_functor
+    SRCS test_leaky_relu_grad_grad_functor.cc
+    DEPS tensor device_context eigen3)
+endif()
+cc_test(
+  share_buffer_op_cpp_test
+  SRCS share_buffer_op_test.cc
+  DEPS lod_tensor device_context generated_static_op)
+
+cc_test_old(op_debug_string_test SRCS op_debug_string_test.cc DEPS
+            elementwise_add_op ${COMMON_OP_DEPS})
+
+if(WITH_GPU)
+  cc_test(
+    copy_cross_scope_test
+    SRCS copy_cross_scope_test.cc
+    DEPS op_registry copy_cross_scope_op scope device_context enforce executor)
+endif()
diff --git a/paddle/fluid/operators/assign_op_test.cc b/test/cpp/fluid/assign_op_test.cc
similarity index 100%
rename from paddle/fluid/operators/assign_op_test.cc
rename to test/cpp/fluid/assign_op_test.cc
diff --git a/paddle/fluid/operators/beam_search_decode_op_test.cc b/test/cpp/fluid/beam_search_decode_op_test.cc
similarity index 100%
rename from paddle/fluid/operators/beam_search_decode_op_test.cc
rename to test/cpp/fluid/beam_search_decode_op_test.cc
diff --git a/paddle/fluid/operators/beam_search_decode_op_xpu_test.cc b/test/cpp/fluid/beam_search_decode_op_xpu_test.cc
similarity index 100%
rename from paddle/fluid/operators/beam_search_decode_op_xpu_test.cc
rename to test/cpp/fluid/beam_search_decode_op_xpu_test.cc
diff --git a/paddle/fluid/operators/copy_cross_scope_test.cc b/test/cpp/fluid/copy_cross_scope_test.cc
similarity index 100%
rename from paddle/fluid/operators/copy_cross_scope_test.cc
rename to test/cpp/fluid/copy_cross_scope_test.cc
diff --git a/paddle/fluid/operators/dropout_op_test.cc b/test/cpp/fluid/dropout_op_test.cc
similarity index 100%
rename from paddle/fluid/operators/dropout_op_test.cc
rename to test/cpp/fluid/dropout_op_test.cc
diff --git a/paddle/fluid/operators/feed_forward_test.cu b/test/cpp/fluid/feed_forward_test.cu
similarity index 100%
rename from paddle/fluid/operators/feed_forward_test.cu
rename to test/cpp/fluid/feed_forward_test.cu
diff --git a/paddle/fluid/operators/gather_test.cc b/test/cpp/fluid/gather_test.cc
similarity index 100%
rename from paddle/fluid/operators/gather_test.cc
rename to test/cpp/fluid/gather_test.cc
diff --git a/paddle/fluid/operators/op_debug_string_test.cc b/test/cpp/fluid/op_debug_string_test.cc
similarity index 100%
rename from paddle/fluid/operators/op_debug_string_test.cc
rename to test/cpp/fluid/op_debug_string_test.cc
diff --git a/paddle/fluid/operators/save_load_combine_op_test.cc b/test/cpp/fluid/save_load_combine_op_test.cc
similarity index 100%
rename from paddle/fluid/operators/save_load_combine_op_test.cc
rename to test/cpp/fluid/save_load_combine_op_test.cc
diff --git a/paddle/fluid/operators/save_load_op_test.cc b/test/cpp/fluid/save_load_op_test.cc
similarity index 100%
rename from paddle/fluid/operators/save_load_op_test.cc
rename to test/cpp/fluid/save_load_op_test.cc
diff --git a/paddle/fluid/operators/scatter_test.cc b/test/cpp/fluid/scatter_test.cc
similarity index 100%
rename from paddle/fluid/operators/scatter_test.cc
rename to test/cpp/fluid/scatter_test.cc
diff --git a/paddle/fluid/operators/share_buffer_op_test.cc b/test/cpp/fluid/share_buffer_op_test.cc
similarity index 100%
rename from paddle/fluid/operators/share_buffer_op_test.cc
rename to test/cpp/fluid/share_buffer_op_test.cc
diff --git a/paddle/fluid/operators/test_common_infer_shape_functions.cc b/test/cpp/fluid/test_common_infer_shape_functions.cc
similarity index 100%
rename from paddle/fluid/operators/test_common_infer_shape_functions.cc
rename to test/cpp/fluid/test_common_infer_shape_functions.cc
diff --git a/paddle/fluid/operators/test_leaky_relu_grad_grad_functor.cc b/test/cpp/fluid/test_leaky_relu_grad_grad_functor.cc
similarity index 93%
rename from paddle/fluid/operators/test_leaky_relu_grad_grad_functor.cc
rename to test/cpp/fluid/test_leaky_relu_grad_grad_functor.cc
index 9a06a9a276203..2dceaf493799c 100644
--- a/paddle/fluid/operators/test_leaky_relu_grad_grad_functor.cc
+++ b/test/cpp/fluid/test_leaky_relu_grad_grad_functor.cc
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/fluid/operators/test_leaky_relu_grad_grad_functor.h"
+#include "test/cpp/fluid/test_leaky_relu_grad_grad_functor.h"
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/test_leaky_relu_grad_grad_functor.cu b/test/cpp/fluid/test_leaky_relu_grad_grad_functor.cu
similarity index 93%
rename from paddle/fluid/operators/test_leaky_relu_grad_grad_functor.cu
rename to test/cpp/fluid/test_leaky_relu_grad_grad_functor.cu
index 6f0f840b8c5d4..9778ab82e7bd5 100644
--- a/paddle/fluid/operators/test_leaky_relu_grad_grad_functor.cu
+++ b/test/cpp/fluid/test_leaky_relu_grad_grad_functor.cu
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/fluid/operators/test_leaky_relu_grad_grad_functor.h"
+#include "test/cpp/fluid/test_leaky_relu_grad_grad_functor.h"
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/test_leaky_relu_grad_grad_functor.h b/test/cpp/fluid/test_leaky_relu_grad_grad_functor.h
similarity index 100%
rename from paddle/fluid/operators/test_leaky_relu_grad_grad_functor.h
rename to test/cpp/fluid/test_leaky_relu_grad_grad_functor.h

From aa8877174be3e5f1d09bb6e51825e229b1908f72 Mon Sep 17 00:00:00 2001
From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com>
Date: Fri, 5 May 2023 10:30:48 +0800
Subject: [PATCH 167/405] =?UTF-8?q?=E3=80=90prim=E3=80=91modify=20assign?=
 =?UTF-8?q?=20api=20setOutput=20in=20by=5Fpass=20(#53417)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* modify concat_grad add sum comp rule

* cast and by_pass modify

* only modify by_pass

* modify by_pass
---
 .../composite_backward_api.h                  |  1 +
 .../api/manual_prim/utils/static_utils.cc     | 10 +--
 .../fluid/tests/unittests/test_softmax_op.py  | 83 ++++++++++---------
 3 files changed, 49 insertions(+), 45 deletions(-)

diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
index 78c4ee61f3d98..8ad54079ca087 100644
--- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
+++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
@@ -116,6 +116,7 @@ void cast_grad(const Tensor& out_grad, DataType dtype, Tensor* x_grad) {
     set_output<T>(res, x_grad);
   }
 }
+
 template <typename T>
 void gather_grad(const Tensor& x,
                  const Tensor& index,
diff --git a/paddle/fluid/prim/api/manual_prim/utils/static_utils.cc b/paddle/fluid/prim/api/manual_prim/utils/static_utils.cc
index f6c4f6b2e8b01..d76a8ad5523bb 100644
--- a/paddle/fluid/prim/api/manual_prim/utils/static_utils.cc
+++ b/paddle/fluid/prim/api/manual_prim/utils/static_utils.cc
@@ -53,20 +53,20 @@ void set_output<DescTensor>(const paddle::Tensor& x_tmp, paddle::Tensor* x) {
 }
 
 template <>
-void by_pass<DescTensor>(const paddle::Tensor& x, paddle::Tensor* out) {
-  Tensor new_out =
-      empty<DescTensor>({}, phi::DataType::FLOAT32, paddle::Place());
+void by_pass<DescTensor>(const paddle::Tensor& x, paddle::Tensor* real_out) {
   framework::BlockDesc* block = StaticCompositeContext::Instance().GetBlock();
   framework::OpDesc* op = block->AppendOp();
   op->SetType("assign");
   op->SetInput("X",
                {std::static_pointer_cast<prim::DescTensor>(x.impl())->Name()});
+  auto out = empty<DescTensor>({}, x.dtype(), paddle::Place());
   op->SetOutput(
-      "Out", {std::static_pointer_cast<prim::DescTensor>(out->impl())->Name()});
+      "Out", {std::static_pointer_cast<prim::DescTensor>(out.impl())->Name()});
   op->CheckAttrs();
   op->InferVarType(block);
   op->InferShape(*block);
-  set_output<DescTensor>(new_out, out);
+
+  set_output<DescTensor>(out, real_out);
 }
 
 }  // namespace prim
diff --git a/python/paddle/fluid/tests/unittests/test_softmax_op.py b/python/paddle/fluid/tests/unittests/test_softmax_op.py
index 61494bc16f964..9dc1f8d54081e 100644
--- a/python/paddle/fluid/tests/unittests/test_softmax_op.py
+++ b/python/paddle/fluid/tests/unittests/test_softmax_op.py
@@ -462,16 +462,17 @@ def executed_api(self):
         self.softmax = F.softmax
 
     def test_static_check(self):
-        with paddle.static.program_guard(paddle.static.Program()):
-            x = paddle.static.data('X', self.x_np.shape, 'float32')
-            out1 = self.softmax(x)
-            m = paddle.nn.Softmax()
-            out2 = m(x)
-            exe = paddle.static.Executor(self.place)
-            res = exe.run(feed={'X': self.x_np}, fetch_list=[out1, out2])
-        out_ref = ref_softmax(self.x_np, axis=-1, dtype=None)
-        for r in res:
-            np.testing.assert_allclose(out_ref, r, rtol=1e-05)
+        with paddle.fluid.framework._static_guard():
+            with paddle.static.program_guard(paddle.static.Program()):
+                x = paddle.static.data('X', self.x_np.shape, 'float32')
+                out1 = self.softmax(x)
+                m = paddle.nn.Softmax()
+                out2 = m(x)
+                exe = paddle.static.Executor(self.place)
+                res = exe.run(feed={'X': self.x_np}, fetch_list=[out1, out2])
+            out_ref = ref_softmax(self.x_np, axis=-1, dtype=None)
+            for r in res:
+                np.testing.assert_allclose(out_ref, r, rtol=1e-05)
 
     def test_dygraph_check(self):
         paddle.disable_static(self.place)
@@ -505,19 +506,20 @@ def test_dygraph_check(self):
         paddle.enable_static()
 
     def test_error(self):
-        with paddle.static.program_guard(paddle.static.Program()):
-            # The input type must be Variable.
-            self.assertRaises(TypeError, self.softmax, 1)
-            # The input dtype must be float16, float32, float64.
-            x_int32 = paddle.static.data(
-                name='x_int32', shape=[2, 3], dtype='int32'
-            )
-            self.assertRaises(TypeError, self.softmax, x_int32)
-            # support the input dtype is float16
-            x_fp16 = paddle.static.data(
-                name='x_fp16', shape=[2, 3], dtype='float16'
-            )
-            self.softmax(x_fp16)
+        with paddle.fluid.framework._static_guard():
+            with paddle.static.program_guard(paddle.static.Program()):
+                # The input type must be Variable.
+                self.assertRaises(TypeError, self.softmax, 1)
+                # The input dtype must be float16, float32, float64.
+                x_int32 = paddle.static.data(
+                    name='x_int32', shape=[2, 3], dtype='int32'
+                )
+                self.assertRaises(TypeError, self.softmax, x_int32)
+                # support the input dtype is float16
+                x_fp16 = paddle.static.data(
+                    name='x_fp16', shape=[2, 3], dtype='float16'
+                )
+                self.softmax(x_fp16)
 
 
 class TestSoftmaxAPI_ZeroDim(unittest.TestCase):
@@ -538,23 +540,24 @@ def test_dygraph(self):
         paddle.enable_static()
 
     def test_static(self):
-        main_prog = fluid.Program()
-        with fluid.program_guard(main_prog, fluid.Program()):
-            x = paddle.rand([])
-            x.stop_gradient = False
-            out = paddle.nn.functional.softmax(x)
-            fluid.backward.append_backward(out)
-
-            # Test compile shape
-            self.assertEqual(x.shape, ())
-            self.assertEqual(out.shape, ())
-
-            exe = fluid.Executor()
-            result = exe.run(main_prog, fetch_list=[x, out])
-
-            # Test runtime shape
-            self.assertEqual(result[0].shape, ())
-            self.assertEqual(result[1].shape, ())
+        with paddle.fluid.framework._static_guard():
+            main_prog = fluid.Program()
+            with fluid.program_guard(main_prog, fluid.Program()):
+                x = paddle.rand([])
+                x.stop_gradient = False
+                out = paddle.nn.functional.softmax(x)
+                fluid.backward.append_backward(out)
+
+                # Test compile shape
+                self.assertEqual(x.shape, ())
+                self.assertEqual(out.shape, ())
+
+                exe = fluid.Executor()
+                result = exe.run(main_prog, fetch_list=[x, out])
+
+                # Test runtime shape
+                self.assertEqual(result[0].shape, ())
+                self.assertEqual(result[1].shape, ())
 
 
 class TestSoftmaxInplaceAPI(TestSoftmaxAPI):

From 8c5c03c2d6d09ce9e5a99d804a5be02e50524b58 Mon Sep 17 00:00:00 2001
From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com>
Date: Fri, 5 May 2023 11:18:32 +0800
Subject: [PATCH 168/405] =?UTF-8?q?=E3=80=90prim=E3=80=91modify=20the=20si?=
 =?UTF-8?q?gnature=20of=20cast=5Fgrad=20for=20keeping=20consistent=20with?=
 =?UTF-8?q?=20yaml=20config.=20(#53498)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* modify concat_grad add sum comp rule

* modify cast
---
 paddle/fluid/operators/cast_op.cc             | 21 +++++++++++--------
 .../composite_backward_api.h                  |  4 ++--
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/paddle/fluid/operators/cast_op.cc b/paddle/fluid/operators/cast_op.cc
index 52ad19c385e97..ec7c820a1440e 100644
--- a/paddle/fluid/operators/cast_op.cc
+++ b/paddle/fluid/operators/cast_op.cc
@@ -72,15 +72,18 @@ class CastCompositeGradOpMaker : public prim::CompositeGradOpMakerBase {
   using prim::CompositeGradOpMakerBase::CompositeGradOpMakerBase;
 
   void Apply() override {
-    paddle::Tensor out_grad = paddle::Tensor(
-        std::make_shared<prim::DescTensor>(this->SingleOutputGrad("Out")));
-    paddle::Tensor x_grad = paddle::Tensor(
-        std::make_shared<prim::DescTensor>(this->SingleInputGrad("X")));
-    auto dx_ptr = this->GetOutputPtr(&x_grad);
-    std::string dx_name = this->GetOutputName(x_grad);
-    auto dtype = phi::TransToPhiDataType((this->Attr<int>("in_dtype")));
-    prim::cast_grad<prim::DescTensor>(out_grad, dtype, dx_ptr);
-    this->RecoverOutputName(x_grad, dx_name);
+    paddle::Tensor x = this->GetSingleForwardInput("X");
+    paddle::Tensor out_grad = this->GetSingleOutputGrad("Out");
+
+    // get outputs
+    paddle::Tensor x_grad_t = this->GetSingleInputGrad("X");
+    paddle::Tensor *x_grad = this->GetOutputPtr(&x_grad_t);
+    std::string x_grad_name = this->GetOutputName(x_grad_t);
+
+    VLOG(6) << "Runing cast_grad composite func";
+    prim::cast_grad<prim::DescTensor>(x, out_grad, x_grad);
+
+    this->RecoverOutputName(x_grad_t, x_grad_name);
   }
 };
 
diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
index 8ad54079ca087..979be562db93a 100644
--- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
+++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
@@ -110,9 +110,9 @@ void softmax_grad(const Tensor& out,
 }
 
 template <typename T>
-void cast_grad(const Tensor& out_grad, DataType dtype, Tensor* x_grad) {
+void cast_grad(const Tensor& x, const Tensor& out_grad, Tensor* x_grad) {
   if (x_grad) {
-    auto res = cast<T>(out_grad, dtype);
+    auto res = cast<T>(out_grad, x.dtype());
     set_output<T>(res, x_grad);
   }
 }

From 8eb3ce5b7e80819e683f52c4c857f3cb94ca051d Mon Sep 17 00:00:00 2001
From: haosicheng <47998305+HarperCy@users.noreply.github.com>
Date: Fri, 5 May 2023 11:25:55 +0800
Subject: [PATCH 169/405] [XPU] add accuracy phi kernel and delete accuracy
 fluid kernel (#53508)

---
 .../operators/metrics/accuracy_op_xpu.cc      | 78 -------------------
 paddle/phi/kernels/xpu/accuracy_kernel.cc     | 66 ++++++++++++++++
 2 files changed, 66 insertions(+), 78 deletions(-)
 delete mode 100644 paddle/fluid/operators/metrics/accuracy_op_xpu.cc
 create mode 100644 paddle/phi/kernels/xpu/accuracy_kernel.cc

diff --git a/paddle/fluid/operators/metrics/accuracy_op_xpu.cc b/paddle/fluid/operators/metrics/accuracy_op_xpu.cc
deleted file mode 100644
index 0ac30b3e87347..0000000000000
--- a/paddle/fluid/operators/metrics/accuracy_op_xpu.cc
+++ /dev/null
@@ -1,78 +0,0 @@
-/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#ifdef PADDLE_WITH_XPU
-
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/framework/tensor.h"
-#include "paddle/fluid/platform/device/device_wrapper.h"
-
-namespace paddle {
-namespace operators {
-
-template <typename T, typename DeviceContext>
-class AccuracyXPUKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
-    auto* inference = ctx.Input<phi::DenseTensor>("Out");
-    auto* indices = ctx.Input<phi::DenseTensor>("Indices");
-    auto* label = ctx.Input<phi::DenseTensor>("Label");
-    auto* accuracy = ctx.Output<phi::DenseTensor>("Accuracy");
-    auto* correct = ctx.Output<phi::DenseTensor>("Correct");
-    auto* total = ctx.Output<phi::DenseTensor>("Total");
-    int* correct_data = correct->mutable_data<int>(ctx.GetPlace());
-    int* total_data = total->mutable_data<int>(ctx.GetPlace());
-    float* accuracy_data = accuracy->mutable_data<float>(ctx.GetPlace());
-    const int64_t* indices_data = indices->data<int64_t>();
-    const int64_t* label_data = label->data<int64_t>();
-    size_t num_samples = inference->dims()[0];
-    size_t class_dim = inference->dims()[1];
-    if (num_samples == 0) {
-      return;
-    }
-    auto& dev_ctx = ctx.template device_context<DeviceContext>();
-    xpu::ctx_guard RAII_GUARD(dev_ctx.x_context());
-    int size = num_samples * class_dim;
-    int* indices_int32_ptr = RAII_GUARD.alloc_l3_or_gm<int>(size);
-    PADDLE_ENFORCE_XDNN_NOT_NULL(indices_int32_ptr);
-    int* label_int32_ptr = RAII_GUARD.alloc_l3_or_gm<int>(size);
-    PADDLE_ENFORCE_XDNN_NOT_NULL(label_int32_ptr);
-
-    int r = xpu::cast<int64_t, int32_t>(
-        dev_ctx.x_context(), indices_data, indices_int32_ptr, size);
-    PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast");
-
-    r = xpu::cast<int64_t, int32_t>(
-        dev_ctx.x_context(), label_data, label_int32_ptr, size);
-    PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast");
-
-    r = xpu::accuracy(dev_ctx.x_context(),
-                      indices_int32_ptr,
-                      label_int32_ptr,
-                      num_samples,
-                      class_dim,
-                      correct_data,
-                      total_data,
-                      accuracy_data);
-    PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast_v2");
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-PD_REGISTER_STRUCT_KERNEL(
-    accuracy, XPU, ALL_LAYOUT, ops::AccuracyXPUKernel, float) {}
-#endif
diff --git a/paddle/phi/kernels/xpu/accuracy_kernel.cc b/paddle/phi/kernels/xpu/accuracy_kernel.cc
new file mode 100644
index 0000000000000..69de35615eda6
--- /dev/null
+++ b/paddle/phi/kernels/xpu/accuracy_kernel.cc
@@ -0,0 +1,66 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/accuracy_kernel.h"
+
+#include "paddle/phi/backends/xpu/enforce_xpu.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+template <typename T, typename Context>
+void AccuracyRawKernel(const Context& dev_ctx,
+                       const DenseTensor& inference,
+                       const DenseTensor& indices,
+                       const DenseTensor& label,
+                       DenseTensor* accuracy,
+                       DenseTensor* correct,
+                       DenseTensor* total) {
+  int* correct_data = dev_ctx.template Alloc<int>(correct);
+  int* total_data = dev_ctx.template Alloc<int>(total);
+  float* accuracy_data = dev_ctx.template Alloc<float>(accuracy);
+
+  const int64_t* indices_data = indices.data<int64_t>();
+  const int64_t* label_data = label.data<int64_t>();
+
+  PADDLE_ENFORCE_EQ(
+      inference.dims().size(),
+      2,
+      phi::errors::InvalidArgument(
+          "Rank(Input) of AccuracyOp must be 2, with shape "
+          "[sample_number, class_dim], But received rank(Input) is %d",
+          inference.dims().size()));
+
+  int64_t num_samples = inference.dims()[0];
+  int64_t class_dim = inference.dims()[1];
+
+  int r = xpu::accuracy<int64_t>(dev_ctx.x_context(),
+                                 indices_data,
+                                 label_data,
+                                 num_samples,
+                                 class_dim,
+                                 correct_data,
+                                 total_data,
+                                 accuracy_data);
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "accuracy");
+}
+}  // namespace phi
+
+// TODO(add supported dtype.)
+PD_REGISTER_KERNEL(accuracy, XPU, ALL_LAYOUT, phi::AccuracyRawKernel, float) {
+  kernel->InputAt(1).SetDataType(phi::DataType::INT64);
+  kernel->InputAt(2).SetDataType(phi::DataType::INT64);
+  kernel->OutputAt(0).SetDataType(phi::DataType::FLOAT32);
+  kernel->OutputAt(1).SetDataType(phi::DataType::INT32);
+  kernel->OutputAt(2).SetDataType(phi::DataType::INT32);
+}

From bd321df59c9e58fdafc89a2666392501d7885235 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Fri, 5 May 2023 14:05:37 +0800
Subject: [PATCH 170/405] Unify the initialize of flashattn arguments.

---
 .../operators/fused/fused_gate_attention.h    | 314 +++++++++---------
 1 file changed, 159 insertions(+), 155 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 630f7b37c99d0..c163c1eac7589 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -57,18 +57,20 @@ void AllocWithDebugInfo(const phi::GPUContext& dev_ctx,
   }
 }
 
-inline std::string TensorDebugString(const phi::DenseTensor* t) {
+inline std::string TensorDebugString(const phi::DenseTensor* t,
+                                     const std::string& info) {
+  std::stringstream ss;
+  ss << info << ": ";
   if (t) {
     if (t->initialized()) {
-      std::stringstream ss;
       ss << "shape=[" << t->dims() << "], ptr=" << t->data();
-      return ss.str();
     } else {
-      return "not initialized";
+      ss << "not initialized";
     }
   } else {
-    return "nullptr";
+    ss << "nullptr";
   }
+  return ss.str();
 }
 
 inline void WaitWithDebugInfo(const phi::GPUContext& dev_ctx) {
@@ -194,6 +196,7 @@ struct GateAttentionConfig {
   }
 
   bool CanUseFlashAttn() const {
+#ifdef PADDLE_WITH_FLASHATTN
     if (!std::is_same<T, phi::dtype::bfloat16>::value) {
       return false;
     }
@@ -211,6 +214,9 @@ struct GateAttentionConfig {
     } else {
       return false;
     }
+#else
+    return false;
+#endif
   }
 
   int64_t GetQuerySize() const {
@@ -924,24 +930,12 @@ class FlashAttnWithGating {
         qkv_transpose_out,
         platform::errors::NotFound("The input qkv_transpose_out can not be "
                                    "nullptr when merge_qkv is true."));
+
     // 1. Dealing with qkv_out for flash_attn.
     phi::DenseTensor* qkv_out = config->GetQKVOut();
     ComputeQKVTransposeForwardForFlashAttn(*qkv_out, qkv_transpose_out);
     config->ClearQKVOut();
 
-    int seq_batch_size = static_cast<int>(config->batch_size) *
-                         static_cast<int>(config->seq_len_m);
-    if (VLOG_IS_ON(5)) {
-      qkv_transpose_out->Resize(
-          {3,
-           seq_batch_size * static_cast<int>(config->seq_len_r),
-           static_cast<int>(config->num_heads),
-           static_cast<int>(config->head_dim)});
-      VLOG(5) << "Reshape qkv_transpose_out: ["
-              << config->qkv_transpose_out_dims << "] -> ["
-              << qkv_transpose_out->dims() << "]";
-    }
-
     // q_size == k_size
     int64_t q_size = config->GetQuerySize();
     T* q_ptr = qkv_transpose_out->data<T>();
@@ -949,7 +943,10 @@ class FlashAttnWithGating {
     T* v_ptr = k_ptr + q_size;
 
     // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
-    phi::DenseTensor cu_seq_q, cu_seq_k;
+    phi::DenseTensor cu_seq_q;
+    phi::DenseTensor cu_seq_k;
+    int seq_batch_size = static_cast<int>(config->batch_size) *
+                         static_cast<int>(config->seq_len_m);
     int64_t step = static_cast<int64_t>(config->seq_len_r);
     AllocAndInitSeqQK(seq_batch_size, step, &cu_seq_q, &cu_seq_k);
 
@@ -977,52 +974,27 @@ class FlashAttnWithGating {
     dims_merge_func(nonbatched_bias, &temp_bias, "temp_bias");
 
     // 4. flash_attn parameter setting.
-    // auto& qkv_dims = qkv_transpose_out->dims();
-    int batch_size_ = seq_batch_size;
-    int total_q_ =
-        seq_batch_size * static_cast<int>(config->seq_len_r);  // qkv_dims[1];
-    int total_k_ = total_q_;                                   // qkv_dims[1];
-    int num_heads_ = static_cast<int>(config->num_heads);      // qkv_dims[2];
-    int head_size_ = static_cast<int>(config->head_dim);       // qkv_dims[3];
-    int max_seqlen_q_ = config->seq_len_r;                     // batch_size_;
-    int max_seqlen_k_ = config->m_size;                        // batch_size_;
-    int num_splits = 0;  // 0 for an internal heuristic, which is optimal
+    InitFlashAttnArguments(config);
 
     // 5. construct softmax_lse
-    int softmax_lse_last_dim = ((max_seqlen_q_ + 16 - 1) / 16) * 16;
-    softmax_lse->Resize({batch_size_, num_heads_, softmax_lse_last_dim});
+    int softmax_lse_last_dim = ((fa_max_seqlen_q_ + 16 - 1) / 16) * 16;
+    softmax_lse->Resize({fa_batch_size_, fa_num_heads_, softmax_lse_last_dim});
     AllocWithDebugInfo<float>(dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
 
-    // 6. construct random seed
-    auto seed_offset_pair = GenerateSeedOffsetPair(batch_size_, num_heads_);
-    uint64_t seed = seed_offset_pair.first;
-    uint64_t offset = seed_offset_pair.second;
-    VLOG(5) << "Construct random seed";
-
     if (VLOG_IS_ON(6)) {
-      VLOG(6) << "batch_size : " << batch_size_;
-      VLOG(6) << "total_q   : " << total_q_;
-      VLOG(6) << "total_k   : " << total_k_;
-      VLOG(6) << "num_heads : " << num_heads_;
-      VLOG(6) << "head_size : " << head_size_;
-      VLOG(6) << "max_seqlen_q : " << max_seqlen_q_;
-      VLOG(6) << "max_seqlen_k : " << max_seqlen_k_;
-      VLOG(5) << "Allocate softmax_lse: shape=[" << softmax_lse->dims() << "]";
-      VLOG(6) << "cu_seq_q: " << TensorDebugString(&cu_seq_q);
-      VLOG(6) << "cu_seq_k: " << TensorDebugString(&cu_seq_k);
-      VLOG(6) << "temp_bias: " << TensorDebugString(&temp_bias);
-      VLOG(6) << "temp_mask: " << TensorDebugString(&temp_mask);
-      VLOG(6) << "nonbatched_bias: " << TensorDebugString(nonbatched_bias);
-      VLOG(6) << "src_mask: " << TensorDebugString(src_mask);
-      VLOG(6) << "qkv_transpose_out: " << TensorDebugString(qkv_transpose_out);
-      VLOG(6) << "softmax_lse: " << TensorDebugString(softmax_lse);
-      VLOG(6) << "fmha_out: " << TensorDebugString(fmha_out);
-      VLOG(6) << "gate_out: " << TensorDebugString(gate_out);
+      VLOG(6) << TensorDebugString(&cu_seq_q, "cu_seq_q");
+      VLOG(6) << TensorDebugString(&cu_seq_k, "cu_seq_k");
+      VLOG(6) << TensorDebugString(&temp_bias, "temp_bias");
+      VLOG(6) << TensorDebugString(&temp_mask, "temp_mask");
+      VLOG(6) << TensorDebugString(nonbatched_bias, "nonbatched_bias");
+      VLOG(6) << TensorDebugString(src_mask, "src_mask");
+      VLOG(6) << TensorDebugString(qkv_transpose_out, "qkv_transpose_out");
+      VLOG(6) << TensorDebugString(softmax_lse, "softmax_lse");
+      VLOG(6) << TensorDebugString(fmha_out, "fmha_out");
+      VLOG(6) << TensorDebugString(gate_out, "gate_out");
     }
 
-    // 7. flas_attn part one, get temp worksapce size.
-    float p_dropout = 0.f;
-    float softmax_scale = static_cast<float>(1.0f / std::sqrt(head_size_));
+    // 6. flas_attn part one, get temp worksapce size.
     cudaStream_t stream = dev_ctx_.stream();
     uint64_t workspace_size;
     bool succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
@@ -1032,26 +1004,26 @@ class FlashAttnWithGating {
         nullptr,  // set out to nullptr to calculate workspace size
         cu_seq_q.data<int32_t>(),
         cu_seq_k.data<int32_t>(),
-        total_q_,
-        total_k_,
-        batch_size_,
-        num_heads_,
-        head_size_,
-        max_seqlen_q_,
-        max_seqlen_k_,
-        p_dropout,
-        softmax_scale,
-        /*zero_tensors=*/false,
-        /*is_causal=*/false,
+        fa_total_q_,
+        fa_total_k_,
+        fa_batch_size_,
+        fa_num_heads_,
+        fa_head_size_,
+        fa_max_seqlen_q_,
+        fa_max_seqlen_k_,
+        fa_dropout_prob_,
+        fa_softmax_scale_,
+        fa_zero_tensors_,
+        fa_is_causal_,
         is_bf16,
-        num_splits,
+        fa_num_splits_,
         softmax_lse->data(),
         nullptr,  // softmax out,
         nullptr,
         &workspace_size,
         stream,
-        seed,
-        offset,
+        fa_seed_,
+        fa_offset_,
         src_mask ? temp_mask.data() : nullptr,
         nonbatched_bias ? temp_bias.data() : nullptr,
         src_mask ? temp_mask.dims().Get() : nullptr,
@@ -1069,26 +1041,26 @@ class FlashAttnWithGating {
         static_cast<void*>(fmha_out->data()),
         cu_seq_q.data<int32_t>(),
         cu_seq_k.data<int32_t>(),
-        total_q_,
-        total_k_,
-        batch_size_,
-        num_heads_,
-        head_size_,
-        max_seqlen_q_,
-        max_seqlen_k_,
-        p_dropout,
-        softmax_scale,
-        /*zero_tensors=*/false,
-        /*is_causal=*/false,
+        fa_total_q_,
+        fa_total_k_,
+        fa_batch_size_,
+        fa_num_heads_,
+        fa_head_size_,
+        fa_max_seqlen_q_,
+        fa_max_seqlen_k_,
+        fa_dropout_probs_,
+        fa_softmax_scale_,
+        fa_zero_tensors_,
+        fa_is_causal_,
         is_bf16,
-        num_splits,
+        fa_num_splits_,
         softmax_lse->data(),
         nullptr,  // softmax out
         (workspace_size > 0) ? static_cast<void*>(workspace.data()) : nullptr,
         &workspace_size,
         stream,
-        seed,
-        offset,
+        fa_seed_,
+        fa_offset_,
         src_mask ? temp_mask.data() : nullptr,
         nonbatched_bias ? temp_bias.data() : nullptr,
         src_mask ? temp_mask.dims().Get() : nullptr,
@@ -1128,19 +1100,19 @@ class FlashAttnWithGating {
     const T* v_ptr = k_ptr + q_size;
 
     phi::DenseTensor qkv_transpose_out_grad;
-    qkv_transpose_out_grad.Resize(config->qkv_transpose_out_dims);
+    qkv_transpose_out_grad.Resize(qkv_transpose_out->dims());
     AllocWithDebugInfo<T>(
         dev_ctx_, "qkv_transpose_out_grad", &qkv_transpose_out_grad);
 
     T* q_grad_ptr = qkv_transpose_out_grad.data<T>();
     T* k_grad_ptr = q_grad_ptr + q_size;
     T* v_grad_ptr = k_grad_ptr + q_size;
-
-    int seq_batch_size = static_cast<int>(config->batch_size) *
-                         static_cast<int>(config->seq_len_m);
+    VLOG(5) << WaitWithDebugInfo(dev_ctx_);
 
     // 2. Init with cu_seq_q and cu_seq_k for flash_attn.
     phi::DenseTensor cu_seq_q, cu_seq_k;
+    int seq_batch_size = static_cast<int>(config->batch_size) *
+                         static_cast<int>(config->seq_len_m);
     int64_t step = static_cast<int64_t>(config->seq_len_r);
     AllocAndInitSeqQK(seq_batch_size, step, &cu_seq_q, &cu_seq_k);
 
@@ -1161,23 +1133,14 @@ class FlashAttnWithGating {
                             dims_[dims_rank - 3],
                             dims_[dims_rank - 2],
                             dims_[dims_rank - 1]});
-        VLOG(6) << prefix << ": " << TensorDebugString(dst_tensor);
+        VLOG(6) << TensorDebugString(dst_tensor, prefix);
       }
     };
     dims_merge_func(src_mask, &temp_mask, "temp_mask");
     dims_merge_func(nonbatched_bias, &temp_bias, "temp_bias");
 
-    // phi::DDim qkv_dims({3,
-    //                     seq_batch_size * static_cast<int>(config->seq_len_r),
-    //                     static_cast<int>(config->num_heads),
-    //                     static_cast<int>(config->head_dim)});
-    int batch_size_ = seq_batch_size;
-    int total_q_ = seq_batch_size * static_cast<int>(config->seq_len_r);
-    int total_k_ = total_q_;                               // qkv_dims[1];
-    int num_heads_ = static_cast<int>(config->num_heads);  // qkv_dims[2];
-    int head_size_ = static_cast<int>(config->head_dim);   // qkv_dims[3];
-    int max_seqlen_q_ = config->seq_len_r;
-    int max_seqlen_k_ = config->m_size;
+    // 4. flash_attn parameter setting.
+    InitFlashAttnArguments(config);
 
     // 5. construct softmax_lse
     phi::DenseTensor softmax_d;
@@ -1186,37 +1149,23 @@ class FlashAttnWithGating {
 
     phi::DenseTensor bias_d;
     if (nonbatched_bias) {
-      bias_d = phi::Empty<T, phi::GPUContext>(
-          dev_ctx_, {batch_size_, num_heads_, max_seqlen_q_, max_seqlen_k_});
+      bias_d.Resize(
+          {fa_batch_size_, fa_num_heads_, fa_max_seqlen_q_, fa_max_seqlen_k_});
+      AllocWithDebugInfo<T>(dev_ctx_, "d_bias", &bias_d);
     }
 
-    // 6. construct random seed
-    auto seed_offset_pair = GenerateSeedOffsetPair(batch_size_, num_heads_);
-    uint64_t seed = seed_offset_pair.first;
-    uint64_t offset = seed_offset_pair.second;
-
     if (VLOG_IS_ON(6)) {
-      VLOG(6) << "batch_size : " << batch_size_;
-      VLOG(6) << "total_q   : " << total_q_;
-      VLOG(6) << "total_k   : " << total_k_;
-      VLOG(6) << "num_heads : " << num_heads_;
-      VLOG(6) << "head_size : " << head_size_;
-      VLOG(6) << "max_seqlen_q : " << max_seqlen_q_;
-      VLOG(6) << "max_seqlen_k : " << max_seqlen_k_;
-      VLOG(6) << "fmha_out: " << TensorDebugString(fmha_out);
-      VLOG(6) << "fmha_out_grad: " << TensorDebugString(fmha_out_grad);
-      VLOG(6) << "softmax_lse: " << TensorDebugString(softmax_lse);
-      VLOG(6) << "softmax_d: " << TensorDebugString(&softmax_d);
-      VLOG(6) << "nonbatched_bias: " << TensorDebugString(nonbatched_bias);
-      VLOG(6) << "bias_d: " << TensorDebugString(&bias_d);
+      VLOG(6) << TensorDebugString(fmha_out, "fmha_out");
+      VLOG(6) << TensorDebugString(fmha_out_grad, "fmha_out_grad");
+      VLOG(6) << TensorDebugString(softmax_lse, "softmax_lse");
+      VLOG(6) << TensorDebugString(&softmax_d, "softmax_d");
+      VLOG(6) << TensorDebugString(nonbatched_bias, "nonbatched_bias");
+      VLOG(6) << TensorDebugString(&bias_d, "bias_d");
     }
 
-    // 7. flas_attn part one, get temp worksapce size.
+    // 6. flas_attn part one, get temp worksapce size.
     uint64_t workspace_size;
-    float p_dropout = 0.f;
-    float softmax_scale = static_cast<float>(1.0f / std::sqrt(head_size_));
     cudaStream_t stream = dev_ctx_.stream();
-    int num_splits = 0;  // 0 for an internal heuristic, which is optimal
     bool succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
         static_cast<const void*>(q_ptr),
         static_cast<const void*>(k_ptr),
@@ -1228,27 +1177,27 @@ class FlashAttnWithGating {
         static_cast<const void*>(fmha_out_grad->data()),
         cu_seq_q.data<int32_t>(),
         cu_seq_k.data<int32_t>(),
-        total_q_,
-        total_k_,
-        batch_size_,
-        num_heads_,
-        head_size_,
-        max_seqlen_q_,
-        max_seqlen_k_,
-        p_dropout,
-        softmax_scale,
-        /*zero_tensors=*/false,
-        /*is_causal=*/false,
+        fa_total_q_,
+        fa_total_k_,
+        fa_batch_size_,
+        fa_num_heads_,
+        fa_head_size_,
+        fa_max_seqlen_q_,
+        fa_max_seqlen_k_,
+        fa_dropout_prob_,
+        fa_softmax_scale_,
+        fa_zero_tensors_,
+        fa_is_causal_,
         is_bf16,
-        num_splits,
+        fa_num_splits_,
         softmax_lse->data(),
         softmax_d.data(),
-        nonbatched_bias_grad ? bias_d.data() : nullptr,
+        nonbatched_bias ? bias_d.data() : nullptr,
         nullptr,
         &workspace_size,
         stream,
-        seed,
-        offset,
+        fa_seed_,
+        fa_offset_,
         src_mask ? temp_mask.data() : nullptr,
         nonbatched_bias ? temp_bias.data() : nullptr,
         src_mask ? temp_mask.dims().Get() : nullptr,
@@ -1256,6 +1205,7 @@ class FlashAttnWithGating {
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
+
     phi::DenseTensor workspace = CreateWorkspace(workspace_size);
     succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
         static_cast<const void*>(q_ptr),
@@ -1268,27 +1218,27 @@ class FlashAttnWithGating {
         static_cast<const void*>(fmha_out_grad->data()),
         cu_seq_q.data<int32_t>(),
         cu_seq_k.data<int32_t>(),
-        total_q_,
-        total_k_,
-        batch_size_,
-        num_heads_,
-        head_size_,
-        max_seqlen_q_,
-        max_seqlen_k_,
-        p_dropout,
-        softmax_scale,
-        /*zero_tensors=*/false,
-        /*is_causal=*/false,
+        fa_total_q_,
+        fa_total_k_,
+        fa_batch_size_,
+        fa_num_heads_,
+        fa_head_size_,
+        fa_max_seqlen_q_,
+        fa_max_seqlen_k_,
+        fa_dropout_prob_,
+        fa_softmax_scale_,
+        fa_zero_tensors_,
+        fa_is_causal_,
         is_bf16,
-        num_splits,
+        fa_num_splits_,
         softmax_lse->data(),
         softmax_d.data(),
         nonbatched_bias ? bias_d.data() : nullptr,
         (workspace_size > 0) ? static_cast<void*>(workspace.data()) : nullptr,
         &workspace_size,
         stream,
-        seed,
-        offset,
+        fa_seed_,
+        fa_offset_,
         src_mask ? temp_mask.data() : nullptr,
         nonbatched_bias ? temp_bias.data() : nullptr,
         src_mask ? temp_mask.dims().Get() : nullptr,
@@ -1362,6 +1312,43 @@ class FlashAttnWithGating {
     return gen->IncrementOffset(inc);
   }
 
+  void InitFlashAttnArguments(GateAttentionConfig<T>* config) {
+    fa_batch_size_ = static_cast<int>(config->batch_size) *
+                     static_cast<int>(config->seq_len_m);
+    fa_total_q_ =
+        fa_batch_size_ * static_cast<int>(config->seq_len_r);  // qkv_dims[1];
+    fa_total_k_ = fa_total_q_;                                 // qkv_dims[1];
+    fa_num_heads_ = static_cast<int>(config->num_heads);       // qkv_dims[2];
+    fa_head_size_ = static_cast<int>(config->head_dim);        // qkv_dims[3];
+    fa_max_seqlen_q_ = config->seq_len_r;
+    fa_max_seqlen_k_ = config->m_size;
+    fa_num_splits_ = 0;  // 0 for an internal heuristic, which is optimal
+
+    fa_softmax_scale_ = static_cast<float>(1.0f / std::sqrt(fa_head_size_));
+    fa_dropout_prob_ = 0.0f;
+
+    auto seed_offset_pair =
+        GenerateSeedOffsetPair(fa_batch_size_, fa_num_heads_);
+    fa_seed_ = seed_offset_pair.first;
+    fa_offset_ = seed_offset_pair.second;
+
+    fa_zero_tensors_ = false;
+    fa_is_causal_ = false;
+
+    if (VLOG_IS_ON(6)) {
+      VLOG(6) << "fa_batch_size    : " << fa_batch_size_;
+      VLOG(6) << "fa_total_q       : " << fa_total_q_;
+      VLOG(6) << "fa_total_k       : " << fa_total_k_;
+      VLOG(6) << "fa_num_heads     : " << fa_num_heads_;
+      VLOG(6) << "fa_head_size     : " << fa_head_size_;
+      VLOG(6) << "fa_max_seqlen_q  : " << fa_max_seqlen_q_;
+      VLOG(6) << "fa_max_seqlen_k  : " << fa_max_seqlen_k_;
+      VLOG(6) << "fa_num_splits    : " << fa_num_splits_;
+      VLOG(6) << "fa_softmax_scale :" << fa_softmax_scale_;
+      VLOG(6) << "fa_dropout_prob  :" << fa_dropout_prob_;
+    }
+  }
+
   // [batch_size, seq_len_m, seq_len_r, 3, num_heads, head_dim] ->
   //         [3, batch_size, seq_len_m, seq_len_r, num_heads, head_dim]
   void ComputeQKVTransposeForwardForFlashAttn(
@@ -1371,6 +1358,8 @@ class FlashAttnWithGating {
         dev_ctx_, qkv_out, perm, qkv_transpose_out);
   }
 
+  // [3, batch_size, seq_len_m, seq_len_r, num_heads, head_dim] ->
+  //        [batch_size, seq_len_m, seq_len_r, 3, num_heads, head_dim]
   void ComputeQKVTransposeBackward(
       const phi::DenseTensor& qkv_transpose_out_grad,
       phi::DenseTensor* qkv_out_grad) {
@@ -1381,6 +1370,21 @@ class FlashAttnWithGating {
 
   const phi::GPUContext& dev_ctx_;
   bool merge_qkv_;
+
+  int fa_batch_size_;
+  int fa_total_q_;
+  int fa_total_k_;
+  int fa_num_heads_;
+  int fa_head_size_;
+  int fa_max_seqlen_q_;
+  int fa_max_seqlen_k_;
+  int fa_num_splits_;
+  float fa_softmax_scale_{1.0f};
+  float fa_dropout_prob_(0.0f);
+  uint64_t fa_seed_{0};
+  uint64_t fa_offset_{0};
+  bool fa_zero_tensors_{false};
+  bool fa_is_causal_{false};
 };
 
 }  // namespace operators

From e85fbac85fe3be0070a29ffa30c4c841de0c7a21 Mon Sep 17 00:00:00 2001
From: tianshuo78520a <707759223@qq.com>
Date: Fri, 5 May 2023 14:17:06 +0800
Subject: [PATCH 171/405] Mv cpp_extension test dir (#53330)

---
 python/paddle/fluid/tests/CMakeLists.txt                         | 1 -
 test/CMakeLists.txt                                              | 1 +
 {python/paddle/fluid/tests => test}/cpp_extension/CMakeLists.txt | 0
 .../fluid/tests => test}/cpp_extension/cpp_extension_setup.py    | 0
 .../fluid/tests => test}/cpp_extension/custom_extension.cc       | 0
 {python/paddle/fluid/tests => test}/cpp_extension/custom_power.h | 0
 .../fluid/tests => test}/cpp_extension/custom_relu_forward.cu    | 0
 {python/paddle/fluid/tests => test}/cpp_extension/custom_sub.cc  | 0
 .../fluid/tests => test}/cpp_extension/mix_relu_and_extension.cc | 0
 .../tests => test}/cpp_extension/mix_relu_and_extension_setup.py | 0
 .../fluid/tests => test}/cpp_extension/test_cpp_extension_jit.py | 0
 .../tests => test}/cpp_extension/test_cpp_extension_setup.py     | 0
 .../tests => test}/cpp_extension/test_mixed_extension_setup.py   | 0
 {python/paddle/fluid/tests => test}/cpp_extension/utils.py       | 0
 14 files changed, 1 insertion(+), 1 deletion(-)
 rename {python/paddle/fluid/tests => test}/cpp_extension/CMakeLists.txt (100%)
 rename {python/paddle/fluid/tests => test}/cpp_extension/cpp_extension_setup.py (100%)
 rename {python/paddle/fluid/tests => test}/cpp_extension/custom_extension.cc (100%)
 rename {python/paddle/fluid/tests => test}/cpp_extension/custom_power.h (100%)
 rename {python/paddle/fluid/tests => test}/cpp_extension/custom_relu_forward.cu (100%)
 rename {python/paddle/fluid/tests => test}/cpp_extension/custom_sub.cc (100%)
 rename {python/paddle/fluid/tests => test}/cpp_extension/mix_relu_and_extension.cc (100%)
 rename {python/paddle/fluid/tests => test}/cpp_extension/mix_relu_and_extension_setup.py (100%)
 rename {python/paddle/fluid/tests => test}/cpp_extension/test_cpp_extension_jit.py (100%)
 rename {python/paddle/fluid/tests => test}/cpp_extension/test_cpp_extension_setup.py (100%)
 rename {python/paddle/fluid/tests => test}/cpp_extension/test_mixed_extension_setup.py (100%)
 rename {python/paddle/fluid/tests => test}/cpp_extension/utils.py (100%)

diff --git a/python/paddle/fluid/tests/CMakeLists.txt b/python/paddle/fluid/tests/CMakeLists.txt
index 8e3a67020b82e..711f501e42022 100644
--- a/python/paddle/fluid/tests/CMakeLists.txt
+++ b/python/paddle/fluid/tests/CMakeLists.txt
@@ -9,4 +9,3 @@ foreach(src ${TEST_OPS})
 endforeach()
 
 add_subdirectory(unittests)
-add_subdirectory(cpp_extension)
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 96bd6ca3c263d..567d9b1e6c32f 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -102,6 +102,7 @@ if(WITH_TESTING)
   add_subdirectory(custom_op)
   add_subdirectory(custom_runtime)
   add_subdirectory(dataset)
+  add_subdirectory(cpp_extension)
   if(WITH_DISTRIBUTE)
     add_subdirectory(collective)
     add_subdirectory(auto_parallel)
diff --git a/python/paddle/fluid/tests/cpp_extension/CMakeLists.txt b/test/cpp_extension/CMakeLists.txt
similarity index 100%
rename from python/paddle/fluid/tests/cpp_extension/CMakeLists.txt
rename to test/cpp_extension/CMakeLists.txt
diff --git a/python/paddle/fluid/tests/cpp_extension/cpp_extension_setup.py b/test/cpp_extension/cpp_extension_setup.py
similarity index 100%
rename from python/paddle/fluid/tests/cpp_extension/cpp_extension_setup.py
rename to test/cpp_extension/cpp_extension_setup.py
diff --git a/python/paddle/fluid/tests/cpp_extension/custom_extension.cc b/test/cpp_extension/custom_extension.cc
similarity index 100%
rename from python/paddle/fluid/tests/cpp_extension/custom_extension.cc
rename to test/cpp_extension/custom_extension.cc
diff --git a/python/paddle/fluid/tests/cpp_extension/custom_power.h b/test/cpp_extension/custom_power.h
similarity index 100%
rename from python/paddle/fluid/tests/cpp_extension/custom_power.h
rename to test/cpp_extension/custom_power.h
diff --git a/python/paddle/fluid/tests/cpp_extension/custom_relu_forward.cu b/test/cpp_extension/custom_relu_forward.cu
similarity index 100%
rename from python/paddle/fluid/tests/cpp_extension/custom_relu_forward.cu
rename to test/cpp_extension/custom_relu_forward.cu
diff --git a/python/paddle/fluid/tests/cpp_extension/custom_sub.cc b/test/cpp_extension/custom_sub.cc
similarity index 100%
rename from python/paddle/fluid/tests/cpp_extension/custom_sub.cc
rename to test/cpp_extension/custom_sub.cc
diff --git a/python/paddle/fluid/tests/cpp_extension/mix_relu_and_extension.cc b/test/cpp_extension/mix_relu_and_extension.cc
similarity index 100%
rename from python/paddle/fluid/tests/cpp_extension/mix_relu_and_extension.cc
rename to test/cpp_extension/mix_relu_and_extension.cc
diff --git a/python/paddle/fluid/tests/cpp_extension/mix_relu_and_extension_setup.py b/test/cpp_extension/mix_relu_and_extension_setup.py
similarity index 100%
rename from python/paddle/fluid/tests/cpp_extension/mix_relu_and_extension_setup.py
rename to test/cpp_extension/mix_relu_and_extension_setup.py
diff --git a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py b/test/cpp_extension/test_cpp_extension_jit.py
similarity index 100%
rename from python/paddle/fluid/tests/cpp_extension/test_cpp_extension_jit.py
rename to test/cpp_extension/test_cpp_extension_jit.py
diff --git a/python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py b/test/cpp_extension/test_cpp_extension_setup.py
similarity index 100%
rename from python/paddle/fluid/tests/cpp_extension/test_cpp_extension_setup.py
rename to test/cpp_extension/test_cpp_extension_setup.py
diff --git a/python/paddle/fluid/tests/cpp_extension/test_mixed_extension_setup.py b/test/cpp_extension/test_mixed_extension_setup.py
similarity index 100%
rename from python/paddle/fluid/tests/cpp_extension/test_mixed_extension_setup.py
rename to test/cpp_extension/test_mixed_extension_setup.py
diff --git a/python/paddle/fluid/tests/cpp_extension/utils.py b/test/cpp_extension/utils.py
similarity index 100%
rename from python/paddle/fluid/tests/cpp_extension/utils.py
rename to test/cpp_extension/utils.py

From b02de1b68148aea511e3e13703e6e4d411a36d30 Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Fri, 5 May 2023 15:16:10 +0800
Subject: [PATCH 172/405] =?UTF-8?q?=E3=80=90Hackathon=20No.61=E3=80=91unif?=
 =?UTF-8?q?orm=5Frandom=20=E7=AE=97=E5=AD=90FP16/BF16=E5=8D=95=E6=B5=8B?=
 =?UTF-8?q?=E5=AE=8C=E5=96=84=20(#52949)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../gpu/uniform_inplace_grad_kernel.cu        |  5 +-
 .../phi/kernels/gpu/uniform_inplace_kernel.cu | 13 +++-
 .../test_uniform_random_inplace_op.py         | 76 +++++++++++++++++++
 3 files changed, 90 insertions(+), 4 deletions(-)

diff --git a/paddle/phi/kernels/gpu/uniform_inplace_grad_kernel.cu b/paddle/phi/kernels/gpu/uniform_inplace_grad_kernel.cu
index 61efe8807643d..3b7f8a931278e 100644
--- a/paddle/phi/kernels/gpu/uniform_inplace_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/uniform_inplace_grad_kernel.cu
@@ -14,6 +14,7 @@ limitations under the License. */
 
 #include "paddle/phi/kernels/uniform_inplace_grad_kernel.h"
 
+#include "paddle/phi/common/amp_type_traits.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/full_kernel.h"
 
@@ -41,4 +42,6 @@ PD_REGISTER_KERNEL(uniform_inplace_grad,
                    ALL_LAYOUT,
                    phi::UniformInplaceGradKernel,
                    float,
-                   double) {}
+                   double,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/paddle/phi/kernels/gpu/uniform_inplace_kernel.cu b/paddle/phi/kernels/gpu/uniform_inplace_kernel.cu
index 29bc2f4de5b86..5c3a886ad87e9 100644
--- a/paddle/phi/kernels/gpu/uniform_inplace_kernel.cu
+++ b/paddle/phi/kernels/gpu/uniform_inplace_kernel.cu
@@ -17,6 +17,7 @@ limitations under the License. */
 #include <thrust/random.h>
 
 #include "gflags/gflags.h"
+#include "paddle/phi/common/amp_type_traits.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/funcs/distribution_helper.h"
 #include "paddle/phi/kernels/funcs/index_impl.cu.h"
@@ -72,8 +73,12 @@ void UniformInplaceKernel(const Context& ctx,
     funcs::distribution_and_transform<T>(ctx, out, dist, trans);
   } else {
     // Use OP seed
-    auto func =
-        UniformGenerator<T>(min, max, seed, diag_num, diag_step, diag_val);
+    auto func = UniformGenerator<T>(static_cast<T>(min),
+                                    static_cast<T>(max),
+                                    seed,
+                                    diag_num,
+                                    diag_step,
+                                    static_cast<T>(diag_val));
     IndexKernel<T, UniformGenerator<T>>(ctx, out, func);
   }
 }
@@ -85,4 +90,6 @@ PD_REGISTER_KERNEL(uniform_inplace,
                    ALL_LAYOUT,
                    phi::UniformInplaceKernel,
                    float,
-                   double) {}
+                   double,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/python/paddle/fluid/tests/unittests/test_uniform_random_inplace_op.py b/python/paddle/fluid/tests/unittests/test_uniform_random_inplace_op.py
index a704ae9a4ee21..c4a243f47897b 100644
--- a/python/paddle/fluid/tests/unittests/test_uniform_random_inplace_op.py
+++ b/python/paddle/fluid/tests/unittests/test_uniform_random_inplace_op.py
@@ -15,9 +15,19 @@
 import unittest
 
 import numpy as np
+from eager_op_test import OpTest, convert_uint16_to_float
 
 import paddle
 from paddle import fluid
+from paddle.fluid import core
+
+
+def output_hist(out):
+    hist, _ = np.histogram(out, range=(-1, 1))
+    hist = hist.astype("float32")
+    hist /= float(out.size)
+    prob = 0.1 * np.ones(10)
+    return hist, prob
 
 
 class TestUniformRandomInplaceOpDtype(unittest.TestCase):
@@ -44,6 +54,72 @@ def test_fp64():
             test_fp64()
 
 
+class TestUniformRandomInplaceFP16Op(OpTest):
+    def setUp(self):
+        self.op_type = "uniform_random_inplace"
+        self.dtype = np.float16
+        self.shape = (1000, 784)
+        x = np.random.random(self.shape).astype(self.dtype)
+        y = np.random.random(self.shape).astype(self.dtype)
+        self.inputs = {"X": x}
+        self.outputs = {"Out": y}
+        self.init_attrs()
+
+    def init_attrs(self):
+        self.output_hist = output_hist
+
+    def test_check_output(self):
+        self.check_output_customized(self.verify_output)
+
+    def verify_output(self, outs):
+        hist, prob = self.output_hist(np.array(outs[0]))
+        np.testing.assert_allclose(hist, prob, rtol=0, atol=0.001)
+
+    # TODO: Due to the lack of the self.python_api=paddle.uniform_random_inplace setting, the dynamic graph is temporarily turned off, set check_dygraph=False
+    def test_check_grad(self):
+        self.check_grad(['X'], 'Out', check_dygraph=False)
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support bfloat16",
+)
+class TestUniformRandomInplaceBF16Op(OpTest):
+    def setUp(self):
+        self.op_type = "uniform_random_inplace"
+        self.dtype = np.uint16
+        self.shape = (1000, 784)
+        x = np.random.random(self.shape).astype(self.dtype)
+        y = np.random.random(self.shape).astype(self.dtype)
+        self.inputs = {'X': x}
+        self.outputs = {'Out': y}
+        self.init_attrs()
+        self.place = core.CUDAPlace(0)
+
+    def init_attrs(self):
+        self.output_hist = output_hist
+
+    def test_check_output(self):
+        self.check_output_with_place_customized(self.verify_output, self.place)
+
+    def verify_output(self, outs):
+        result = convert_uint16_to_float(np.array(outs[0]))
+        hist, prob = self.output_hist(result)
+        np.testing.assert_allclose(hist, prob, rtol=0, atol=0.002)
+
+    # TODO: Due to the lack of the self.python_api=paddle.uniform_random_inplace setting, the dynamic graph is temporarily turned off, set check_dygraph=False
+    def test_check_grad(self):
+        grads = [paddle.zeros(self.shape, dtype=self.dtype)]
+        self.check_grad_with_place(
+            self.place,
+            ['X'],
+            'Out',
+            check_dygraph=False,
+            user_defined_grads=grads,
+        )
+
+
 class TestUniformRandomInplaceOpIsInplace(unittest.TestCase):
     def setUp(self):
         self.shape = (1000, 784)

From d27f15ed2a4d15acd1c3ea6525282b571399d030 Mon Sep 17 00:00:00 2001
From: sprouteer <89541335+sprouteer@users.noreply.github.com>
Date: Fri, 5 May 2023 16:52:27 +0800
Subject: [PATCH 173/405] [XPU] Fix the out_max of  the branch in xpu_conv2d
 op(#53343)

---
 .../framework/ir/xpu/link_xpu_op_max_pass.cc      | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/paddle/fluid/framework/ir/xpu/link_xpu_op_max_pass.cc b/paddle/fluid/framework/ir/xpu/link_xpu_op_max_pass.cc
index 5b3f1332907a9..29f8aaea50a67 100644
--- a/paddle/fluid/framework/ir/xpu/link_xpu_op_max_pass.cc
+++ b/paddle/fluid/framework/ir/xpu/link_xpu_op_max_pass.cc
@@ -87,7 +87,7 @@ class LinkXPUOpMaxPass : public FusePassBase {
                  const std::string& op_type,
                  bool with_branch) const;
 
-  const std::string name_scope_{"multi_encoder_xpu_slice_fuse_pass"};
+  const std::string name_scope_{"link_xpu_op_max_pass"};
   // ops with x_max/out_max
   std::set<std::string> op_types_{"fc_xpu", "conv2d_xpu"};
 };
@@ -157,7 +157,15 @@ void LinkXPUOpMaxPass::ApplyImpl(ir::Graph* graph,
     GET_IR_NODE(branch);
 
     auto* fusion_op_desc = fusion_op->Op();
-    if (input->inputs[0]->Op()->HasOutput("out_max")) {
+    if (fusion_op_desc->HasAttr("has_branch")) {
+      bool fusion_op_branch =
+          PADDLE_GET_CONST(bool, fusion_op_desc->GetAttr("has_branch"));
+      if (fusion_op_branch != with_branch) {
+        return;
+      }
+    }
+    if (input->inputs.size() > 0 && input->inputs[0]->IsOp() &&
+        input->inputs[0]->Op()->HasOutput("out_max")) {
       auto input_max_name = input->inputs[0]->Op()->Output("out_max");
       for (auto max_node : input->inputs[0]->outputs) {
         if (input_max_name[0] == max_node->Name()) {
@@ -169,7 +177,8 @@ void LinkXPUOpMaxPass::ApplyImpl(ir::Graph* graph,
     }
 
     if (with_branch) {
-      if (branch->inputs[0]->Op()->HasOutput("out_max")) {
+      if (branch->inputs.size() > 0 && branch->inputs[0]->IsOp() &&
+          branch->inputs[0]->Op()->HasOutput("out_max")) {
         auto branch_max_name = branch->inputs[0]->Op()->Output("out_max");
         for (auto max_node : branch->inputs[0]->outputs) {
           if (branch_max_name[0] == max_node->Name()) {

From 2039115c51af202df0e845f2efbea1319a497d7a Mon Sep 17 00:00:00 2001
From: shentanyue <34421038+shentanyue@users.noreply.github.com>
Date: Fri, 5 May 2023 17:06:47 +0800
Subject: [PATCH 174/405] [XPU] Fusion of gather and assign operators to
 fused_mt op for reducing memory usage (#53262)

---
 paddle/fluid/framework/ir/CMakeLists.txt      |   8 +-
 paddle/fluid/framework/ir/pass.cc             |   2 +-
 ...cc => fused_multi_transformer_xpu_pass.cc} | 142 +++++++++++++++---
 ...used_multi_transformer_xpu_pass_tester.cc} |  71 +++++++--
 .../ir/xpu/one_beam_size_fuse_pass.cc         |  17 ---
 paddle/fluid/framework/ir/xpu/pass_utils.cc   |  17 +++
 paddle/fluid/framework/ir/xpu/pass_utils.h    |   3 +
 .../inference/api/paddle_pass_builder.cc      |   2 +-
 paddle/phi/api/yaml/fused_ops.yaml            |   4 +-
 paddle/phi/infermeta/fusion.cc                |   9 +-
 paddle/phi/infermeta/fusion.h                 |   2 +
 .../xpu/fused_multi_transformer_xpu_kernel.cc |  83 +++++++---
 12 files changed, 277 insertions(+), 83 deletions(-)
 rename paddle/fluid/framework/ir/xpu/{fused_multi_transformer_xpu_quant_pass.cc => fused_multi_transformer_xpu_pass.cc} (81%)
 rename paddle/fluid/framework/ir/xpu/{fused_multi_transformer_xpu_quant_pass_tester.cc => fused_multi_transformer_xpu_pass_tester.cc} (69%)

diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt
index 944dbd07bab5c..46bac61bda13b 100755
--- a/paddle/fluid/framework/ir/CMakeLists.txt
+++ b/paddle/fluid/framework/ir/CMakeLists.txt
@@ -242,7 +242,7 @@ if(WITH_XPU)
   pass_library(one_beam_size_fuse_pass inference DIR xpu DEPS ${XPU_PASS_DEPS})
   pass_library(delete_isolated_node_pass inference DIR xpu DEPS
                ${XPU_PASS_DEPS})
-  pass_library(fused_multi_transformer_xpu_quant_pass inference DIR xpu DEPS
+  pass_library(fused_multi_transformer_xpu_pass inference DIR xpu DEPS
                ${XPU_PASS_DEPS})
   pass_library(stack_fuse_pass inference DIR xpu DEPS ${XPU_PASS_DEPS})
   pass_library(fused_multi_transformer_cachekv_layout_trans_pass inference DIR
@@ -519,9 +519,9 @@ if(WITH_XPU)
     SRCS xpu/delete_isolated_node_pass_test.cc
     DEPS delete_isolated_node_pass)
   cc_test(
-    test_fused_multi_transformer_xpu_quant_pass
-    SRCS xpu/fused_multi_transformer_xpu_quant_pass_tester.cc
-    DEPS fused_multi_transformer_xpu_quant_pass)
+    test_fused_multi_transformer_xpu_pass
+    SRCS xpu/fused_multi_transformer_xpu_pass_tester.cc
+    DEPS fused_multi_transformer_xpu_pass)
   cc_test(
     test_one_beam_size_fuse_pass
     SRCS xpu/one_beam_size_fuse_pass_test.cc
diff --git a/paddle/fluid/framework/ir/pass.cc b/paddle/fluid/framework/ir/pass.cc
index 6611be59fcc80..a84c9b84e9466 100644
--- a/paddle/fluid/framework/ir/pass.cc
+++ b/paddle/fluid/framework/ir/pass.cc
@@ -65,7 +65,7 @@ static const std::vector<std::string> xpu_support_subgraph_passes = {
     "fused_multi_transformer_cachekv_layout_trans_pass",
     "one_beam_size_fuse_pass",
     "stack_fuse_pass",
-    "fused_multi_transformer_xpu_quant_pass",
+    "fused_multi_transformer_xpu_pass",
     "fc_xpu_fuse_pass",
     "link_xpu_op_max_pass",
 };
diff --git a/paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_quant_pass.cc b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_pass.cc
similarity index 81%
rename from paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_quant_pass.cc
rename to paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_pass.cc
index b9868a81135d4..5676465e7131a 100644
--- a/paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_quant_pass.cc
+++ b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_pass.cc
@@ -39,6 +39,32 @@ namespace framework {
 namespace ir {
 namespace patterns {
 
+struct FusedMultiTransformerAssignPattern : public PatternBase {
+  FusedMultiTransformerAssignPattern(PDPattern* pattern,
+                                     const std::string& name_scope);
+  // declare operator node's name
+  PATTERN_DECL_NODE(assign);
+  // declare variable node's name
+  PATTERN_DECL_NODE(assign_out);
+};
+
+FusedMultiTransformerAssignPattern::FusedMultiTransformerAssignPattern(
+    PDPattern* pattern, const std::string& name_scope)
+    : PatternBase(pattern, name_scope, name_scope) {
+  auto* assign =
+      pattern->NewNode(assign_repr())
+          ->assert_is_op("assign")
+          ->assert_more([&](Node* node) {
+            auto pre_op_nodes = node->inputs[0]->inputs;
+            return pre_op_nodes.size() == 1 &&
+                   pre_op_nodes[0]->Op()->Type() == "fused_multi_transformer";
+          });
+  auto* assign_out =
+      pattern->NewNode(assign_out_repr())->assert_is_op_output("assign", "Out");
+
+  assign->LinksTo({assign_out});
+}
+
 struct FusedMultiTransformerPattern : public PatternBase {
   FusedMultiTransformerPattern(PDPattern* pattern,
                                const std::string& name_scope,
@@ -47,7 +73,6 @@ struct FusedMultiTransformerPattern : public PatternBase {
                                bool with_time_step,
                                bool with_seq_lengths,
                                bool with_src_mask);
-
   // declare operator node's name
   PATTERN_DECL_NODE(fused_mt);
   // declare variable node's name
@@ -234,44 +259,106 @@ FusedMultiTransformerPattern::FusedMultiTransformerPattern(
 }  // namespace patterns
 
 /*
-1. transpose and quantify the weights of fused_multi_transformer op from fp32 to
+1. Remove gather and assign op to reduce graphics memory consumption
+2. transpose and quantify the weights of fused_multi_transformer op from fp32 to
 int16
 */
-class FusedMultiTransformerXPUQuantPass : public FusePassBase {
+class FusedMultiTransformerXPUPass : public FusePassBase {
  protected:
   void ApplyImpl(ir::Graph* graph) const override;
 
  private:
-  int ApplyImpl(ir::Graph* graph,
-                bool with_pre_caches,
-                bool with_rotary_pos_emb,
-                bool with_time_step,
-                bool with_seq_lengths,
-                bool with_src_mask) const;
-
-  const std::string name_scope_{"fused_multi_transformer_xpu_quant_pass"};
+  /*
+  Origin subgraph:
+              fused_multi_transformer
+               |        |        |
+             assign   assign    ...
+               |        |        |
+             gather   gather    ...
+
+  Fused subgraph:
+              fused_multi_transformer
+ */
+  void RemoveAssignGather(ir::Graph* graph) const;
+
+  /*
+  Origin subgraph:
+              fused_multi_transformer
+
+  Fused subgraph:
+              fused_multi_transformer_xpu
+ */
+  int FusedMultiTransformerXPUQuant(ir::Graph* graph,
+                                    bool with_pre_caches,
+                                    bool with_rotary_pos_emb,
+                                    bool with_time_step,
+                                    bool with_seq_lengths,
+                                    bool with_src_mask) const;
+
+  const std::string name_scope_{"fused_multi_transformer_xpu_pass"};
 };
 
-void FusedMultiTransformerXPUQuantPass::ApplyImpl(ir::Graph* graph) const {
+void FusedMultiTransformerXPUPass::ApplyImpl(ir::Graph* graph) const {
   PADDLE_ENFORCE_NOT_NULL(
       graph, platform::errors::PreconditionNotMet("graph should not be null."));
   Init(name_scope_, graph);
-  VLOG(3) << "in FusedMultiTransformerXPUQuantPass::ApplyImpl";
+  VLOG(3) << "in FusedMultiTransformerXPUPass::ApplyImpl";
 
   int found_subgraph_count = 0;
+  RemoveAssignGather(graph);
   for (bool with_time_step : {true, false}) {
-    found_subgraph_count +=
-        ApplyImpl(graph, false, false, with_time_step, false, true);
+    found_subgraph_count += FusedMultiTransformerXPUQuant(
+        graph, false, false, with_time_step, false, true);
   }
   AddStatis(found_subgraph_count);
 }
 
-int FusedMultiTransformerXPUQuantPass::ApplyImpl(ir::Graph* graph,
-                                                 bool with_pre_caches,
-                                                 bool with_rotary_pos_emb,
-                                                 bool with_time_step,
-                                                 bool with_seq_lengths,
-                                                 bool with_src_mask) const {
+void FusedMultiTransformerXPUPass::RemoveAssignGather(ir::Graph* graph) const {
+  // detect assign + gather
+  GraphPatternDetector gpd;
+  patterns::FusedMultiTransformerAssignPattern pattern(gpd.mutable_pattern(),
+                                                       name_scope_);
+  int found_subgraph_count = 0;
+
+  auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph,
+                     Graph* graph) {
+    VLOG(1) << "handle RemoveAssignGather";
+    GET_IR_NODE(assign);
+    GET_IR_NODE(assign_out);
+    // Assign_out may not link to gather, so we find gather by input name.
+    auto next_ops = FindOpNodeByInputName(graph, assign_out->Name());
+    if (next_ops.size() != 1 || next_ops[0]->Name() != "gather") return;
+    auto* gather = next_ops[0];
+
+    // "assign_out" is used in multi blocks. "assign_out" should be reserved.
+    auto* gather_index = gather->inputs[0];
+    auto* assign_in = assign->inputs[0];
+    auto* fused_multi_transformer = assign_in->inputs[0];
+    fused_multi_transformer->Op()->Rename(assign_in->Name(),
+                                          assign_out->Name());
+    fused_multi_transformer->Op()->SetInput("gather_index",
+                                            gather->Op()->Input("Index"));
+    fused_multi_transformer->Op()->SetAttr("gather_axis",
+                                           gather->Op()->GetAttr("axis"));
+    IR_NODE_LINK_TO(gather_index, fused_multi_transformer);
+    IR_NODE_LINK_TO(fused_multi_transformer, assign_out);
+
+    std::unordered_set<const Node*> delete_nodes{assign, assign_in, gather};
+    GraphSafeRemoveNodes(graph, delete_nodes);
+    found_subgraph_count++;
+  };
+
+  gpd(graph, handler);
+  AddStatis(found_subgraph_count);
+}
+
+int FusedMultiTransformerXPUPass::FusedMultiTransformerXPUQuant(
+    ir::Graph* graph,
+    bool with_pre_caches,
+    bool with_rotary_pos_emb,
+    bool with_time_step,
+    bool with_seq_lengths,
+    bool with_src_mask) const {
   GraphPatternDetector gpd;
   patterns::FusedMultiTransformerPattern pattern(gpd.mutable_pattern(),
                                                  name_scope_,
@@ -286,7 +373,7 @@ int FusedMultiTransformerXPUQuantPass::ApplyImpl(ir::Graph* graph,
   int found_subgraph_count = 0;
   auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph,
                      Graph* graph) {
-    VLOG(4) << "handle FusedMultiTransformerXPUQuantPass fuse";
+    VLOG(4) << "handle FusedMultiTransformerXPUQuant";
 
     GET_IR_NODE(x);
     GET_IR_NODE(ln_scale);
@@ -459,6 +546,13 @@ int FusedMultiTransformerXPUQuantPass::ApplyImpl(ir::Graph* graph,
     if (name_caches.count("CacheKV") > 0) {
       fused_mt_xpu_op_desc->SetInput("cache_kv", name_caches.at("CacheKV"));
     }
+    if (name_caches.count("gather_index") > 0) {
+      fused_mt_xpu_op_desc->SetInput("gather_index",
+                                     name_caches.at("gather_index"));
+    }
+    if (!fused_mt_xpu_op_desc->HasAttr("gather_axis")) {
+      fused_mt_xpu_op_desc->SetAttr("gather_axis", 0);
+    }
     if (pre_caches) {
       fused_mt_xpu_op_desc->SetInput("pre_caches", name_caches.at("PreCaches"));
     }
@@ -529,5 +623,5 @@ int FusedMultiTransformerXPUQuantPass::ApplyImpl(ir::Graph* graph,
 }  // namespace framework
 }  // namespace paddle
 
-REGISTER_PASS(fused_multi_transformer_xpu_quant_pass,
-              paddle::framework::ir::FusedMultiTransformerXPUQuantPass);
+REGISTER_PASS(fused_multi_transformer_xpu_pass,
+              paddle::framework::ir::FusedMultiTransformerXPUPass);
diff --git a/paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_quant_pass_tester.cc b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_pass_tester.cc
similarity index 69%
rename from paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_quant_pass_tester.cc
rename to paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_pass_tester.cc
index cdcc20cb2f765..9251387f867eb 100644
--- a/paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_quant_pass_tester.cc
+++ b/paddle/fluid/framework/ir/xpu/fused_multi_transformer_xpu_pass_tester.cc
@@ -64,7 +64,62 @@ Scope* CreateParamScope() {
   return param_scope;
 }
 
-TEST(FusedMultiTransformerXPUQuantPass, context_stage) {
+VarDesc* Data(paddle::framework::BlockDesc* block,
+              std::string name,
+              std::vector<int64_t> shape = {},
+              bool is_persistable = false,
+              proto::VarType::Type data_type = proto::VarType::FP32) {
+  auto* var = block->Var(name);
+  var->SetType(proto::VarType::LOD_TENSOR);
+  var->SetDataType(data_type);
+  var->SetShape(shape);
+  var->SetPersistable(is_persistable);
+  return var;
+}
+
+TEST(RemoveAssignGather, basic) {
+  paddle::framework::ProgramDesc program;
+  auto* block = program.MutableBlock(0);
+
+  auto* x = Data(block, "fused_multi_transformer_x", {1, 1, 1536});
+  auto* cache_kv =
+      Data(block, "fused_multi_transformer_cache_kv", {2, 1, 24, 512, 64});
+  OpDesc* fused_multi_transformer_op = block->AppendOp();
+  fused_multi_transformer_op->SetType("fused_multi_transformer");
+  fused_multi_transformer_op->SetInput("X", {x->Name()});
+  fused_multi_transformer_op->SetInput("CacheKV", {cache_kv->Name()});
+  fused_multi_transformer_op->SetOutput("CacheKVOut", {cache_kv->Name()});
+
+  auto* assign_out = Data(block, "assign_out", cache_kv->GetShape());
+  OpDesc* assign_op = block->AppendOp();
+  assign_op->SetType("assign");
+  assign_op->SetInput("X", {cache_kv->Name()});
+  assign_op->SetOutput("Out", {assign_out->Name()});
+
+  OpDesc* gather_op = block->AppendOp();
+  auto gather_index = Data(block, "gather_index", {10});
+  gather_op->SetType("gather");
+  gather_op->SetInput("X", {assign_out->Name()});
+  gather_op->SetInput("Index", {gather_index->Name()});
+  gather_op->SetAttr("axis", {1});
+  gather_op->SetOutput("Out", {cache_kv->Name()});
+
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(program));
+  auto pass = PassRegistry::Instance().Get("fused_multi_transformer_xpu_pass");
+  pass->Apply(graph.get());
+  auto assign_num = GetNumOpNodes(graph, "assign");
+  auto gather_num = GetNumOpNodes(graph, "gather");
+  PADDLE_ENFORCE_EQ(assign_num,
+                    0,
+                    platform::errors::PreconditionNotMet(
+                        "assign op should be removed from the graph."));
+  PADDLE_ENFORCE_EQ(gather_num,
+                    0,
+                    platform::errors::PreconditionNotMet(
+                        "gather op should be removed from the graph."));
+}
+
+TEST(FusedMultiTransformerXPUPass, context_stage) {
   DEF_INPUT_DATA
 
   auto* cache_kv = layers.fill_constant_batch_size_like(
@@ -95,10 +150,9 @@ TEST(FusedMultiTransformerXPUQuantPass, context_stage) {
   std::unique_ptr<ir::Graph> graph(new ir::Graph(layers.main_program()));
   graph->Set("__param_scope__", CreateParamScope());
 
-  auto pass =
-      PassRegistry::Instance().Get("fused_multi_transformer_xpu_quant_pass");
+  auto pass = PassRegistry::Instance().Get("fused_multi_transformer_xpu_pass");
   if (pass.get() == nullptr) {
-    LOG(INFO) << "get fused_multi_transformer_xpu_quant_pass failed";
+    LOG(INFO) << "get fused_multi_transformer_xpu_pass failed";
   }
 
   graph.reset(pass->Apply(graph.release()));
@@ -114,7 +168,7 @@ TEST(FusedMultiTransformerXPUQuantPass, context_stage) {
           num_nodes_after));
 }
 
-TEST(FusedMultiTransformerXPUQuantPass, decoder_stage) {
+TEST(FusedMultiTransformerXPUPass, decoder_stage) {
   DEF_INPUT_DATA
 
   auto* cache_kv = layers.fill_constant_batch_size_like(
@@ -146,10 +200,9 @@ TEST(FusedMultiTransformerXPUQuantPass, decoder_stage) {
   std::unique_ptr<ir::Graph> graph(new ir::Graph(layers.main_program()));
   graph->Set("__param_scope__", CreateParamScope());
 
-  auto pass =
-      PassRegistry::Instance().Get("fused_multi_transformer_xpu_quant_pass");
+  auto pass = PassRegistry::Instance().Get("fused_multi_transformer_xpu_pass");
   if (pass.get() == nullptr) {
-    LOG(INFO) << "get fused_multi_transformer_xpu_quant_pass failed";
+    LOG(INFO) << "get fused_multi_transformer_xpu_pass failed";
   }
 
   graph.reset(pass->Apply(graph.release()));
@@ -169,4 +222,4 @@ TEST(FusedMultiTransformerXPUQuantPass, decoder_stage) {
 }  // namespace framework
 }  // namespace paddle
 
-USE_PASS(fused_multi_transformer_xpu_quant_pass);
+USE_PASS(fused_multi_transformer_xpu_pass);
diff --git a/paddle/fluid/framework/ir/xpu/one_beam_size_fuse_pass.cc b/paddle/fluid/framework/ir/xpu/one_beam_size_fuse_pass.cc
index 9ab6cd031b5c7..f50ee81eaaefd 100644
--- a/paddle/fluid/framework/ir/xpu/one_beam_size_fuse_pass.cc
+++ b/paddle/fluid/framework/ir/xpu/one_beam_size_fuse_pass.cc
@@ -259,23 +259,6 @@ bool OnlyOneBeamSearchAndOneBeamSize(ir::Graph* graph) {
          beam_search_nodes[0]->Op()->GetAttrIfExists<int>("beam_size") == 1;
 }
 
-std::vector<Node*> FindOpNodeByInputName(Graph* graph,
-                                         const std::string& var_name) {
-  std::vector<Node*> ret;
-  for (auto* node : graph->Nodes()) {
-    if (!node->IsOp()) continue;
-    auto inputs = node->Op()->Inputs();
-    for (auto input : inputs) {
-      auto in_names = input.second;
-      if (std::count(in_names.begin(), in_names.end(), var_name) > 0) {
-        ret.push_back(node);
-        break;
-      }
-    }
-  }
-  return ret;
-}
-
 void OneBeamSizeFusePass::RemoveAssignGather(ir::Graph* graph) const {
   // detect assign + gather
   GraphPatternDetector gpd;
diff --git a/paddle/fluid/framework/ir/xpu/pass_utils.cc b/paddle/fluid/framework/ir/xpu/pass_utils.cc
index aaa117d363a5f..eeb0e23e19ecd 100644
--- a/paddle/fluid/framework/ir/xpu/pass_utils.cc
+++ b/paddle/fluid/framework/ir/xpu/pass_utils.cc
@@ -71,6 +71,23 @@ Node* FindNodeWithName(Graph* graph, std::string name) {
   return nullptr;
 }
 
+std::vector<Node*> FindOpNodeByInputName(Graph* graph,
+                                         const std::string& var_name) {
+  std::vector<Node*> ret;
+  for (auto* node : graph->Nodes()) {
+    if (!node->IsOp()) continue;
+    auto inputs = node->Op()->Inputs();
+    for (auto input : inputs) {
+      auto in_names = input.second;
+      if (std::count(in_names.begin(), in_names.end(), var_name) > 0) {
+        ret.push_back(node);
+        break;
+      }
+    }
+  }
+  return ret;
+}
+
 template <typename T>
 std::string IntTypeToString() {
   LOG(FATAL) << "Not support type.";
diff --git a/paddle/fluid/framework/ir/xpu/pass_utils.h b/paddle/fluid/framework/ir/xpu/pass_utils.h
index 68cfb2953e1d5..d1e7b218a0b46 100644
--- a/paddle/fluid/framework/ir/xpu/pass_utils.h
+++ b/paddle/fluid/framework/ir/xpu/pass_utils.h
@@ -51,6 +51,9 @@ int ConvertActivationType(std::string act_type);
 
 Node* FindNodeWithName(Graph* graph, std::string name);
 
+std::vector<Node*> FindOpNodeByInputName(Graph* graph,
+                                         const std::string& var_name);
+
 template <typename T>
 size_t HashTensor(const phi::DenseTensor& in);
 
diff --git a/paddle/fluid/inference/api/paddle_pass_builder.cc b/paddle/fluid/inference/api/paddle_pass_builder.cc
index 134d56180b617..580578f2a04ad 100644
--- a/paddle/fluid/inference/api/paddle_pass_builder.cc
+++ b/paddle/fluid/inference/api/paddle_pass_builder.cc
@@ -523,7 +523,7 @@ XpuPassStrategy::XpuPassStrategy() : PassStrategy({}) {
       "one_beam_size_fuse_pass",
       "delete_cast_op_pass",
       "stack_fuse_pass",
-      "fused_multi_transformer_xpu_quant_pass",
+      "fused_multi_transformer_xpu_pass",
       "fc_xpu_fuse_pass",
       "conv2d_xpu_fuse_pass",
       "link_xpu_op_max_pass",
diff --git a/paddle/phi/api/yaml/fused_ops.yaml b/paddle/phi/api/yaml/fused_ops.yaml
index f4570f711412d..87400ecd61c9f 100644
--- a/paddle/phi/api/yaml/fused_ops.yaml
+++ b/paddle/phi/api/yaml/fused_ops.yaml
@@ -58,14 +58,14 @@
   support_dygraph_mode : true
 
 - op : fused_multi_transformer_xpu
-  args : (Tensor x, Tensor[] ln_scale, Tensor[] ln_bias, Tensor[] qkvw, Tensor[] qkvw_max, Tensor[] qkv_bias, Tensor[] out_linear_w, Tensor[] out_linear_wmax, Tensor[] out_linear_bias, Tensor[] ffn_ln_scale, Tensor[] ffn_ln_bias, Tensor[] ffn1_weight, Tensor[] ffn1_weight_max, Tensor[] ffn1_bias, Tensor[] ffn2_weight, Tensor[] ffn2_weight_max, Tensor[] ffn2_bias, Tensor[] cache_kv, Tensor[] pre_caches, Tensor rotary_pos_emb, Tensor time_step, Tensor seq_lengths, Tensor src_mask, bool pre_layer_norm, int rotary_emb_dims, float epsilon, float dropout_rate, bool is_test, str dropout_implementation, str act_method, bool trans_qkvw, int ring_id)
+  args : (Tensor x, Tensor[] ln_scale, Tensor[] ln_bias, Tensor[] qkvw, Tensor[] qkvw_max, Tensor[] qkv_bias, Tensor[] out_linear_w, Tensor[] out_linear_wmax, Tensor[] out_linear_bias, Tensor[] ffn_ln_scale, Tensor[] ffn_ln_bias, Tensor[] ffn1_weight, Tensor[] ffn1_weight_max, Tensor[] ffn1_bias, Tensor[] ffn2_weight, Tensor[] ffn2_weight_max, Tensor[] ffn2_bias, Tensor[] cache_kv, Tensor[] pre_caches, Tensor rotary_pos_emb, Tensor time_step, Tensor seq_lengths, Tensor src_mask, Tensor gather_index, bool pre_layer_norm, int rotary_emb_dims, float epsilon, float dropout_rate, bool is_test, str dropout_implementation, str act_method, bool trans_qkvw, int ring_id, int gather_axis)
   output : Tensor(out), Tensor[](cache_kv_out){out_linear_w.size()}
   infer_meta :
     func : FusedMultiTransformerXpuInferMeta
   kernel :
     func : fused_multi_transformer_xpu
     data_type : x
-  optional : cache_kv, pre_caches, rotary_pos_emb, time_step, seq_lengths, src_mask
+  optional : cache_kv, pre_caches, rotary_pos_emb, time_step, seq_lengths, src_mask, gather_index
 
 - op : generate_sequence_xpu
   args : (Tensor x, DataType dtype)
diff --git a/paddle/phi/infermeta/fusion.cc b/paddle/phi/infermeta/fusion.cc
index fb7ede4c9e742..262c5fb04e19f 100644
--- a/paddle/phi/infermeta/fusion.cc
+++ b/paddle/phi/infermeta/fusion.cc
@@ -278,6 +278,7 @@ void FusedMultiTransformerXpuInferMeta(
     const std::vector<const MetaTensor*>& time_step,
     const std::vector<const MetaTensor*>& seq_lengths,
     const std::vector<const MetaTensor*>& src_mask,
+    const std::vector<const MetaTensor*>& gather_index,
     bool pre_layer_norm,
     int rotary_emb_dims,
     float epsilon,
@@ -287,6 +288,7 @@ void FusedMultiTransformerXpuInferMeta(
     const std::string& act_method,
     bool trans_qkvw,
     int ring_id,
+    int gather_axis,
     MetaTensor* out,
     std::vector<MetaTensor*> cache_kv_out) {
   auto x_dim = x.dims();
@@ -325,13 +327,6 @@ void FusedMultiTransformerXpuInferMeta(
                       phi::errors::InvalidArgument(
                           "The first dim of CacheKV must be 2, but got %d",
                           c_dim[0]));  // 2
-    PADDLE_ENFORCE_EQ(
-        c_dim[2],
-        x_dim[0],
-        phi::errors::InvalidArgument("The third dim of CacheKV must be equal "
-                                     "with batch size %d, but got %d",
-                                     x_dim[0],
-                                     c_dim[2]));  // batch_size
     PADDLE_ENFORCE_EQ(
         c_dim[3],
         trans_qkvw ? y_dim[1] : y_dim[2],
diff --git a/paddle/phi/infermeta/fusion.h b/paddle/phi/infermeta/fusion.h
index 3105ea8a6d578..38f4bc8c6c5be 100644
--- a/paddle/phi/infermeta/fusion.h
+++ b/paddle/phi/infermeta/fusion.h
@@ -108,6 +108,7 @@ void FusedMultiTransformerXpuInferMeta(
     const std::vector<const MetaTensor*>& time_step,
     const std::vector<const MetaTensor*>& seq_lengths,
     const std::vector<const MetaTensor*>& src_mask,
+    const std::vector<const MetaTensor*>& gather_index,
     bool pre_layer_norm,
     int rotary_emb_dims,
     float epsilon,
@@ -117,6 +118,7 @@ void FusedMultiTransformerXpuInferMeta(
     const std::string& act_method,
     bool trans_qkvw,
     int ring_id,
+    int gather_axis,
     MetaTensor* out,
     std::vector<MetaTensor*> cache_kv_out);
 }  // namespace phi
diff --git a/paddle/phi/kernels/fusion/xpu/fused_multi_transformer_xpu_kernel.cc b/paddle/phi/kernels/fusion/xpu/fused_multi_transformer_xpu_kernel.cc
index 0d27a398e449b..29a3c7d61adff 100644
--- a/paddle/phi/kernels/fusion/xpu/fused_multi_transformer_xpu_kernel.cc
+++ b/paddle/phi/kernels/fusion/xpu/fused_multi_transformer_xpu_kernel.cc
@@ -17,6 +17,8 @@
 #include "glog/logging.h"
 
 #include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/meta_tensor.h"
+#include "paddle/phi/infermeta/binary.h"
 #include "paddle/phi/kernels/memcpy_kernel.h"
 #ifdef PADDLE_WITH_XPU_XFT
 #include "models/fused_multi_transformer_op.h"
@@ -52,6 +54,7 @@ void FusedMultiTransformerXpuKernel(
     const paddle::optional<DenseTensor>& time_step,
     const paddle::optional<DenseTensor>& seq_lengths,
     const paddle::optional<DenseTensor>& src_mask,
+    const paddle::optional<DenseTensor>& gather_index,
     bool pre_layer_norm,
     int rotary_emb_dims,
     float epsilon,
@@ -61,6 +64,7 @@ void FusedMultiTransformerXpuKernel(
     const std::string& act_method,
     bool trans_qkvw,
     int ring_id,
+    int gather_axis,
     DenseTensor* out,
     std::vector<DenseTensor*> cache_kv_out) {
 #ifdef PADDLE_WITH_XPU_XFT
@@ -160,6 +164,21 @@ void FusedMultiTransformerXpuKernel(
   std::vector<xft::xftTensor<XPUTypeT, 5>> xft_cache_kv;
   std::vector<xft::xftTensor<XPUTypeT, 5>> xft_cache_kv_out;
 
+  // Create a temporary Tensor to store the gather output of cache_kv
+  auto gather_index_t = gather_index.get_ptr();
+  auto cache_kv_dims = cache_kv.get_ptr()->at(0)->dims();
+  auto cache_kv_gather_dims = cache_kv_dims;
+  phi::DenseTensor cache_kv_gather_tensor;
+  if (gather_index_t) {
+    MetaTensor cache_kv_gather_meta(&cache_kv_gather_tensor);
+    phi::GatherInferMeta(*cache_kv.get_ptr()->at(0),
+                         *gather_index_t,
+                         Scalar(gather_axis),
+                         &cache_kv_gather_meta);
+    cache_kv_gather_dims = cache_kv_gather_meta.dims();
+    ctx.template Alloc<T>(&cache_kv_gather_tensor);
+  }
+
   int layers = qkvw.size();
   for (int i = 0; i < layers; ++i) {
     // step1. layer_norm
@@ -211,27 +230,55 @@ void FusedMultiTransformerXpuKernel(
     xft_ffn2_bias.emplace_back(const_cast<float*>(ffn2_bias[i]->data<float>()),
                                std::array<int64_t, 1>{ffn2_bias[i]->dims()[0]});
     // cache kv in
-    if (time_step_value > 0) {
-      auto cachekv_dims = cache_kv.get_ptr()->at(i)->dims();
-      xft_cache_kv.emplace_back(reinterpret_cast<XPUTypeT*>(const_cast<T*>(
-                                    cache_kv.get_ptr()->at(i)->data<T>())),
-                                std::array<int64_t, 5>{cachekv_dims[0],
-                                                       cachekv_dims[1],
-                                                       cachekv_dims[2],
-                                                       cachekv_dims[3],
-                                                       cachekv_dims[4]});
+    auto cache_kv_data = reinterpret_cast<XPUTypeT*>(
+        const_cast<T*>(cache_kv.get_ptr()->at(i)->data<T>()));
+    if (gather_index_t) {
+      const auto& index_type = gather_index_t->dtype();
+      if (index_type == DataType::INT32) {
+        r = xpu::gather<XPUTypeT, int32_t>(
+            ctx.x_context(),
+            cache_kv_data,
+            gather_index_t->data<int32_t>(),
+            reinterpret_cast<XPUTypeT*>(cache_kv_gather_tensor.data<T>()),
+            phi::vectorize<int32_t>(cache_kv_dims),
+            gather_index_t->dims().size() == 0 ? 1 : gather_index_t->dims()[0],
+            gather_axis);
+      } else {
+        r = xpu::gather<XPUTypeT, int64_t>(
+            ctx.x_context(),
+            cache_kv_data,
+            gather_index_t->data<int64_t>(),
+            reinterpret_cast<XPUTypeT*>(cache_kv_gather_tensor.data<T>()),
+            phi::vectorize<int32_t>(cache_kv_dims),
+            gather_index_t->dims().size() == 0 ? 1 : gather_index_t->dims()[0],
+            gather_axis);
+      }
+      PADDLE_ENFORCE_XDNN_SUCCESS(r, "xpu::gather");
+      cache_kv_out[i]->ResizeAndAllocate(cache_kv_gather_dims);
+      r = xpu::copy<XPUTypeT>(
+          ctx.x_context(),
+          reinterpret_cast<XPUTypeT*>(cache_kv_gather_tensor.data<T>()),
+          reinterpret_cast<XPUTypeT*>(ctx.template Alloc<T>(cache_kv_out[i])),
+          cache_kv_out[i]->numel());
+      PADDLE_ENFORCE_XDNN_SUCCESS(r, "xpu::copy");
     }
-    // cache kv out
-    auto cachekv_out_dims = cache_kv_out[i]->dims();
+    cache_kv_data = reinterpret_cast<XPUTypeT*>(
+        const_cast<T*>(cache_kv.get_ptr()->at(i)->data<T>()));
+    xft_cache_kv.emplace_back(cache_kv_data,
+                              std::array<int64_t, 5>{cache_kv_gather_dims[0],
+                                                     cache_kv_gather_dims[1],
+                                                     cache_kv_gather_dims[2],
+                                                     cache_kv_gather_dims[3],
+                                                     cache_kv_gather_dims[4]});
+    // cache kv out direct use cache_kv_data
     xft_cache_kv_out.emplace_back(
-        reinterpret_cast<XPUTypeT*>(ctx.template Alloc<T>(cache_kv_out[i])),
-        std::array<int64_t, 5>{cachekv_out_dims[0],
-                               cachekv_out_dims[1],
-                               cachekv_out_dims[2],
-                               cachekv_out_dims[3],
-                               cachekv_out_dims[4]});
+        cache_kv_data,
+        std::array<int64_t, 5>{cache_kv_gather_dims[0],
+                               cache_kv_gather_dims[1],
+                               cache_kv_gather_dims[2],
+                               cache_kv_gather_dims[3],
+                               cache_kv_gather_dims[4]});
   }
-
   xft::NlpParam param;
   param.num_layer = layers;
   param.n_head = num_head;

From 58435ae5be55d0eaf1a8ba90cdf0c8f18d4a6fe9 Mon Sep 17 00:00:00 2001
From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com>
Date: Fri, 5 May 2023 18:01:02 +0800
Subject: [PATCH 175/405] remove some [-Wunused-parameter]warning (#53397)

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop
---
 paddle/fluid/eager/amp_auto_cast.h                      | 2 +-
 paddle/phi/kernels/funcs/activation_functor.h           | 2 +-
 paddle/phi/kernels/funcs/compound_functors.h            | 2 +-
 paddle/phi/kernels/funcs/reduce_functor.h               | 4 ++--
 paddle/phi/kernels/fusion/onednn/fused_matmul_kernel.cc | 2 +-
 paddle/phi/kernels/impl/slice_grad_kernel_impl.h        | 2 +-
 paddle/phi/kernels/npu_identity_kernel.cc               | 4 ++--
 paddle/phi/kernels/sparse/sparse_utils_grad_kernel.cc   | 2 +-
 8 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/paddle/fluid/eager/amp_auto_cast.h b/paddle/fluid/eager/amp_auto_cast.h
index 8200f41f568c5..3f02d68b2aa2e 100644
--- a/paddle/fluid/eager/amp_auto_cast.h
+++ b/paddle/fluid/eager/amp_auto_cast.h
@@ -43,7 +43,7 @@ inline std::vector<paddle::Tensor> AmpAutoCasts(
     const std::string& inputs_name,
     const std::vector<paddle::Tensor>& inputs,
     const phi::DataType& dst_dtype,
-    std::string op_name) {
+    std::string op_name UNUSED) {
   VLOG(6) << "AMP AmpAutoCasts:"
           << " inputs(" << inputs_name << ") dst_dtype("
           << phi::DataTypeToString(dst_dtype) << ").";
diff --git a/paddle/phi/kernels/funcs/activation_functor.h b/paddle/phi/kernels/funcs/activation_functor.h
index aefa653928633..bc1b21c1c8cce 100644
--- a/paddle/phi/kernels/funcs/activation_functor.h
+++ b/paddle/phi/kernels/funcs/activation_functor.h
@@ -1147,7 +1147,7 @@ struct ReluGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x UNUSED, Out out, dOut dout, dX dx) const {
     dx.device(d) = dout * (out > static_cast<T>(0)).template cast<T>();
   }
 
diff --git a/paddle/phi/kernels/funcs/compound_functors.h b/paddle/phi/kernels/funcs/compound_functors.h
index 020c6dcd87f30..d51c13950aa33 100644
--- a/paddle/phi/kernels/funcs/compound_functors.h
+++ b/paddle/phi/kernels/funcs/compound_functors.h
@@ -247,7 +247,7 @@ struct UnaryCompoundGradDIntermediateFunctor {
     }
   }
 
-  inline HOSTDEVICE T UseIntermediateOut(T x,
+  inline HOSTDEVICE T UseIntermediateOut(T x UNUSED,
                                          T intermediate_out,
                                          T out,
                                          T dout) {
diff --git a/paddle/phi/kernels/funcs/reduce_functor.h b/paddle/phi/kernels/funcs/reduce_functor.h
index 596be9bc33058..80217520b126f 100644
--- a/paddle/phi/kernels/funcs/reduce_functor.h
+++ b/paddle/phi/kernels/funcs/reduce_functor.h
@@ -112,8 +112,8 @@ struct MeanGradFunctor {
             typename DY,
             typename Dim>
   void operator()(const DeviceContext& place,
-                  X* x,
-                  Y* y,
+                  X* x UNUSED,
+                  Y* y UNUSED,
                   DX* dx,
                   DY* dy,
                   const Dim& dim,
diff --git a/paddle/phi/kernels/fusion/onednn/fused_matmul_kernel.cc b/paddle/phi/kernels/fusion/onednn/fused_matmul_kernel.cc
index b5ae7e39f4ede..fea93f414161c 100644
--- a/paddle/phi/kernels/fusion/onednn/fused_matmul_kernel.cc
+++ b/paddle/phi/kernels/fusion/onednn/fused_matmul_kernel.cc
@@ -142,7 +142,7 @@ class FusedMatmulOneDNNHandler
   float ComputeOutputScale(float matmul_alpha,
                            const float scale_x,
                            const float scale_y,
-                           const float scale_in_eltwise,
+                           const float scale_in_eltwise UNUSED,
                            const float scale_out,
                            const bool force_fp32_output) {
     float f_scale_out = force_fp32_output ? 1.0f : scale_out;
diff --git a/paddle/phi/kernels/impl/slice_grad_kernel_impl.h b/paddle/phi/kernels/impl/slice_grad_kernel_impl.h
index 6128f8c7e1902..33ec8f8a84109 100644
--- a/paddle/phi/kernels/impl/slice_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/slice_grad_kernel_impl.h
@@ -357,7 +357,7 @@ void SliceArrayGradKernel(const Context& dev_ctx,
                           const TensorArray& input,
                           const TensorArray& out_grad,
                           const IntArray& starts,
-                          const IntArray& ends,
+                          const IntArray& ends UNUSED,
                           TensorArray* input_grad) {
   int64_t d_in_size = input.size();
   input_grad->resize(d_in_size);
diff --git a/paddle/phi/kernels/npu_identity_kernel.cc b/paddle/phi/kernels/npu_identity_kernel.cc
index 9a96c149637e9..89a0c63c8a495 100644
--- a/paddle/phi/kernels/npu_identity_kernel.cc
+++ b/paddle/phi/kernels/npu_identity_kernel.cc
@@ -22,9 +22,9 @@
 namespace phi {
 
 template <typename T, typename Context>
-void NPUIdentityKernel(const Context& dev_ctx,
+void NPUIdentityKernel(const Context& dev_ctx UNUSED,
                        const DenseTensor& x,
-                       const int format,
+                       const int format UNUSED,
                        DenseTensor* out) {
   VLOG(4) << "npu_identity op is only for NPU, please avoid using this kernel!";
   out->ShareDataWith(x);
diff --git a/paddle/phi/kernels/sparse/sparse_utils_grad_kernel.cc b/paddle/phi/kernels/sparse/sparse_utils_grad_kernel.cc
index 5c4aa7b6b56b6..064867610d719 100644
--- a/paddle/phi/kernels/sparse/sparse_utils_grad_kernel.cc
+++ b/paddle/phi/kernels/sparse/sparse_utils_grad_kernel.cc
@@ -20,7 +20,7 @@ namespace phi {
 namespace sparse {
 
 template <typename T, typename Context>
-void ValuesCooGradKernel(const Context& dev_ctx,
+void ValuesCooGradKernel(const Context& dev_ctx UNUSED,
                          const SparseCooTensor& x,
                          const DenseTensor& out_grad,
                          SparseCooTensor* x_grad) {

From 0d9a23b4c36714f493a4c0468cdc663cf2e233ba Mon Sep 17 00:00:00 2001
From: Haohongxiang <86215757+haohongxiang@users.noreply.github.com>
Date: Fri, 5 May 2023 21:40:45 +0800
Subject: [PATCH 176/405] [Dygraph] Fix bugs in dp_pp_comm_overlap for
 HybridParallel (#53384)

---
 .../distributed/fleet/meta_parallel/pipeline_parallel.py | 3 +++
 python/paddle/distributed/fleet/optimizer.py             | 9 ++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py
index ab1cf9701dd63..2a7dd4d0bb710 100755
--- a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py
+++ b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py
@@ -75,6 +75,9 @@ def __init__(self, layers, hcg, strategy):
         ].dp_comm_overlap
         self._dp_comm_buffers = []
 
+        if self._dp_comm_overlap:
+            assert self.use_data_parallel and self.num_stages > 1
+
         p2p.initialize_p2p_groups(
             hcg, self._using_cache, self._enable_partial_send_recv
         )
diff --git a/python/paddle/distributed/fleet/optimizer.py b/python/paddle/distributed/fleet/optimizer.py
index 89b0456fe6abf..5abe7c47e9b25 100755
--- a/python/paddle/distributed/fleet/optimizer.py
+++ b/python/paddle/distributed/fleet/optimizer.py
@@ -62,9 +62,16 @@ def _dygraph_distributed_optimizer(optimizer, strategy=None):
 
     if fleet_env.worker_num() > 1:
         if not fleet_env._user_defined_strategy.heter_ccl_mode:
-            return HybridParallelOptimizer(
+            hp_optim = HybridParallelOptimizer(
                 optimizer, fleet_env._hcg, fleet_env._user_defined_strategy
             )
+
+            if fleet_env._user_defined_strategy.hybrid_configs[
+                "pp_configs"
+            ].dp_comm_overlap:
+                hp_optim._dp_enable = False
+
+            return hp_optim
         else:
             return HeterParallelOptimizer(
                 optimizer, fleet_env._user_defined_strategy

From d463f8ee3871408a7e5e691a69122d534c6fb958 Mon Sep 17 00:00:00 2001
From: iLeGend <824040212@qq.com>
Date: Fri, 5 May 2023 23:48:41 +0800
Subject: [PATCH 177/405] =?UTF-8?q?Revert=20"=E3=80=90Hackathon=20No.52?=
 =?UTF-8?q?=E3=80=91=E4=B8=BA=20Paddle=20dist=20=E7=AE=97=E5=AD=90?=
 =?UTF-8?q?=E5=AE=9E=E7=8E=B0=20float16=20=E6=95=B0=E6=8D=AE=E7=B1=BB?=
 =?UTF-8?q?=E5=9E=8B=E6=94=AF=E6=8C=81=20(#50915)"=20(#53527)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit 9c4065316d89f3f57a9b58437104a29d1837b84a.
---
 paddle/phi/kernels/dist_grad_kernel.cc        |  9 +-
 paddle/phi/kernels/funcs/math_cuda_utils.h    | 27 +++---
 paddle/phi/kernels/gpu/dist_kernel.cu         | 84 ++++++++-----------
 .../fluid/tests/unittests/test_dist_op.py     | 40 ---------
 python/paddle/tensor/linalg.py                | 12 +--
 5 files changed, 56 insertions(+), 116 deletions(-)

diff --git a/paddle/phi/kernels/dist_grad_kernel.cc b/paddle/phi/kernels/dist_grad_kernel.cc
index 3c7ffc01edf07..17c24fa905b5c 100644
--- a/paddle/phi/kernels/dist_grad_kernel.cc
+++ b/paddle/phi/kernels/dist_grad_kernel.cc
@@ -98,11 +98,6 @@ PD_REGISTER_KERNEL(
     dist_grad, CPU, ALL_LAYOUT, phi::DistGradKernel, float, double) {}
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-PD_REGISTER_KERNEL(dist_grad,
-                   GPU,
-                   ALL_LAYOUT,
-                   phi::DistGradKernel,
-                   phi::dtype::float16,
-                   float,
-                   double) {}
+PD_REGISTER_KERNEL(
+    dist_grad, GPU, ALL_LAYOUT, phi::DistGradKernel, float, double) {}
 #endif
diff --git a/paddle/phi/kernels/funcs/math_cuda_utils.h b/paddle/phi/kernels/funcs/math_cuda_utils.h
index 26ead61e4e5f9..b493e2ac41bfd 100644
--- a/paddle/phi/kernels/funcs/math_cuda_utils.h
+++ b/paddle/phi/kernels/funcs/math_cuda_utils.h
@@ -23,9 +23,6 @@ limitations under the License. */
 
 #include <algorithm>
 
-#include "paddle/phi/backends/gpu/gpu_device_function.h"
-#include "paddle/phi/common/data_type.h"
-
 namespace phi {
 namespace funcs {
 
@@ -173,7 +170,11 @@ struct KeyValuePair<half> {
 template <typename T>
 __inline__ __device__ T WarpReduceSum(T val, unsigned lane_mask) {
   for (int mask = HALF_WARP; mask > 0; mask >>= 1)
-    val += phi::backends::gpu::CudaShuffleXorSync(lane_mask, val, mask);
+#if defined(PADDLE_WITH_CUDA) && (__CUDA_ARCH__ >= 350 && CUDA_VERSION >= 9000)
+    val += __shfl_xor_sync(lane_mask, val, mask, warpSize);
+#else
+    val += __shfl_xor(val, mask, warpSize);
+#endif
   return val;
 }
 
@@ -242,8 +243,11 @@ __inline__ __device__ T BlockReduceSumV2(T *val) {
 template <typename T>
 __inline__ __device__ T WarpReduceMax(T val, unsigned lane_mask) {
   for (int mask = HALF_WARP; mask > 0; mask >>= 1)
-    val = std::max(
-        val, phi::backends::gpu::CudaShuffleXorSync(lane_mask, val, mask));
+#if defined(PADDLE_WITH_CUDA) && (__CUDA_ARCH__ >= 350 && CUDA_VERSION >= 9000)
+    val = max(val, __shfl_xor_sync(lane_mask, val, mask, warpSize));
+#else
+    val = max(val, __shfl_xor(val, mask, warpSize));
+#endif
   return val;
 }
 
@@ -261,8 +265,11 @@ __inline__ __device__ T WarpReduceMaxV2(T *val) {
 template <typename T>
 __inline__ __device__ T WarpReduceMin(T val, unsigned lane_mask) {
   for (int mask = HALF_WARP; mask > 0; mask >>= 1)
-    val = std::min(
-        val, phi::backends::gpu::CudaShuffleXorSync(lane_mask, val, mask));
+#if defined(PADDLE_WITH_CUDA) && (__CUDA_ARCH__ >= 350 && CUDA_VERSION >= 9000)
+    val = min(val, __shfl_xor_sync(lane_mask, val, mask, warpSize));
+#else
+    val = min(val, __shfl_xor(val, mask, warpSize));
+#endif
   return val;
 }
 
@@ -303,7 +310,7 @@ __inline__ __device__ T BlockReduceMax(T val, unsigned mask) {
 
   // align block_span to warpSize
   int block_span = (blockDim.x + warpSize - 1) >> 5;
-  val = (lane < block_span) ? shared[lane] : std::numeric_limits<T>::min();
+  val = (lane < block_span) ? shared[lane] : -1e10f;
   val = WarpReduceMax(val, mask);
 
   return val;
@@ -351,7 +358,7 @@ __inline__ __device__ T BlockReduceMin(T val, unsigned mask) {
 
   // align block_span to warpSize
   int block_span = (blockDim.x + warpSize - 1) >> 5;
-  val = (lane < block_span) ? shared[lane] : std::numeric_limits<T>::max();
+  val = (lane < block_span) ? shared[lane] : 1e10f;
   val = WarpReduceMin(val, mask);
 
   return val;
diff --git a/paddle/phi/kernels/gpu/dist_kernel.cu b/paddle/phi/kernels/gpu/dist_kernel.cu
index 0a197053e1016..5040be8eaaca7 100644
--- a/paddle/phi/kernels/gpu/dist_kernel.cu
+++ b/paddle/phi/kernels/gpu/dist_kernel.cu
@@ -12,12 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <algorithm>
-
+#include "paddle/phi/kernels/dist_kernel.h"
 #include "paddle/phi/backends/gpu/gpu_launch_config.h"
-#include "paddle/phi/common/float16.h"
 #include "paddle/phi/core/kernel_registry.h"
-#include "paddle/phi/kernels/dist_kernel.h"
 #include "paddle/phi/kernels/elementwise_subtract_kernel.h"
 #include "paddle/phi/kernels/funcs/math_cuda_utils.h"
 #include "paddle/phi/kernels/gpu/reduce.h"
@@ -27,56 +24,47 @@ namespace phi {
 
 #define FULL_MASK 0xffffffff
 
-template <typename Tx, typename Ty = Tx>
+template <typename T>
 struct ZeroOrderFunctor {
-  HOSTDEVICE explicit inline ZeroOrderFunctor() {}
-
-  HOSTDEVICE inline Ty operator()(const Tx& x, const Tx& y) const {
-    return static_cast<Ty>(x != y);
+ public:
+  __device__ T operator()(const T& x, const T& y) const {
+    return static_cast<T>((x - y) != 0);
   }
 };
 
-template <typename Tx, typename Ty = Tx>
+template <typename T>
 struct OtherOrderFunctor {
-  HOSTDEVICE explicit inline OtherOrderFunctor(const Ty& _p_order)
-      : p_order(_p_order) {}
-
-  HOSTDEVICE inline Ty operator()(const Tx& x, const Tx& y) const {
-    return static_cast<Ty>(
-        pow(abs(static_cast<Ty>(x) - static_cast<Ty>(y)), p_order));
+  explicit OtherOrderFunctor(const T& p_order) : p_order_(p_order) {}
+  __device__ T operator()(const T& x, const T& y) const {
+    return static_cast<T>(pow(abs(x - y), p_order_));
   }
 
  private:
-  Ty p_order;
+  T p_order_;
 };
 
-template <typename Tx, typename Ty = Tx>
+template <typename T>
 struct PowFunctor {
-  HOSTDEVICE explicit inline PowFunctor(const Ty& _p_order)
-      : p_order(_p_order) {}
-
-  HOSTDEVICE inline Tx operator()(const Tx x) const {
-    return static_cast<Tx>(pow(static_cast<Ty>(x), p_order));
+  explicit PowFunctor(const T& p_order) : p_order_(p_order) {}
+  HOSTDEVICE inline T operator()(const T x) const {
+    return static_cast<T>(pow(x, p_order_));
   }
-
- private:
-  Ty p_order;
+  T p_order_;
 };
 
 template <typename T, typename Functor>
 __global__ void ReduceSumWithSubtract(
     const T* x, const T* y, T* out, int64_t N, Functor func) {
-  using MT = typename phi::dtype::MPTypeTrait<T>::Type;
-  MT sum_val(0.0);
+  T sum_val = 0;
   for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N;
        i += blockDim.x * gridDim.x) {
-    sum_val += static_cast<MT>(func(x[i], y[i]));
+    sum_val += func(x[i], y[i]);
   }
 
   __syncthreads();
-  sum_val = phi::funcs::BlockReduceSum<MT>(sum_val, FULL_MASK);
+  sum_val = phi::funcs::BlockReduceSum<T>(sum_val, FULL_MASK);
   if (threadIdx.x == 0) {
-    out[blockIdx.x] = static_cast<T>(sum_val);
+    out[blockIdx.x] = sum_val;
   }
 }
 
@@ -85,10 +73,10 @@ __global__ void ReduceMaxWithSubtract(const T* x,
                                       const T* y,
                                       T* out,
                                       int64_t N) {
-  T max_val = std::numeric_limits<T>::min();
+  T max_val = -1e10f;
   for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N;
        i += blockDim.x * gridDim.x) {
-    max_val = std::max(max_val, abs(x[i] - y[i]));
+    max_val = max(max_val, abs(x[i] - y[i]));
   }
 
   __syncthreads();
@@ -103,10 +91,10 @@ __global__ void ReduceMinWithSubtract(const T* x,
                                       const T* y,
                                       T* out,
                                       int64_t N) {
-  T min_val = std::numeric_limits<T>::max();
+  T min_val = 1e10f;
   for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N;
        i += blockDim.x * gridDim.x) {
-    min_val = std::min(min_val, abs(x[i] - y[i]));
+    min_val = min(min_val, abs(x[i] - y[i]));
   }
 
   __syncthreads();
@@ -122,7 +110,6 @@ void DistKernel(const Context& dev_ctx,
                 const DenseTensor& y,
                 float p,
                 DenseTensor* out) {
-  using MT = typename phi::dtype::MPTypeTrait<T>::Type;
   DenseTensor intermediate;
   const T* x_ptr = x.data<T>();
   const T* y_ptr = y.data<T>();
@@ -144,8 +131,9 @@ void DistKernel(const Context& dev_ctx,
       ReduceSumWithSubtract<T>
           <<<config.block_per_grid.x, config.thread_per_block.x, 0, stream>>>(
               x_ptr, y_ptr, i_ptr, n, ZeroOrderFunctor<T>());
-      phi::funcs::ReduceKernel<T, T, kps::AddFunctor, kps::IdentityFunctor<MT>>(
-          dev_ctx, intermediate, out, kps::IdentityFunctor<MT>(), reduce_axis);
+      phi::funcs::ReduceKernel<T, T, kps::AddFunctor, kps::IdentityFunctor<T>>(
+          dev_ctx, intermediate, out, kps::IdentityFunctor<T>(), reduce_axis);
+
     } else if (p == INFINITY) {
       ReduceMaxWithSubtract<T>
           <<<config.block_per_grid.x, config.thread_per_block.x, 0, stream>>>(
@@ -162,19 +150,19 @@ void DistKernel(const Context& dev_ctx,
           dev_ctx, intermediate, out, kps::IdentityFunctor<T>(), reduce_axis);
 
     } else {
-      MT p_order = static_cast<MT>(p);
+      T p_order = static_cast<T>(p);
       ReduceSumWithSubtract<T>
           <<<config.block_per_grid.x, config.thread_per_block.x, 0, stream>>>(
-              x_ptr, y_ptr, i_ptr, n, OtherOrderFunctor<T, MT>(p_order));
-      phi::funcs::ReduceKernel<T, T, kps::AddFunctor, kps::IdentityFunctor<MT>>(
-          dev_ctx, intermediate, out, kps::IdentityFunctor<MT>(), reduce_axis);
+              x_ptr, y_ptr, i_ptr, n, OtherOrderFunctor<T>(p_order));
+      phi::funcs::ReduceKernel<T, T, kps::AddFunctor, kps::IdentityFunctor<T>>(
+          dev_ctx, intermediate, out, kps::IdentityFunctor<T>(), reduce_axis);
 
       const DenseTensor* tmp_norm = out;
       std::vector<const DenseTensor*> ins = {tmp_norm};
       std::vector<DenseTensor*> outs = {out};
-      MT p_order_ = static_cast<MT>(static_cast<MT>(1.) / p_order);
+      T p_order_ = static_cast<T>(1. / p_order);
       phi::funcs::ElementwiseKernel<T>(
-          dev_ctx, ins, &outs, PowFunctor<T, MT>(p_order_));
+          dev_ctx, ins, &outs, PowFunctor<T>(p_order_));
     }
 
   } else {
@@ -185,10 +173,4 @@ void DistKernel(const Context& dev_ctx,
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(dist,
-                   GPU,
-                   ALL_LAYOUT,
-                   phi::DistKernel,
-                   phi::dtype::float16,
-                   float,
-                   double) {}
+PD_REGISTER_KERNEL(dist, GPU, ALL_LAYOUT, phi::DistKernel, float, double) {}
diff --git a/python/paddle/fluid/tests/unittests/test_dist_op.py b/python/paddle/fluid/tests/unittests/test_dist_op.py
index 3654da4e00fae..96c0de915cff2 100644
--- a/python/paddle/fluid/tests/unittests/test_dist_op.py
+++ b/python/paddle/fluid/tests/unittests/test_dist_op.py
@@ -158,46 +158,6 @@ def init_case(self):
         self.p = 1.5
 
 
-class TestDistFP16Op(OpTest):
-    def init_data_type(self):
-        self.data_type = 'float16'
-
-
-class TestDistFP16OpCase1(TestDistFP16Op):
-    def init_case(self):
-        self.x_shape = (3, 5, 5, 6)
-        self.y_shape = (5, 5, 6)
-        self.p = 1.0
-
-
-class TestDistFP16OpCase2(TestDistFP16Op):
-    def init_case(self):
-        self.x_shape = (10, 10)
-        self.y_shape = (4, 10, 10)
-        self.p = 2.0
-
-
-class TestDistFP16OpCase3(TestDistFP16Op):
-    def init_case(self):
-        self.x_shape = (15, 10)
-        self.y_shape = (15, 10)
-        self.p = float("inf")
-
-
-class TestDistFP16OpCase4(TestDistFP16Op):
-    def init_case(self):
-        self.x_shape = (2, 3, 4, 5, 8)
-        self.y_shape = (3, 1, 5, 8)
-        self.p = float("-inf")
-
-
-class TestDistFP16OpCase5(TestDistFP16Op):
-    def init_case(self):
-        self.x_shape = (4, 1, 4, 8)
-        self.y_shape = (2, 2, 1, 4, 4, 8)
-        self.p = 1.5
-
-
 class TestDistAPI(unittest.TestCase):
     def init_data_type(self):
         self.data_type = (
diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py
index 57550c86368a2..2235cf93cfb60 100644
--- a/python/paddle/tensor/linalg.py
+++ b/python/paddle/tensor/linalg.py
@@ -675,8 +675,8 @@ def dist(x, y, p=2, name=None):
         ||z||_{p}=(\sum_{i=1}^{m}|z_i|^p)^{\\frac{1}{p}}
 
     Args:
-        x (Tensor): 1-D to 6-D Tensor, its data type is float16, float32 or float64.
-        y (Tensor): 1-D to 6-D Tensor, its data type is float16, float32 or float64.
+        x (Tensor): 1-D to 6-D Tensor, its data type is float32 or float64.
+        y (Tensor): 1-D to 6-D Tensor, its data type is float32 or float64.
         p (float, optional): The norm to be computed, its data type is float32 or float64. Default: 2.
         name (str, optional): The default value is `None`. Normally there is no need for
             user to set this property. For more information, please refer to :ref:`api_guide_Name`.
@@ -706,12 +706,8 @@ def dist(x, y, p=2, name=None):
     if in_dygraph_mode():
         return _C_ops.dist(x, y, p)
 
-    check_variable_and_dtype(
-        x, 'dtype', ['float16', 'float32', 'float64'], 'dist'
-    )
-    check_variable_and_dtype(
-        y, 'dtype', ['float16', 'float32', 'float64'], 'dist'
-    )
+    check_variable_and_dtype(x, 'dtype', ['float32', 'float64'], 'dist')
+    check_variable_and_dtype(y, 'dtype', ['float32', 'float64'], 'dist')
     check_type(p, 'p', (float, int), 'dist')
     helper = LayerHelper("dist", **locals())
     out = helper.create_variable_for_type_inference(x.dtype)

From 13e2e10cfddccc75fc483698f72a5cb78ea51111 Mon Sep 17 00:00:00 2001
From: zhangyuqin1998 <75946871+zhangyuqin1998@users.noreply.github.com>
Date: Sat, 6 May 2023 10:04:01 +0800
Subject: [PATCH 178/405] move UniformRawKernel to legacy (#53158)

* move UniformRawKernel to legacy

* Update uniform_kernel.cc

* Update uniform_kernel.cu

* Update uniform_kernel.cc

* Update uniform_kernel.cu

* Update uniform_kernel.h

* Update uniform_kernel.cc

* Empty Commit to setup deployments
---
 paddle/phi/kernels/CMakeLists.txt             |  1 +
 paddle/phi/kernels/cpu/uniform_kernel.cc      | 38 ++------
 paddle/phi/kernels/gpu/uniform_kernel.cu      | 27 +++---
 .../phi/kernels/legacy/cpu/uniform_kernel.cc  | 70 ++++++++++++++
 .../phi/kernels/legacy/gpu/uniform_kernel.cu  | 96 +++++++++++++++++++
 paddle/phi/kernels/legacy/uniform_kernel.h    | 36 +++++++
 .../phi/kernels/legacy/xpu/uniform_kernel.cc  | 80 ++++++++++++++++
 .../kernels/selected_rows/uniform_kernel.cc   |  1 +
 paddle/phi/kernels/uniform_kernel.cc          | 65 -------------
 paddle/phi/kernels/uniform_kernel.h           | 12 ---
 paddle/phi/kernels/xpu/uniform_kernel.cc      | 23 +++--
 11 files changed, 316 insertions(+), 133 deletions(-)
 create mode 100644 paddle/phi/kernels/legacy/cpu/uniform_kernel.cc
 create mode 100644 paddle/phi/kernels/legacy/gpu/uniform_kernel.cu
 create mode 100644 paddle/phi/kernels/legacy/uniform_kernel.h
 create mode 100644 paddle/phi/kernels/legacy/xpu/uniform_kernel.cc
 delete mode 100644 paddle/phi/kernels/uniform_kernel.cc

diff --git a/paddle/phi/kernels/CMakeLists.txt b/paddle/phi/kernels/CMakeLists.txt
index de67958d5fe91..9b78cbf424618 100644
--- a/paddle/phi/kernels/CMakeLists.txt
+++ b/paddle/phi/kernels/CMakeLists.txt
@@ -112,6 +112,7 @@ file(
   "gpudnn/*.cu"
   "kps/*.cu"
   "legacy/kps/*.cu"
+  "legacy/gpu/*.cu"
   "selected_rows/gpu/*.cu"
   "sparse/gpu/*.cu"
   "strings/gpu/*.cu"
diff --git a/paddle/phi/kernels/cpu/uniform_kernel.cc b/paddle/phi/kernels/cpu/uniform_kernel.cc
index 1b1503473d968..17ee545918824 100644
--- a/paddle/phi/kernels/cpu/uniform_kernel.cc
+++ b/paddle/phi/kernels/cpu/uniform_kernel.cc
@@ -20,16 +20,13 @@
 namespace phi {
 
 template <typename T, typename Context>
-void UniformRawKernel(const Context &dev_ctx,
-                      const IntArray &shape,
-                      DataType dtype,
-                      const Scalar &min,
-                      const Scalar &max,
-                      int seed,
-                      int diag_num,
-                      int diag_step,
-                      float diag_val,
-                      DenseTensor *out) {
+void UniformKernel(const Context &dev_ctx,
+                   const IntArray &shape,
+                   DataType dtype,
+                   const Scalar &min,
+                   const Scalar &max,
+                   int seed,
+                   DenseTensor *out) {
   out->Resize(phi::make_ddim(shape.GetData()));
   T *data = dev_ctx.template Alloc<T>(out);
   auto size = out->numel();
@@ -42,31 +39,14 @@ void UniformRawKernel(const Context &dev_ctx,
   }
   UniformRealDistribution<T>(
       data, size, min.to<float>(), max.to<float>(), engine);
-  if (diag_num > 0) {
-    PADDLE_ENFORCE_GT(
-        size,
-        (diag_num - 1) * (diag_step + 1),
-        phi::errors::InvalidArgument(
-            "ShapeInvalid: the diagonal's elements is equal (num-1) "
-            "* (step-1) with num %d, step %d,"
-            "It should be smaller than %d, but received %d",
-            diag_num,
-            diag_step,
-            (diag_num - 1) * (diag_step + 1),
-            size));
-    for (int64_t i = 0; i < diag_num; ++i) {
-      int64_t pos = i * diag_step + i;
-      data[pos] = diag_val;
-    }
-  }
 }
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(uniform_raw,
+PD_REGISTER_KERNEL(uniform,
                    CPU,
                    ALL_LAYOUT,
-                   phi::UniformRawKernel,
+                   phi::UniformKernel,
                    float,
                    double,
                    phi::dtype::bfloat16) {}
diff --git a/paddle/phi/kernels/gpu/uniform_kernel.cu b/paddle/phi/kernels/gpu/uniform_kernel.cu
index fe36fe5fc6e1a..1ba5847fa2932 100644
--- a/paddle/phi/kernels/gpu/uniform_kernel.cu
+++ b/paddle/phi/kernels/gpu/uniform_kernel.cu
@@ -54,16 +54,13 @@ struct UniformGenerator {
 };
 
 template <typename T, typename Context>
-void UniformRawKernel(const Context& dev_ctx,
-                      const IntArray& shape,
-                      DataType dtype,
-                      const Scalar& min,
-                      const Scalar& max,
-                      int seed,
-                      int diag_num,
-                      int diag_step,
-                      float diag_val,
-                      DenseTensor* out) {
+void UniformKernel(const Context& dev_ctx,
+                   const IntArray& shape,
+                   DataType dtype,
+                   const Scalar& min,
+                   const Scalar& max,
+                   int seed,
+                   DenseTensor* out) {
   out->Resize(phi::make_ddim(shape.GetData()));
   dev_ctx.template Alloc<T>(out);
   if (seed == 0) {
@@ -77,19 +74,19 @@ void UniformRawKernel(const Context& dev_ctx,
     auto func = UniformGenerator<T>(static_cast<T>(min.to<float>()),
                                     static_cast<T>(max.to<float>()),
                                     seed,
-                                    diag_num,
-                                    diag_step,
-                                    static_cast<T>(diag_val));
+                                    0,
+                                    0,
+                                    static_cast<T>(0.0));
     IndexKernel<T, UniformGenerator<T>>(dev_ctx, out, func);
   }
 }
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(uniform_raw,
+PD_REGISTER_KERNEL(uniform,
                    GPU,
                    ALL_LAYOUT,
-                   phi::UniformRawKernel,
+                   phi::UniformKernel,
                    float,
                    double,
                    phi::dtype::float16,
diff --git a/paddle/phi/kernels/legacy/cpu/uniform_kernel.cc b/paddle/phi/kernels/legacy/cpu/uniform_kernel.cc
new file mode 100644
index 0000000000000..ecea86874a7a2
--- /dev/null
+++ b/paddle/phi/kernels/legacy/cpu/uniform_kernel.cc
@@ -0,0 +1,70 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/uniform_real_distribution.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void UniformRawKernel(const Context &dev_ctx,
+                      const IntArray &shape,
+                      DataType dtype,
+                      const Scalar &min,
+                      const Scalar &max,
+                      int seed,
+                      int diag_num,
+                      int diag_step,
+                      float diag_val,
+                      DenseTensor *out) {
+  out->Resize(phi::make_ddim(shape.GetData()));
+  T *data = dev_ctx.template Alloc<T>(out);
+  auto size = out->numel();
+  std::shared_ptr<std::mt19937_64> engine;
+  if (seed) {
+    engine = std::make_shared<std::mt19937_64>();
+    engine->seed(seed);
+  } else {
+    engine = dev_ctx.GetGenerator()->GetCPUEngine();
+  }
+  UniformRealDistribution<T>(
+      data, size, min.to<float>(), max.to<float>(), engine);
+  if (diag_num > 0) {
+    PADDLE_ENFORCE_GT(
+        size,
+        (diag_num - 1) * (diag_step + 1),
+        phi::errors::InvalidArgument(
+            "ShapeInvalid: the diagonal's elements is equal (num-1) "
+            "* (step-1) with num %d, step %d,"
+            "It should be smaller than %d, but received %d",
+            diag_num,
+            diag_step,
+            (diag_num - 1) * (diag_step + 1),
+            size));
+    for (int64_t i = 0; i < diag_num; ++i) {
+      int64_t pos = i * diag_step + i;
+      data[pos] = diag_val;
+    }
+  }
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(uniform_raw,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::UniformRawKernel,
+                   float,
+                   double,
+                   phi::dtype::bfloat16) {}
diff --git a/paddle/phi/kernels/legacy/gpu/uniform_kernel.cu b/paddle/phi/kernels/legacy/gpu/uniform_kernel.cu
new file mode 100644
index 0000000000000..211c7accf6fc4
--- /dev/null
+++ b/paddle/phi/kernels/legacy/gpu/uniform_kernel.cu
@@ -0,0 +1,96 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/uniform_kernel.h"
+
+#include <thrust/random.h>
+
+#include "gflags/gflags.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/distribution_helper.h"
+#include "paddle/phi/kernels/funcs/index_impl.cu.h"
+
+namespace phi {
+
+template <typename T>
+struct UniformGenerator {
+  T min_, max_;
+  unsigned int seed_;
+  T diag_val_;
+  unsigned int diag_num_;
+  unsigned int diag_step_;
+  __host__ __device__ UniformGenerator(
+      T min, T max, int seed, int diag_num, int diag_step, T diag_val)
+      : min_(min),
+        max_(max),
+        seed_(seed),
+        diag_num_(diag_num),
+        diag_step_(diag_step),
+        diag_val_(diag_val) {}
+
+  __host__ __device__ T operator()(const unsigned int n) const {
+    thrust::minstd_rand rng;
+    rng.seed(seed_);
+    thrust::uniform_real_distribution<T> dist(min_, max_);
+    rng.discard(n);
+    T out = dist(rng);
+    unsigned int remainder = n % (diag_step_ + 1);
+    if (remainder == 0 && diag_num_ > n / (diag_step_ + 1)) {
+      out = diag_val_;
+    }
+    return out;
+  }
+};
+
+template <typename T, typename Context>
+void UniformRawKernel(const Context& dev_ctx,
+                      const IntArray& shape,
+                      DataType dtype,
+                      const Scalar& min,
+                      const Scalar& max,
+                      int seed,
+                      int diag_num,
+                      int diag_step,
+                      float diag_val,
+                      DenseTensor* out) {
+  out->Resize(phi::make_ddim(shape.GetData()));
+  dev_ctx.template Alloc<T>(out);
+  if (seed == 0) {
+    // Use global Generator seed
+    using MT = typename kps::details::MPTypeTrait<T>::Type;
+    funcs::uniform_distribution<MT> dist;
+    funcs::uniform_real_transform<MT> trans(min.to<float>(), max.to<float>());
+    funcs::distribution_and_transform<T>(dev_ctx, out, dist, trans);
+  } else {
+    // Use OP seed
+    auto func = UniformGenerator<T>(static_cast<T>(min.to<float>()),
+                                    static_cast<T>(max.to<float>()),
+                                    seed,
+                                    diag_num,
+                                    diag_step,
+                                    static_cast<T>(diag_val));
+    IndexKernel<T, UniformGenerator<T>>(dev_ctx, out, func);
+  }
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(uniform_raw,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::UniformRawKernel,
+                   float,
+                   double,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/paddle/phi/kernels/legacy/uniform_kernel.h b/paddle/phi/kernels/legacy/uniform_kernel.h
new file mode 100644
index 0000000000000..5c3b9966fc906
--- /dev/null
+++ b/paddle/phi/kernels/legacy/uniform_kernel.h
@@ -0,0 +1,36 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/common/int_array.h"
+#include "paddle/phi/common/scalar.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/device_context.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void UniformRawKernel(const Context& dev_ctx,
+                      const IntArray& shape,
+                      DataType dtype,
+                      const Scalar& min,
+                      const Scalar& max,
+                      int seed,
+                      int diag_num,
+                      int diag_step,
+                      float diag_val,
+                      DenseTensor* out);
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/legacy/xpu/uniform_kernel.cc b/paddle/phi/kernels/legacy/xpu/uniform_kernel.cc
new file mode 100644
index 0000000000000..f1907b13e5f96
--- /dev/null
+++ b/paddle/phi/kernels/legacy/xpu/uniform_kernel.cc
@@ -0,0 +1,80 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/uniform_kernel.h"
+
+#include <string>
+
+#include "paddle/phi/backends/xpu/xpu_context.h"
+#include "paddle/phi/common/memory_utils.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/uniform_real_distribution.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void UniformRawKernel(const Context &dev_ctx,
+                      const IntArray &shape,
+                      DataType dtype,
+                      const Scalar &min,
+                      const Scalar &max,
+                      int seed,
+                      int diag_num,
+                      int diag_step,
+                      float diag_val,
+                      DenseTensor *out) {
+  out->Resize(phi::make_ddim(shape.GetData()));
+  T *data = dev_ctx.template Alloc<T>(out);
+  int64_t size = out->numel();
+
+  std::unique_ptr<T[]> data_cpu(new T[size]);
+
+  std::shared_ptr<std::mt19937_64> engine;
+  if (seed) {
+    engine = std::make_shared<std::mt19937_64>();
+    engine->seed(seed);
+  } else {
+    engine = dev_ctx.GetGenerator()->GetCPUEngine();
+  }
+  UniformRealDistribution<T>(
+      data_cpu.get(), size, min.to<float>(), max.to<float>(), engine);
+  if (diag_num > 0) {
+    PADDLE_ENFORCE_GT(
+        size,
+        (diag_num - 1) * (diag_step + 1),
+        phi::errors::InvalidArgument(
+            "ShapeInvalid: the diagonal's elements is equal (num-1) "
+            "* (step-1) with num %d, step %d,"
+            "It should be smaller than %d, but received %d",
+            diag_num,
+            diag_step,
+            (diag_num - 1) * (diag_step + 1),
+            size));
+    for (int64_t i = 0; i < diag_num; ++i) {
+      int64_t pos = i * diag_step + i;
+      data_cpu[pos] = diag_val;
+    }
+  }
+
+  memory_utils::Copy(dev_ctx.GetPlace(),
+                     data,
+                     phi::CPUPlace(),
+                     reinterpret_cast<void *>(data_cpu.get()),
+                     size * sizeof(T));
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(uniform_raw, XPU, ALL_LAYOUT, phi::UniformRawKernel, float) {
+}
diff --git a/paddle/phi/kernels/selected_rows/uniform_kernel.cc b/paddle/phi/kernels/selected_rows/uniform_kernel.cc
index 89707018002a6..0af5d8788c71f 100644
--- a/paddle/phi/kernels/selected_rows/uniform_kernel.cc
+++ b/paddle/phi/kernels/selected_rows/uniform_kernel.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "paddle/phi/kernels/selected_rows/uniform_kernel.h"
+#include "paddle/phi/kernels/legacy/uniform_kernel.h"
 
 #include "paddle/phi/backends/cpu/cpu_context.h"
 #include "paddle/phi/backends/gpu/gpu_context.h"
diff --git a/paddle/phi/kernels/uniform_kernel.cc b/paddle/phi/kernels/uniform_kernel.cc
deleted file mode 100644
index 7e8138f6e1d63..0000000000000
--- a/paddle/phi/kernels/uniform_kernel.cc
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/kernels/uniform_kernel.h"
-
-#include "paddle/phi/common/int_array.h"
-#include "paddle/phi/common/scalar.h"
-#include "paddle/phi/core/dense_tensor.h"
-#include "paddle/phi/core/device_context.h"
-#include "paddle/phi/core/kernel_registry.h"
-#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-#include "paddle/phi/backends/gpu/gpu_context.h"
-#endif
-#ifdef PADDLE_WITH_XPU
-#include "paddle/phi/backends/xpu/xpu_context.h"
-#endif
-
-namespace phi {
-
-template <typename T, typename Context>
-void UniformKernel(const Context& dev_ctx,
-                   const IntArray& shape,
-                   DataType dtype,
-                   const Scalar& min,
-                   const Scalar& max,
-                   int seed,
-                   DenseTensor* out) {
-  UniformRawKernel<T>(dev_ctx, shape, dtype, min, max, seed, 0, 0, 0.0f, out);
-}
-
-}  // namespace phi
-
-PD_REGISTER_KERNEL(uniform,
-                   CPU,
-                   ALL_LAYOUT,
-                   phi::UniformKernel,
-                   float,
-                   double,
-                   phi::dtype::bfloat16) {}
-
-#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-PD_REGISTER_KERNEL(uniform,
-                   GPU,
-                   ALL_LAYOUT,
-                   phi::UniformKernel,
-                   float,
-                   double,
-                   phi::dtype::float16,
-                   phi::dtype::bfloat16) {}
-#endif
-
-#ifdef PADDLE_WITH_XPU
-PD_REGISTER_KERNEL(uniform, XPU, ALL_LAYOUT, phi::UniformKernel, float) {}
-#endif
diff --git a/paddle/phi/kernels/uniform_kernel.h b/paddle/phi/kernels/uniform_kernel.h
index ef19c20b93284..1d0160e4056c8 100644
--- a/paddle/phi/kernels/uniform_kernel.h
+++ b/paddle/phi/kernels/uniform_kernel.h
@@ -21,18 +21,6 @@
 
 namespace phi {
 
-template <typename T, typename Context>
-void UniformRawKernel(const Context& dev_ctx,
-                      const IntArray& shape,
-                      DataType dtype,
-                      const Scalar& min,
-                      const Scalar& max,
-                      int seed,
-                      int diag_num,
-                      int diag_step,
-                      float diag_val,
-                      DenseTensor* out);
-
 template <typename T, typename Context>
 void UniformKernel(const Context& dev_ctx,
                    const IntArray& shape,
diff --git a/paddle/phi/kernels/xpu/uniform_kernel.cc b/paddle/phi/kernels/xpu/uniform_kernel.cc
index 48f9a6e8d779f..99388e31e5881 100644
--- a/paddle/phi/kernels/xpu/uniform_kernel.cc
+++ b/paddle/phi/kernels/xpu/uniform_kernel.cc
@@ -24,16 +24,16 @@ limitations under the License. */
 namespace phi {
 
 template <typename T, typename Context>
-void UniformRawKernel(const Context &dev_ctx,
-                      const IntArray &shape,
-                      DataType dtype,
-                      const Scalar &min,
-                      const Scalar &max,
-                      int seed,
-                      int diag_num,
-                      int diag_step,
-                      float diag_val,
-                      DenseTensor *out) {
+void UniformKernel(const Context &dev_ctx,
+                   const IntArray &shape,
+                   DataType dtype,
+                   const Scalar &min,
+                   const Scalar &max,
+                   int seed,
+                   DenseTensor *out) {
+  int diag_num = 0;
+  int diag_step = 0;
+  float diag_val = 0.0f;
   out->Resize(phi::make_ddim(shape.GetData()));
   T *data = dev_ctx.template Alloc<T>(out);
   int64_t size = out->numel();
@@ -76,5 +76,4 @@ void UniformRawKernel(const Context &dev_ctx,
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(uniform_raw, XPU, ALL_LAYOUT, phi::UniformRawKernel, float) {
-}
+PD_REGISTER_KERNEL(uniform, XPU, ALL_LAYOUT, phi::UniformKernel, float) {}

From a49973119806cc6804c8558cac2036cfaf5d0a88 Mon Sep 17 00:00:00 2001
From: Kim  Yann <baochen.yang@intel.com>
Date: Sat, 6 May 2023 10:12:23 +0800
Subject: [PATCH 179/405] rem npu in test (#53469)

* rem npu in test

* restore some code
---
 test/CMakeLists.txt                          | 26 +++++---------
 test/cpp/inference/api/api_impl_tester.cc    |  4 ---
 test/cpp/inference/test_helper.h             |  3 --
 test/cpp/phi/common/test_backend.cc          |  4 ---
 test/cpp/phi/core/test_dense_tensor.cc       | 37 +-------------------
 test/xpu/test_merged_momentum_op_xpu_base.py |  1 -
 6 files changed, 9 insertions(+), 66 deletions(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 567d9b1e6c32f..258ea9025dd55 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -186,24 +186,14 @@ if(${len} GREATER_EQUAL 1)
       if(WITH_XPU)
         target_link_libraries(${test_name} xpulib)
       endif()
-      if(NOT
-         ("${test_name}" STREQUAL "c_broadcast_op_npu_test"
-          OR "${test_name}" STREQUAL "c_allreduce_sum_op_npu_test"
-          OR "${test_name}" STREQUAL "c_allreduce_max_op_npu_test"
-          OR "${test_name}" STREQUAL "c_reducescatter_op_npu_test"
-          OR "${test_name}" STREQUAL "c_allgather_op_npu_test"
-          OR "${test_name}" STREQUAL "send_v2_op_npu_test"
-          OR "${test_name}" STREQUAL "c_reduce_sum_op_npu_test"
-          OR "${test_name}" STREQUAL "recv_v2_op_npu_test"))
-        cc_test_run(
-          ${test_name}
-          COMMAND
-          ${test_name}
-          ARGS
-          ${test_arg}
-          DIR
-          ${CC_TESTS_DIR})
-      endif()
+      cc_test_run(
+        ${test_name}
+        COMMAND
+        ${test_name}
+        ARGS
+        ${test_arg}
+        DIR
+        ${CC_TESTS_DIR})
     elseif(WITH_TESTING AND NOT TEST ${test_name})
       add_test(NAME ${test_name} COMMAND ${CMAKE_COMMAND} -E echo CI skip
                                          ${test_name}.)
diff --git a/test/cpp/inference/api/api_impl_tester.cc b/test/cpp/inference/api/api_impl_tester.cc
index 3270e216586e1..d3d289371359d 100644
--- a/test/cpp/inference/api/api_impl_tester.cc
+++ b/test/cpp/inference/api/api_impl_tester.cc
@@ -72,7 +72,6 @@ void MainWord2Vec(const paddle::PaddlePlace& place) {
   auto predictor = CreatePaddlePredictor<NativeConfig>(config);
   config.use_gpu = paddle::gpu_place_used(place);
   config.use_xpu = paddle::xpu_place_used(place);
-  config.use_npu = paddle::npu_place_used(place);
 
   phi::DenseTensor first_word, second_word, third_word, fourth_word;
   framework::LoD lod{{0, 1}};
@@ -125,7 +124,6 @@ void MainImageClassification(const paddle::PaddlePlace& place) {
   NativeConfig config = GetConfig();
   config.use_gpu = paddle::gpu_place_used(place);
   config.use_xpu = paddle::xpu_place_used(place);
-  config.use_npu = paddle::npu_place_used(place);
   config.model_dir =
       FLAGS_book_dirname + "/image_classification_resnet.inference.model";
 
@@ -169,7 +167,6 @@ void MainThreadsWord2Vec(const paddle::PaddlePlace& place) {
   NativeConfig config = GetConfig();
   config.use_gpu = paddle::gpu_place_used(place);
   config.use_xpu = paddle::xpu_place_used(place);
-  config.use_npu = paddle::npu_place_used(place);
   auto main_predictor = CreatePaddlePredictor<NativeConfig>(config);
 
   // prepare inputs data and reference results
@@ -234,7 +231,6 @@ void MainThreadsImageClassification(const paddle::PaddlePlace& place) {
   NativeConfig config = GetConfig();
   config.use_gpu = paddle::gpu_place_used(place);
   config.use_xpu = paddle::xpu_place_used(place);
-  config.use_npu = paddle::npu_place_used(place);
   config.model_dir =
       FLAGS_book_dirname + "/image_classification_resnet.inference.model";
 
diff --git a/test/cpp/inference/test_helper.h b/test/cpp/inference/test_helper.h
index 0685b90de1fa2..f66712401858a 100644
--- a/test/cpp/inference/test_helper.h
+++ b/test/cpp/inference/test_helper.h
@@ -35,9 +35,6 @@ bool gpu_place_used(const paddle::PaddlePlace& place) {
 bool xpu_place_used(const paddle::PaddlePlace& place) {
   return place == paddle::PaddlePlace::kXPU;
 }
-bool npu_place_used(const paddle::PaddlePlace& place) {
-  return place == paddle::PaddlePlace::kNPU;
-}
 bool cpu_place_used(const paddle::PaddlePlace& place) {
   return place == paddle::PaddlePlace::kCPU;
 }
diff --git a/test/cpp/phi/common/test_backend.cc b/test/cpp/phi/common/test_backend.cc
index 791167ffe62a6..516deeee34af2 100644
--- a/test/cpp/phi/common/test_backend.cc
+++ b/test/cpp/phi/common/test_backend.cc
@@ -36,9 +36,6 @@ TEST(Backend, OStream) {
   oss << phi::Backend::XPU;
   EXPECT_EQ(oss.str(), "XPU");
   oss.str("");
-  oss << phi::Backend::NPU;
-  EXPECT_EQ(oss.str(), "NPU");
-  oss.str("");
   oss << phi::Backend::ONEDNN;
   EXPECT_EQ(oss.str(), "ONEDNN");
   oss.str("");
@@ -62,7 +59,6 @@ TEST(Backend, StringToBackend) {
   EXPECT_EQ(phi::Backend::CPU, pexp::StringToBackend("CPU"));
   EXPECT_EQ(phi::Backend::GPU, pexp::StringToBackend("GPU"));
   EXPECT_EQ(phi::Backend::XPU, pexp::StringToBackend("XPU"));
-  EXPECT_EQ(phi::Backend::NPU, pexp::StringToBackend("NPU"));
   EXPECT_EQ(phi::Backend::ONEDNN, pexp::StringToBackend("OneDNN"));
   EXPECT_EQ(phi::Backend::GPUDNN, pexp::StringToBackend("GPUDNN"));
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
diff --git a/test/cpp/phi/core/test_dense_tensor.cc b/test/cpp/phi/core/test_dense_tensor.cc
index b0c02441ef1e2..424c4bca75f40 100644
--- a/test/cpp/phi/core/test_dense_tensor.cc
+++ b/test/cpp/phi/core/test_dense_tensor.cc
@@ -129,13 +129,6 @@ TEST(dense_tensor, shallow_copy) {
   CHECK(tensor_0.meta() == tensor_1.meta());
 }
 
-struct TestStorageProperties
-    : public StorageProperties,
-      public TypeInfoTraits<StorageProperties, NPUStorageProperties> {
-  virtual ~TestStorageProperties() = default;
-  static const char* name() { return "TestStorageProperties"; }
-};
-
 TEST(dense_tensor, storage_properties) {
   const DataType dtype{DataType::FLOAT32};
   const DDim dims({1, 2});
@@ -144,29 +137,9 @@ TEST(dense_tensor, storage_properties) {
   auto fancy_allocator = std::unique_ptr<Allocator>(new FancyAllocator);
   DenseTensor tensor(fancy_allocator.get(), meta);
 
-  // test no storage properties
-  bool caught_exception = false;
-  try {
-    tensor.storage_properties<NPUStorageProperties>();
-  } catch (phi::enforce::EnforceNotMet& error) {
-    caught_exception = true;
-  }
-  EXPECT_TRUE(caught_exception);
-
-  // test custom device storage properties
-  EXPECT_FALSE(tensor.storage_properties_initialized());
-  auto npu_properties = std::make_unique<NPUStorageProperties>();
-  npu_properties->storage_format = 3;
-  npu_properties->storage_dims = {1, 1, 1, 1, 16};
-  tensor.set_storage_properties(std::move(npu_properties));
-  EXPECT_TRUE(tensor.storage_properties_initialized());
-  auto get_npu_properties = tensor.storage_properties<NPUStorageProperties>();
-  CHECK_EQ(get_npu_properties.storage_format, 3);
-  CHECK_EQ(get_npu_properties.storage_dims.size(), 5);
-
   // test error type storage properties
 #ifdef PADDLE_WITH_MKLDNN
-  caught_exception = false;
+  bool caught_exception = false;
   try {
     tensor.storage_properties<OneDNNStorageProperties>();
   } catch (phi::enforce::EnforceNotMet& error) {
@@ -174,14 +147,6 @@ TEST(dense_tensor, storage_properties) {
   }
   EXPECT_TRUE(caught_exception);
 #endif
-
-  // test copy storage properties
-  auto cp_tensor = tensor;
-  auto get_cp_npu_properties =
-      cp_tensor.storage_properties<NPUStorageProperties>();
-  CHECK_EQ(get_cp_npu_properties.storage_format, 3);
-  CHECK_EQ(get_cp_npu_properties.storage_dims.size(), 5);
 }
-
 }  // namespace tests
 }  // namespace phi
diff --git a/test/xpu/test_merged_momentum_op_xpu_base.py b/test/xpu/test_merged_momentum_op_xpu_base.py
index 10868585a903f..62f534a1b41a6 100644
--- a/test/xpu/test_merged_momentum_op_xpu_base.py
+++ b/test/xpu/test_merged_momentum_op_xpu_base.py
@@ -211,7 +211,6 @@ def check_with_place(self, place, dtype, multi_precision=False):
         )
 
         def run_op(use_nesterov, use_merged):
-            # NPU Momentum Op does not support rescale_grad
             rescale_grad = 1.0
             return run_momentum_op(
                 params,

From 5a44bf7e39e5fa6e6618e3baad12100522424aaf Mon Sep 17 00:00:00 2001
From: Wilber <jiweibo@baidu.com>
Date: Sat, 6 May 2023 10:39:57 +0800
Subject: [PATCH 180/405] Add trt pow converter. (#53462)

* Add trt pow converter.

* update to use AddConstantLayer

* add dims=0 ut
---
 .../tensorrt/convert/elementwise_op.cc        |  33 +++++
 paddle/fluid/inference/tensorrt/op_teller.cc  |  26 ++++
 .../inference/test_trt_convert_elementwise.py | 122 ++++++++++++++++++
 3 files changed, 181 insertions(+)

diff --git a/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc b/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc
index c1625c51688c0..7e8f50d0de088 100644
--- a/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc
+++ b/paddle/fluid/inference/tensorrt/convert/elementwise_op.cc
@@ -319,6 +319,37 @@ class ElementwiseTensorModOpConverter : public ElementwiseTensorOpConverter {
  public:
   ElementwiseTensorModOpConverter() { op_type_ = "mod"; }
 };
+
+// The diff between `pow` and `elementwise_pow` is in:
+// https://github.com/PaddlePaddle/Paddle/blob/release/2.4/python/paddle/tensor/math.py#L420
+class PowOpConverter : public OpConverter {
+ public:
+  PowOpConverter() {}
+  void operator()(const framework::proto::OpDesc& op,
+                  const framework::Scope& scope,
+                  bool test_mode) override {
+    VLOG(3) << "Convert a pow op to TensorRT IElementWiseLayer";
+    framework::OpDesc op_desc(op, nullptr);
+    auto* X = engine_->GetITensor(op_desc.Input("X").front());
+    float factor = PADDLE_GET_CONST(float, op_desc.GetAttr("factor"));
+    nvinfer1::Dims dims_x = X->getDimensions();
+    auto output_name = op_desc.Output("Out")[0];
+
+    nvinfer1::Dims trt_dims_y;
+    trt_dims_y.nbDims = dims_x.nbDims;
+    for (int i = 0; i < trt_dims_y.nbDims; i++) {
+      trt_dims_y.d[i] = 1;
+    }
+
+    std::vector<float> w_data{factor};
+    auto* Y = AddConstantLayer(w_data.data(), trt_dims_y);
+
+    auto* layer = TRT_ENGINE_ADD_LAYER(
+        engine_, ElementWise, *X, *Y, nvinfer1::ElementWiseOperation::kPOW);
+    RreplenishLayerAndOutput(layer, "elementwise", {output_name}, test_mode);
+  }
+};
+
 }  // namespace tensorrt
 }  // namespace inference
 }  // namespace paddle
@@ -369,3 +400,5 @@ REGISTER_TRT_OP_CONVERTER(logical_and, ElementwiseTensorLogicalAndOpConverter);
 REGISTER_TRT_OP_CONVERTER(less_equal, ElementwiseTensorLessEqualOpConverter);
 REGISTER_TRT_OP_CONVERTER(greater_equal,
                           ElementwiseTensorGreaterEqualOpConverter);
+
+REGISTER_TRT_OP_CONVERTER(pow, PowOpConverter);
diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index 6aff2fb3bf6ca..ccbb20d32a6fe 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -1498,6 +1498,31 @@ struct SimpleOpTypeSetTeller : public Teller {
       }
     }
 
+    if (op_type == "pow") {
+      auto* block = desc.Block();
+      if (block == nullptr) {
+        VLOG(3) << "The block desc is nullptr, we can't continue to analyze. "
+                   "Developers need to check whether block_desc is passed in "
+                   "the pass.";
+        return false;
+      }
+      auto* x_var_desc = block->FindVar(desc.Input("X")[0]);
+      const auto x_shape = x_var_desc->GetShape();
+      if (!with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
+        VLOG(3) << op_type
+                << " op does not support input's dim is 1 or 0 in tensorrt "
+                   "static shape mode.";
+        return false;
+      }
+      // the same as `elementwise_pow`.
+      if (x_var_desc->GetDataType() ==
+          paddle::framework::proto::VarType_Type::VarType_Type_INT32) {
+        VLOG(3) << "These operations (pow) do not support int32 "
+                   "datatype.";
+        return false;
+      }
+    }
+
     if (op_type == "stack") {
       if (!with_dynamic_shape) {
         VLOG(3)
@@ -2885,6 +2910,7 @@ struct SimpleOpTypeSetTeller : public Teller {
       "elementwise_mul",
       "elementwise_div",
       "elementwise_pow",
+      "pow",
       "elementwise_min",
       "elementwise_max",
       "elementwise_floordiv",
diff --git a/test/ir/inference/test_trt_convert_elementwise.py b/test/ir/inference/test_trt_convert_elementwise.py
index 417591c453694..a6faff0787be5 100644
--- a/test/ir/inference/test_trt_convert_elementwise.py
+++ b/test/ir/inference/test_trt_convert_elementwise.py
@@ -1092,5 +1092,127 @@ def test(self):
         self.run_test()
 
 
+class TrtConvertPowOp(TrtLayerAutoScanTest):
+    def is_program_valid(self, program_config: ProgramConfig) -> bool:
+        return True
+
+    def sample_program_configs(self):
+        def generate_input(shape):
+            if len(shape) == 0:
+                return np.random.random([]).astype(np.float32)
+            return np.random.random(shape).astype(np.float32)
+
+        for batch in [1, 4]:
+            for shape in [
+                [],
+                [32],
+                [batch, 32],
+                [batch, 32, 32],
+                [batch, 32, 16, 32],
+            ]:
+                for factor in [1.0, 2.0, -1.0, 0.5, -2]:
+                    self.dims = len(shape)
+                    dics = [{"factor": factor}]
+                    ops_config = [
+                        {
+                            "op_type": "pow",
+                            "op_inputs": {
+                                "X": ["input_data"],
+                            },
+                            "op_outputs": {"Out": ["output_data"]},
+                            "op_attrs": dics[0],
+                            "outputs_dtype": {"output_data": np.float32},
+                        }
+                    ]
+                    ops = self.generate_op_config(ops_config)
+
+                    program_config = ProgramConfig(
+                        ops=ops,
+                        weights={},
+                        inputs={
+                            "input_data": TensorConfig(
+                                data_gen=partial(generate_input, shape)
+                            ),
+                        },
+                        outputs=["output_data"],
+                    )
+
+                    yield program_config
+
+    def sample_predictor_configs(
+        self, program_config
+    ) -> (paddle_infer.Config, List[int], float):
+        def generate_dynamic_shape(attrs):
+            if self.dims == 0:
+                self.dynamic_shape.min_input_shape = {"input_data": []}
+                self.dynamic_shape.max_input_shape = {"input_data": []}
+                self.dynamic_shape.opt_input_shape = {"input_data": []}
+            elif self.dims == 1:
+                self.dynamic_shape.min_input_shape = {"input_data": [4]}
+                self.dynamic_shape.max_input_shape = {"input_data": [32]}
+                self.dynamic_shape.opt_input_shape = {"input_data": [16]}
+            elif self.dims == 2:
+                self.dynamic_shape.min_input_shape = {"input_data": [1, 32]}
+                self.dynamic_shape.max_input_shape = {"input_data": [4, 32]}
+                self.dynamic_shape.opt_input_shape = {"input_data": [2, 32]}
+            elif self.dims == 3:
+                self.dynamic_shape.min_input_shape = {"input_data": [1, 32, 4]}
+                self.dynamic_shape.max_input_shape = {"input_data": [4, 32, 32]}
+                self.dynamic_shape.opt_input_shape = {"input_data": [2, 32, 32]}
+            elif self.dims == 4:
+                self.dynamic_shape.min_input_shape = {
+                    "input_data": [1, 32, 4, 4]
+                }
+                self.dynamic_shape.max_input_shape = {
+                    "input_data": [4, 32, 32, 32]
+                }
+                self.dynamic_shape.opt_input_shape = {
+                    "input_data": [4, 32, 16, 32]
+                }
+
+        def clear_dynamic_shape():
+            self.dynamic_shape.max_input_shape = {}
+            self.dynamic_shape.min_input_shape = {}
+            self.dynamic_shape.opt_input_shape = {}
+
+        def generate_trt_nodes_num(attrs, dynamic_shape):
+            if (self.dims == 1 or self.dims == 0) and not dynamic_shape:
+                return 0, 3
+            return 1, 2
+
+        attrs = [
+            program_config.ops[i].attrs for i in range(len(program_config.ops))
+        ]
+
+        # for static_shape
+        clear_dynamic_shape()
+        self.trt_param.precision = paddle_infer.PrecisionType.Float32
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, False
+        ), (1e-5, 1e-5)
+        self.trt_param.precision = paddle_infer.PrecisionType.Half
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, False
+        ), (1e-3, 1e-3)
+
+        # for dynamic_shape
+        generate_dynamic_shape(attrs)
+        self.trt_param.precision = paddle_infer.PrecisionType.Float32
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), (1e-5, 1e-5)
+        self.trt_param.precision = paddle_infer.PrecisionType.Half
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), (1e-3, 1e-3)
+
+    def add_skip_trt_case(self):
+        pass
+
+    def test(self):
+        self.add_skip_trt_case()
+        self.run_test()
+
+
 if __name__ == "__main__":
     unittest.main()

From 3e7be9c973c87bf6a97abf0885abfc769a1bcc98 Mon Sep 17 00:00:00 2001
From: zhangyuqin1998 <75946871+zhangyuqin1998@users.noreply.github.com>
Date: Sat, 6 May 2023 10:44:24 +0800
Subject: [PATCH 181/405] Rename randint_raw and move it to legacy (#53157)

* Rename randint_raw and move it to legacy

* Update fetch_v2_op.cc

* Update randint_kernel.cc

* Update randint_kernel.cu

* Empty Commit to setup deployments
---
 paddle/phi/kernels/cpu/randint_kernel.cc      | 27 ++------
 paddle/phi/kernels/gpu/randint_kernel.cu      | 25 ++------
 .../phi/kernels/legacy/cpu/randint_kernel.cc  | 55 ++++++++++++++++
 .../phi/kernels/legacy/gpu/randint_kernel.cu  | 48 ++++++++++++++
 .../phi/kernels/legacy/xpu/randint_kernel.cc  | 64 +++++++++++++++++++
 paddle/phi/kernels/randint_kernel.h           |  9 ---
 paddle/phi/kernels/xpu/randint_kernel.cc      | 27 ++------
 paddle/phi/ops/compat/randint_sig.cc          |  6 +-
 8 files changed, 190 insertions(+), 71 deletions(-)
 create mode 100644 paddle/phi/kernels/legacy/cpu/randint_kernel.cc
 create mode 100644 paddle/phi/kernels/legacy/gpu/randint_kernel.cu
 create mode 100644 paddle/phi/kernels/legacy/xpu/randint_kernel.cc

diff --git a/paddle/phi/kernels/cpu/randint_kernel.cc b/paddle/phi/kernels/cpu/randint_kernel.cc
index 2c7433f5d2100..72f3b15b9346b 100644
--- a/paddle/phi/kernels/cpu/randint_kernel.cc
+++ b/paddle/phi/kernels/cpu/randint_kernel.cc
@@ -22,13 +22,13 @@
 namespace phi {
 
 template <typename T, typename Context>
-void RandintRawKernel(const Context& dev_ctx,
-                      int low,
-                      int high,
-                      const IntArray& shape,
-                      DataType dtype,
-                      int seed,
-                      DenseTensor* out) {
+void RandintKernel(const Context& dev_ctx,
+                   int low,
+                   int high,
+                   const IntArray& shape,
+                   DataType dtype,
+                   DenseTensor* out) {
+  int seed = 0;
   out->Resize(phi::make_ddim(shape.GetData()));
   T* data = dev_ctx.template Alloc<T>(out);
   auto numel = out->numel();
@@ -45,20 +45,7 @@ void RandintRawKernel(const Context& dev_ctx,
   }
 }
 
-template <typename T, typename Context>
-void RandintKernel(const Context& dev_ctx,
-                   int low,
-                   int high,
-                   const IntArray& shape,
-                   DataType dtype,
-                   DenseTensor* out) {
-  RandintRawKernel<T>(dev_ctx, low, high, shape, dtype, 0, out);
-}
-
 }  // namespace phi
 
-PD_REGISTER_KERNEL(
-    randint_raw, CPU, ALL_LAYOUT, phi::RandintRawKernel, int, int64_t) {}
-
 PD_REGISTER_KERNEL(randint, CPU, ALL_LAYOUT, phi::RandintKernel, int, int64_t) {
 }
diff --git a/paddle/phi/kernels/gpu/randint_kernel.cu b/paddle/phi/kernels/gpu/randint_kernel.cu
index 80deff47970a3..39a57a5a6e865 100644
--- a/paddle/phi/kernels/gpu/randint_kernel.cu
+++ b/paddle/phi/kernels/gpu/randint_kernel.cu
@@ -23,21 +23,6 @@
 
 namespace phi {
 
-template <typename T, typename Context>
-void RandintRawKernel(const Context& dev_ctx,
-                      int low,
-                      int high,
-                      const IntArray& shape,
-                      DataType dtype,
-                      int seed,
-                      DenseTensor* out) {
-  out->Resize(phi::make_ddim(shape.GetData()));
-  T* data = dev_ctx.template Alloc<T>(out);
-  funcs::uniform_distribution<uint32_t> dist;
-  funcs::uniform_int_transform<T, uint32_t> trans(low, high);
-  funcs::distribution_and_transform<T>(dev_ctx, out, dist, trans);
-}
-
 template <typename T, typename Context>
 void RandintKernel(const Context& dev_ctx,
                    int low,
@@ -45,13 +30,15 @@ void RandintKernel(const Context& dev_ctx,
                    const IntArray& shape,
                    DataType dtype,
                    DenseTensor* out) {
-  RandintRawKernel<T>(dev_ctx, low, high, shape, dtype, 0, out);
+  int seed = 0;
+  out->Resize(phi::make_ddim(shape.GetData()));
+  T* data = dev_ctx.template Alloc<T>(out);
+  funcs::uniform_distribution<uint32_t> dist;
+  funcs::uniform_int_transform<T, uint32_t> trans(low, high);
+  funcs::distribution_and_transform<T>(dev_ctx, out, dist, trans);
 }
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(
-    randint_raw, GPU, ALL_LAYOUT, phi::RandintRawKernel, int, int64_t) {}
-
 PD_REGISTER_KERNEL(randint, GPU, ALL_LAYOUT, phi::RandintKernel, int, int64_t) {
 }
diff --git a/paddle/phi/kernels/legacy/cpu/randint_kernel.cc b/paddle/phi/kernels/legacy/cpu/randint_kernel.cc
new file mode 100644
index 0000000000000..f246ef8faf742
--- /dev/null
+++ b/paddle/phi/kernels/legacy/cpu/randint_kernel.cc
@@ -0,0 +1,55 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <random>
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/common/int_array.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void RandintWithSeedKernel(const Context& dev_ctx,
+                           int low,
+                           int high,
+                           const IntArray& shape,
+                           DataType dtype,
+                           int seed,
+                           DenseTensor* out) {
+  out->Resize(phi::make_ddim(shape.GetData()));
+  T* data = dev_ctx.template Alloc<T>(out);
+  auto numel = out->numel();
+  std::shared_ptr<std::mt19937_64> engine;
+  if (seed) {
+    engine = std::make_shared<std::mt19937_64>();
+    engine->seed(seed);
+  } else {
+    engine = dev_ctx.GetGenerator()->GetCPUEngine();
+  }
+  std::uniform_int_distribution<T> dist(low, high - 1);
+  for (int64_t i = 0; i < numel; ++i) {
+    data[i] = dist(*engine);
+  }
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(randint_with_seed,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::RandintWithSeedKernel,
+                   int,
+                   int64_t) {}
diff --git a/paddle/phi/kernels/legacy/gpu/randint_kernel.cu b/paddle/phi/kernels/legacy/gpu/randint_kernel.cu
new file mode 100644
index 0000000000000..b4aa5e9d8c47a
--- /dev/null
+++ b/paddle/phi/kernels/legacy/gpu/randint_kernel.cu
@@ -0,0 +1,48 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <random>
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/common/int_array.h"
+#include "paddle/phi/common/memory_utils.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/distribution_helper.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void RandintWithSeedKernel(const Context& dev_ctx,
+                           int low,
+                           int high,
+                           const IntArray& shape,
+                           DataType dtype,
+                           int seed,
+                           DenseTensor* out) {
+  out->Resize(phi::make_ddim(shape.GetData()));
+  T* data = dev_ctx.template Alloc<T>(out);
+  funcs::uniform_distribution<uint32_t> dist;
+  funcs::uniform_int_transform<T, uint32_t> trans(low, high);
+  funcs::distribution_and_transform<T>(dev_ctx, out, dist, trans);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(randint_with_seed,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::RandintWithSeedKernel,
+                   int,
+                   int64_t) {}
diff --git a/paddle/phi/kernels/legacy/xpu/randint_kernel.cc b/paddle/phi/kernels/legacy/xpu/randint_kernel.cc
new file mode 100644
index 0000000000000..0349ad964c41a
--- /dev/null
+++ b/paddle/phi/kernels/legacy/xpu/randint_kernel.cc
@@ -0,0 +1,64 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <random>
+
+#include "paddle/phi/backends/xpu/enforce_xpu.h"
+#include "paddle/phi/common/int_array.h"
+#include "paddle/phi/common/memory_utils.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/generator.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void RandintWithSeedKernel(const Context& dev_ctx,
+                           int low,
+                           int high,
+                           const IntArray& shape,
+                           DataType dtype,
+                           int seed,
+                           DenseTensor* out) {
+  int64_t size = out->numel();
+  out->Resize(phi::make_ddim(shape.GetData()));
+  T* data = dev_ctx.template Alloc<T>(out);
+  auto numel = out->numel();
+  std::shared_ptr<std::mt19937_64> engine;
+  if (seed) {
+    engine = std::make_shared<std::mt19937_64>();
+    engine->seed(seed);
+  } else {
+    engine = dev_ctx.GetGenerator()->GetCPUEngine();
+  }
+  std::unique_ptr<T[]> data_cpu(new T[size]);
+  std::uniform_int_distribution<T> dist(low, high - 1);
+  for (int64_t i = 0; i < numel; ++i) {
+    data_cpu[i] = dist(*engine);
+  }
+  memory_utils::Copy(dev_ctx.GetPlace(),
+                     data,
+                     phi::CPUPlace(),
+                     reinterpret_cast<void*>(data_cpu.get()),
+                     size * sizeof(T));
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(randint_with_seed,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::RandintWithSeedKernel,
+                   int,
+                   int64_t) {}
diff --git a/paddle/phi/kernels/randint_kernel.h b/paddle/phi/kernels/randint_kernel.h
index 85d440e305635..0fe7db803b947 100644
--- a/paddle/phi/kernels/randint_kernel.h
+++ b/paddle/phi/kernels/randint_kernel.h
@@ -27,13 +27,4 @@ void RandintKernel(const Context& dev_ctx,
                    DataType dtype,
                    DenseTensor* out);
 
-template <typename T, typename Context>
-void RandintRawKernel(const Context& dev_ctx,
-                      int low,
-                      int high,
-                      const IntArray& shape,
-                      DataType dtype,
-                      int seed,
-                      DenseTensor* out);
-
 }  // namespace phi
diff --git a/paddle/phi/kernels/xpu/randint_kernel.cc b/paddle/phi/kernels/xpu/randint_kernel.cc
index ec17f9a20dfa0..ce86d7e77a9fd 100644
--- a/paddle/phi/kernels/xpu/randint_kernel.cc
+++ b/paddle/phi/kernels/xpu/randint_kernel.cc
@@ -24,13 +24,13 @@
 namespace phi {
 
 template <typename T, typename Context>
-void RandintRawKernel(const Context& dev_ctx,
-                      int low,
-                      int high,
-                      const IntArray& shape,
-                      DataType dtype,
-                      int seed,
-                      DenseTensor* out) {
+void RandintKernel(const Context& dev_ctx,
+                   int low,
+                   int high,
+                   const IntArray& shape,
+                   DataType dtype,
+                   DenseTensor* out) {
+  int seed = 0;
   int64_t size = out->numel();
   out->Resize(phi::make_ddim(shape.GetData()));
   T* data = dev_ctx.template Alloc<T>(out);
@@ -54,20 +54,7 @@ void RandintRawKernel(const Context& dev_ctx,
                      size * sizeof(T));
 }
 
-template <typename T, typename Context>
-void RandintKernel(const Context& dev_ctx,
-                   int low,
-                   int high,
-                   const IntArray& shape,
-                   DataType dtype,
-                   DenseTensor* out) {
-  RandintRawKernel<T>(dev_ctx, low, high, shape, dtype, 0, out);
-}
-
 }  // namespace phi
 
-PD_REGISTER_KERNEL(
-    randint_raw, XPU, ALL_LAYOUT, phi::RandintRawKernel, int, int64_t) {}
-
 PD_REGISTER_KERNEL(randint, XPU, ALL_LAYOUT, phi::RandintKernel, int, int64_t) {
 }
diff --git a/paddle/phi/ops/compat/randint_sig.cc b/paddle/phi/ops/compat/randint_sig.cc
index eb6da78a258bc..e075a07967701 100644
--- a/paddle/phi/ops/compat/randint_sig.cc
+++ b/paddle/phi/ops/compat/randint_sig.cc
@@ -21,7 +21,7 @@ KernelSignature RandintOpArgumentMapping(const ArgumentMappingContext& ctx) {
   if (seed) {
     if (ctx.InputSize("ShapeTensorList") > 0) {
       return KernelSignature(
-          "randint_raw",
+          "randint_with_seed",
           {},
           {"low", "high", "ShapeTensorList", "seed", "dtype"},
           {"Out"});
@@ -29,12 +29,12 @@ KernelSignature RandintOpArgumentMapping(const ArgumentMappingContext& ctx) {
       const auto& shape =
           paddle::any_cast<std::vector<int64_t>>(ctx.Attr("shape"));
       if (ctx.HasInput("ShapeTensor") && shape.empty()) {
-        return KernelSignature("randint_raw",
+        return KernelSignature("randint_with_seed",
                                {},
                                {"low", "high", "ShapeTensor", "seed", "dtype"},
                                {"Out"});
       } else {
-        return KernelSignature("randint_raw",
+        return KernelSignature("randint_with_seed",
                                {},
                                {"low", "high", "shape", "seed", "dtype"},
                                {"Out"});

From 12406cadb2c5ea6e72d963aff21da20d49392edf Mon Sep 17 00:00:00 2001
From: Zhang Jun <ewalker@live.cn>
Date: Sat, 6 May 2023 11:07:16 +0800
Subject: [PATCH 182/405] [inference][trt] add reduce_all and reduce_any
 (#53088)

---
 .../fluid/inference/api/analysis_predictor.cc |  2 +
 .../inference/tensorrt/convert/reduce_op.cc   | 78 ++++++++++++++++
 paddle/fluid/inference/tensorrt/op_teller.cc  | 37 ++++++--
 test/ir/inference/test_trt_convert_reduce.py  | 93 ++++++++++---------
 4 files changed, 159 insertions(+), 51 deletions(-)

diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc
index e46d4c600dc74..30734e38b3e8f 100644
--- a/paddle/fluid/inference/api/analysis_predictor.cc
+++ b/paddle/fluid/inference/api/analysis_predictor.cc
@@ -2615,6 +2615,8 @@ USE_TRT_CONVERTER(reduce_max);
 USE_TRT_CONVERTER(reduce_min);
 USE_TRT_CONVERTER(reduce_sum);
 USE_TRT_CONVERTER(reduce_prod);
+USE_TRT_CONVERTER(reduce_any);
+USE_TRT_CONVERTER(reduce_all);
 USE_TRT_CONVERTER(tile);
 USE_TRT_CONVERTER(conv3d);
 USE_TRT_CONVERTER(conv3d_transpose);
diff --git a/paddle/fluid/inference/tensorrt/convert/reduce_op.cc b/paddle/fluid/inference/tensorrt/convert/reduce_op.cc
index 525968b5010e6..16221851ae045 100644
--- a/paddle/fluid/inference/tensorrt/convert/reduce_op.cc
+++ b/paddle/fluid/inference/tensorrt/convert/reduce_op.cc
@@ -95,6 +95,8 @@ const std::unordered_map<std::string, std::vector<nvinfer1::ReduceOperation>>
         {"reduce_max", {nvinfer1::ReduceOperation::kMAX}},
         {"reduce_min", {nvinfer1::ReduceOperation::kMIN}},
         {"reduce_prod", {nvinfer1::ReduceOperation::kPROD}},
+        {"reduce_any", {nvinfer1::ReduceOperation::kMAX}},
+        {"reduce_all", {nvinfer1::ReduceOperation::kMIN}},
 };
 
 class ReduceSumOpConverter : public ReduceOpConverter {
@@ -122,6 +124,80 @@ class ReduceProdOpConverter : public ReduceOpConverter {
   ReduceProdOpConverter() { op_type = "reduce_prod"; }
 };
 
+class ReduceAnyOpConverter : public ReduceOpConverter {
+ public:
+  ReduceAnyOpConverter() { op_type = "reduce_any"; }
+  void operator()(const framework::proto::OpDesc& op,
+                  const framework::Scope& scope,
+                  bool test_mode) override {
+    VLOG(4) << "convert a paddle " << op_type << " op to tensorrt reduce layer";
+    framework::OpDesc op_desc(op, nullptr);
+    auto reduce_type = ops_.find(op_type);
+    auto* x = engine_->GetITensor(op_desc.Input("X").front());
+    // Cast the DataType to float
+    nvinfer1::IReduceLayer* reduce_layer = nullptr;
+    auto* cast_layer = TRT_ENGINE_ADD_LAYER(engine_, Identity, *x);
+    cast_layer->setOutputType(0, nvinfer1::DataType::kINT32);
+    cast_layer->getOutput(0)->setType(nvinfer1::DataType::kINT32);
+
+    nvinfer1::Dims input_shape = x->getDimensions();
+    int input_dims = input_shape.nbDims;
+    // Discriminate DataType between int and bool.
+    bool keep_dim = PADDLE_GET_CONST(bool, op_desc.GetAttr("keep_dim"));
+    std::vector<int32_t> dim =
+        PADDLE_GET_CONST(std::vector<int32_t>, op_desc.GetAttr("dim"));
+    bool reduce_all = PADDLE_GET_CONST(bool, op_desc.GetAttr("reduce_all"));
+
+    if (reduce_all) {
+      uint32_t reduce_dim = 0;
+      for (int i = 0; i < input_dims; ++i) {
+        reduce_dim |= 1 << i;
+      }
+      reduce_layer = TRT_ENGINE_ADD_LAYER(engine_,
+                                          Reduce,
+                                          *cast_layer->getOutput(0),
+                                          reduce_type->second.front(),
+                                          reduce_dim,
+                                          keep_dim);
+    } else {
+      auto CvtToBitMask = [&](const std::vector<int32_t>& dims) -> uint32_t {
+        uint32_t res = 0;
+        for (auto x : dims) {
+          if (x < 0) {
+            res |= 1 << (x + input_dims);
+          } else {
+            if (!engine_->with_dynamic_shape()) x = x - 1;
+            res |= 1 << x;
+          }
+        }
+        return res;
+      };
+      reduce_layer = TRT_ENGINE_ADD_LAYER(engine_,
+                                          Reduce,
+                                          *cast_layer->getOutput(0),
+                                          reduce_type->second.front(),
+                                          CvtToBitMask(dim),
+                                          keep_dim);
+    }
+
+    auto output_name = op_desc.Output("Out")[0];
+
+    auto* layer =
+        TRT_ENGINE_ADD_LAYER(engine_, Identity, *reduce_layer->getOutput(0));
+    layer->setOutputType(0, nvinfer1::DataType::kBOOL);
+    layer->getOutput(0)->setType(nvinfer1::DataType::kBOOL);
+    // Ensure that the output type and input type are consistent.
+    layer->getOutput(0)->setType(cast_layer->getInput(0)->getType());
+
+    RreplenishLayerAndOutput(layer, op_type, {output_name}, test_mode);
+  };
+};
+
+class ReduceAllOpConverter : public ReduceAnyOpConverter {
+ public:
+  ReduceAllOpConverter() { op_type = "reduce_all"; }
+};
+
 }  // namespace tensorrt
 }  // namespace inference
 }  // namespace paddle
@@ -131,3 +207,5 @@ REGISTER_TRT_OP_CONVERTER(reduce_mean, ReduceMeanOpConverter);
 REGISTER_TRT_OP_CONVERTER(reduce_max, ReduceMaxOpConverter);
 REGISTER_TRT_OP_CONVERTER(reduce_min, ReduceMinOpConverter);
 REGISTER_TRT_OP_CONVERTER(reduce_prod, ReduceProdOpConverter);
+REGISTER_TRT_OP_CONVERTER(reduce_any, ReduceAnyOpConverter);
+REGISTER_TRT_OP_CONVERTER(reduce_all, ReduceAllOpConverter);
diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index ccbb20d32a6fe..0aea348566bff 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -2193,7 +2193,8 @@ struct SimpleOpTypeSetTeller : public Teller {
 
     if (op_type == "reduce_sum" || op_type == "reduce_mean" ||
         op_type == "reduce_max" || op_type == "reduce_min" ||
-        op_type == "reduce_prod") {
+        op_type == "reduce_prod" || op_type == "reduce_any" ||
+        op_type == "reduce_all") {
       if (!desc.HasAttr("dim", /*with_attr_var=*/false)) {
         VLOG(3) << "Skip to convert into TRT while found Attribute('dim') is "
                    "Variable type in "
@@ -2234,14 +2235,28 @@ struct SimpleOpTypeSetTeller : public Teller {
           return false;
       }
 
-#if IS_TRT_VERSION_LT(7000)
       auto dtype = x_var_desc->GetDataType();
-      if (dtype != framework::proto::VarType::FP32) {
-        VLOG(3) << "reduce op input data type must be float32 using TensorRT "
-                   "< 7.0";
-        return false;
-      }
+      if (op_type == "reduce_all" || op_type == "reduce_any") {
+        if (dtype != framework::proto::VarType::BOOL) {
+          VLOG(3)
+              << "reduce_all and reduce_any op input data type must be bool";
+          return false;
+        }
+      } else {
+#if IS_TRT_VERSION_GE(7000)
+        if (dtype != framework::proto::VarType::INT32 &&
+            dtype != framework::proto::VarType::FP32) {
+          VLOG(3) << "reduce op input data type must be int32 or float32";
+          return false;
+        }
+#else
+        if (dtype != framework::proto::VarType::FP32) {
+          VLOG(3) << "reduce op input data type must be float32 using TensorRT "
+                     "< 7.0";
+          return false;
+        }
 #endif
+      }
     }
 #if IS_TRT_VERSION_GE(7000)
     if (op_type == "tile") {
@@ -2804,8 +2819,12 @@ struct SimpleOpTypeSetTeller : public Teller {
       "nearest_interp",
       "anchor_generator",
       "reduce_max",
+      "reduce_min",
       "reduce_mean",
       "reduce_sum",
+      "reduce_prod",
+      "reduce_any",
+      "reduce_all",
       "conv3d",
       "conv3d_transpose",
       "mish",
@@ -2961,8 +2980,12 @@ struct SimpleOpTypeSetTeller : public Teller {
       "nearest_interp",
       "anchor_generator",
       "reduce_max",
+      "reduce_min",
       "reduce_mean",
       "reduce_sum",
+      "reduce_prod",
+      "reduce_any",
+      "reduce_all",
       "conv3d",
       "conv3d_transpose",
       "mish",
diff --git a/test/ir/inference/test_trt_convert_reduce.py b/test/ir/inference/test_trt_convert_reduce.py
index 1c04e40fe6d87..5db38121c5f3c 100644
--- a/test/ir/inference/test_trt_convert_reduce.py
+++ b/test/ir/inference/test_trt_convert_reduce.py
@@ -51,6 +51,8 @@ def generate_input1(dtype, attrs: List[Dict[str, Any]]):
                 return np.random.random([1, 3, 64, 64]).astype(np.float32)
             elif dtype == 2:
                 return np.random.random([1, 3, 64, 64]).astype(np.int32)
+            elif dtype == 0:
+                return np.random.random([1, 3, 64, 64]).astype(np.bool_)
 
         for keep_dim in [True, False]:
             for dim in [
@@ -65,15 +67,24 @@ def generate_input1(dtype, attrs: List[Dict[str, Any]]):
                 [3, 4, 5],
             ]:
                 for reduce_all in [True, False]:
-                    for out_dtype in [-1, 2, 5]:
-                        for op_type in [
-                            "reduce_max",
-                            "reduce_min",
-                            "reduce_mean",
-                            "reduce_sum",
-                            "reduce_prod",
-                        ]:
-                            dics1 = [
+                    for out_dtype in [-1, 0, 2, 5]:
+                        if out_dtype != 0:
+                            reduce_type_list = [
+                                "reduce_max",
+                                "reduce_min",
+                                "reduce_mean",
+                                "reduce_sum",
+                                "reduce_prod",
+                            ]
+                        else:
+                            reduce_type_list = [
+                                "reduce_all",
+                                "reduce_any",
+                            ]
+
+                        for op_type in reduce_type_list:
+
+                            dics = [
                                 {
                                     "keep_dim": keep_dim,
                                     "dim": dim,
@@ -83,46 +94,40 @@ def generate_input1(dtype, attrs: List[Dict[str, Any]]):
                                 },
                                 {},
                             ]
-                            dics2 = [
+
+                            ops_config = [
                                 {
-                                    "keep_dim": keep_dim,
-                                    "dim": dim,
-                                    "reduce_all": reduce_all,
-                                    "out_dtype": out_dtype,
-                                    "in_dtype": out_dtype,
-                                },
-                                {},
+                                    "op_type": op_type,
+                                    "op_inputs": {"X": ["input_data"]},
+                                    "op_outputs": {
+                                        "Out": ["reduce_output_data"]
+                                    },
+                                    "op_attrs": dics[0],
+                                }
                             ]
-                            for dics in [dics1, dics2]:
-                                ops_config = [
-                                    {
-                                        "op_type": op_type,
-                                        "op_inputs": {"X": ["input_data"]},
-                                        "op_outputs": {
-                                            "Out": ["reduce_output_data"]
-                                        },
-                                        "op_attrs": dics[0],
-                                    }
-                                ]
-                                ops = self.generate_op_config(ops_config)
-
-                                program_config = ProgramConfig(
-                                    ops=ops,
-                                    weights={},
-                                    inputs={
-                                        "input_data": TensorConfig(
-                                            data_gen=partial(
-                                                generate_input1, out_dtype, dics
-                                            )
+                            if op_type in ["reduce_any", "reduce_all"]:
+                                ops_config[0]["outputs_dtype"] = {
+                                    "reduce_output_data": np.bool_
+                                }
+                            ops = self.generate_op_config(ops_config)
+
+                            program_config = ProgramConfig(
+                                ops=ops,
+                                weights={},
+                                inputs={
+                                    "input_data": TensorConfig(
+                                        data_gen=partial(
+                                            generate_input1, out_dtype, dics
                                         )
-                                    },
-                                    outputs=["reduce_output_data"],
-                                )
+                                    )
+                                },
+                                outputs=["reduce_output_data"],
+                            )
 
-                                if not self.is_program_valid(program_config):
-                                    continue
+                            if not self.is_program_valid(program_config):
+                                continue
 
-                                yield program_config
+                            yield program_config
 
     def sample_predictor_configs(
         self, program_config

From 03fe3ce56c15aa8d61d255f479db182b46d0f161 Mon Sep 17 00:00:00 2001
From: liuzhenhai93 <liuzhenhai93@outlook.com>
Date: Sat, 6 May 2023 11:33:26 +0800
Subject: [PATCH 183/405] fix brpc double link (#53512)

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish

* polish
---
 CMakeLists.txt                        | 5 +++++
 paddle/fluid/inference/CMakeLists.txt | 8 ++++++++
 2 files changed, 13 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 34110edff3038..0509e26a6059e 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -92,6 +92,11 @@ message(STATUS "C compiler: ${CMAKE_C_COMPILER}, version: "
                "${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}")
 message(STATUS "AR tools: ${CMAKE_AR}")
 
+if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND CMAKE_CXX_COMPILER_VERSION
+                                              VERSION_GREATER 10.4)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=uninitialized")
+endif()
+
 # MUSL build turn off warnings
 if(WITH_MUSL)
   set(CMAKE_CXX_FLAGS
diff --git a/paddle/fluid/inference/CMakeLists.txt b/paddle/fluid/inference/CMakeLists.txt
index 59035ab8cd09e..36d01c0f059a4 100644
--- a/paddle/fluid/inference/CMakeLists.txt
+++ b/paddle/fluid/inference/CMakeLists.txt
@@ -151,6 +151,14 @@ if(NOT APPLE AND NOT WIN32)
   # TODO(liuyiqun): Temporarily disable the link flag because it is not support on Mac.
   set(LINK_FLAGS
       "-Wl,--version-script ${CMAKE_CURRENT_SOURCE_DIR}/paddle_inference.map")
+
+  # temporarily fix brpc double link issue(52842)
+  if(WITH_INFERENCE_API_TEST
+     AND (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+     AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 10.4)
+    set(LINK_FLAGS "${LINK_FLAGS} -Wl,-Bsymbolic")
+  endif()
+
   set_target_properties(paddle_inference_shared PROPERTIES LINK_FLAGS
                                                            "${LINK_FLAGS}")
   # check symbol hidden

From da963eab7aa891e3758bbf38e00f567ec76b6ebb Mon Sep 17 00:00:00 2001
From: Yuang Liu <liuyuang@baidu.com>
Date: Sat, 6 May 2023 12:30:06 +0800
Subject: [PATCH 184/405] use int64 to calc dim for c softmax (#53541)

* use int64 to calc dim for c softmax

* fix complie bug
---
 .../c_softmax_with_cross_entropy_op.cu        | 50 +++++++++----------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu
index 114c52f608f34..5a78689819714 100644
--- a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu
+++ b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu
@@ -30,9 +30,9 @@ namespace paddle {
 namespace operators {
 
 static constexpr int kNumCUDAThreads = 512;
-static constexpr int kNumMaxinumNumBlocks = 4096;
+static constexpr int64_t kNumMaxinumNumBlocks = 4096;
 
-static inline int NumBlocks(const int N) {
+static inline int64_t NumBlocks(const int64_t N) {
   return std::min((N + kNumCUDAThreads - 1) / kNumCUDAThreads,
                   kNumMaxinumNumBlocks);
 }
@@ -42,12 +42,12 @@ __global__ void MaskLabelByIndex(T* predicted_logits,
                                  const T* logit,
                                  const IndexT* label,
                                  const IndexT ignore_index,
-                                 const int start_index,
-                                 const int end_index,
+                                 const int64_t start_index,
+                                 const int64_t end_index,
                                  const int64_t N,
                                  const int64_t D,
                                  const int nranks) {
-  CUDA_KERNEL_LOOP(i, N) {
+  CUDA_KERNEL_LOOP_TYPE(i, N, int64_t) {
     auto real_label = label[i];
     PADDLE_ENFORCE(((real_label < D * nranks) && (real_label >= 0)) ||
                        (real_label == ignore_index),
@@ -71,8 +71,8 @@ __global__ void CaculateLoss(T* loss,
                              const T* sum_exp_logits,
                              const IndexT* label,
                              const int64_t ignore_index,
-                             const int N) {
-  CUDA_KERNEL_LOOP(i, N) {
+                             const int64_t N) {
+  CUDA_KERNEL_LOOP_TYPE(i, N, int64_t) {
     auto real_label = static_cast<int64_t>(label[i]);
     loss[i] = ignore_index == real_label
                   ? static_cast<T>(0)
@@ -87,12 +87,12 @@ template <typename T, typename IndexT>
 __global__ void MaskLabelByIndexGrad(T* logits_grad,
                                      const T* loss_grad,
                                      const IndexT* labels,
-                                     const int start_index,
-                                     const int end_index,
+                                     const int64_t start_index,
+                                     const int64_t end_index,
                                      const int64_t N,
                                      const int64_t D,
                                      const int64_t ignore_index) {
-  CUDA_KERNEL_LOOP(i, N * D) {
+  CUDA_KERNEL_LOOP_TYPE(i, N * D, int64_t) {
     auto row = i / D;
     auto col = i % D;
     auto lbl = static_cast<int64_t>(labels[row]);
@@ -152,8 +152,8 @@ struct CSoftmaxWithCrossEntropyFunctor<phi::GPUContext, T> {
     const auto& labels_dims = labels->dims();
 
     const int axis = logits_dims.size() - 1;
-    const int N = phi::funcs::SizeToAxis(axis, logits_dims);
-    const int D = phi::funcs::SizeFromAxis(axis, logits_dims);
+    const int64_t N = phi::funcs::SizeToAxis<int64_t>(axis, logits_dims);
+    const int64_t D = phi::funcs::SizeFromAxis<int64_t>(axis, logits_dims);
 
     phi::DenseTensor logits_2d, softmax_2d, loss_2d;
     logits_2d.ShareDataWith(*logits).Resize({N, D});
@@ -200,10 +200,10 @@ struct CSoftmaxWithCrossEntropyFunctor<phi::GPUContext, T> {
     auto t = framework::EigenVector<T>::Flatten(predicted_logits);
     t.device(*dev_ctx.eigen_device()) = t.constant(static_cast<T>(0));
 
-    const int start_index = rank * D;
-    const int end_index = start_index + D;
+    const int64_t start_index = rank * D;
+    const int64_t end_index = start_index + D;
 
-    int blocks = NumBlocks(N);
+    int64_t blocks = NumBlocks(N);
     int threads = kNumCUDAThreads;
     const auto& label_type = framework::TransToProtoVarType(labels->dtype());
 
@@ -318,8 +318,8 @@ struct CSoftmaxWithCrossEntropyProcessGroupFunctor<phi::GPUContext, T> {
     const auto& labels_dims = labels->dims();
 
     const int axis = logits_dims.size() - 1;
-    const int N = phi::funcs::SizeToAxis(axis, logits_dims);
-    const int D = phi::funcs::SizeFromAxis(axis, logits_dims);
+    const int64_t N = phi::funcs::SizeToAxis<int64_t>(axis, logits_dims);
+    const int64_t D = phi::funcs::SizeFromAxis<int64_t>(axis, logits_dims);
 
     phi::DenseTensor logits_2d, softmax_2d, loss_2d;
     logits_2d.ShareDataWith(*logits).Resize({N, D});
@@ -358,10 +358,10 @@ struct CSoftmaxWithCrossEntropyProcessGroupFunctor<phi::GPUContext, T> {
     auto t = framework::EigenVector<T>::Flatten(predicted_logits);
     t.device(*dev_ctx.eigen_device()) = t.constant(static_cast<T>(0));
 
-    const int start_index = rank * D;
-    const int end_index = start_index + D;
+    const int64_t start_index = rank * D;
+    const int64_t end_index = start_index + D;
 
-    int blocks = NumBlocks(N);
+    int64_t blocks = NumBlocks(N);
     int threads = kNumCUDAThreads;
     const auto& label_type = framework::TransToProtoVarType(labels->dtype());
 
@@ -454,17 +454,17 @@ class CSoftmaxWithCrossEntropyGradCUDAKernel : public framework::OpKernel<T> {
     }
     const auto sofrmax_dims = softmax->dims();
     const int axis = sofrmax_dims.size() - 1;
-    const int N = phi::funcs::SizeToAxis(axis, sofrmax_dims);
-    const int D = phi::funcs::SizeFromAxis(axis, sofrmax_dims);
+    const int64_t N = phi::funcs::SizeToAxis<int64_t>(axis, sofrmax_dims);
+    const int64_t D = phi::funcs::SizeFromAxis<int64_t>(axis, sofrmax_dims);
 
     phi::DenseTensor logit_grad_2d;
     logit_grad_2d.ShareDataWith(*logit_grad).Resize({N, D});
 
-    int blocks = NumBlocks(N * D);
+    int64_t blocks = NumBlocks(N * D);
     int threads = kNumCUDAThreads;
     const auto& label_type = framework::TransToProtoVarType(labels->dtype());
-    const int start_index = rank * D;
-    const int end_index = start_index + D;
+    const int64_t start_index = rank * D;
+    const int64_t end_index = start_index + D;
 
     if (label_type == framework::proto::VarType::INT32) {
       MaskLabelByIndexGrad<T, int32_t>

From 08a8b75de5dc35c0ca7e8373ce0e2e78a88804ad Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Fri, 5 May 2023 14:27:58 +0800
Subject: [PATCH 185/405] Rewirte the reshape of temp_mask and temp_bias.

---
 cmake/external/flashattn.cmake                |   2 +-
 .../operators/fused/fused_gate_attention.h    | 125 +++++++-----------
 2 files changed, 52 insertions(+), 75 deletions(-)

diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index 29c0b8efb426c..625259ce47b62 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -20,7 +20,7 @@ set(FLASHATTN_PREFIX_DIR ${THIRD_PARTY_PATH}/flashattn)
 set(FLASHATTN_SOURCE_SUBDIR csrc/flash_attn)
 set(FLASHATTN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flashattn)
 set(FLASHATTN_REPOSITORY ${GIT_URL}/JamesLim-sy/flash-attention.git)
-set(FLASHATTN_TAG 782a98a1fc02fb36d07129a97750430e4fdf2022)
+set(FLASHATTN_TAG bedc55b0ebc26776909b2b989401fabf87c9bfa3)
 
 set(FLASHATTN_INCLUDE_DIR
     "${FLASHATTN_INSTALL_DIR}/include"
diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index c163c1eac7589..8447d27d3032c 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -951,27 +951,8 @@ class FlashAttnWithGating {
     AllocAndInitSeqQK(seq_batch_size, step, &cu_seq_q, &cu_seq_k);
 
     // 3. Dealing with mask and bias for flash_attn.
-    phi::DenseTensor temp_mask, temp_bias;
-    auto dims_merge_func = [&](const phi::DenseTensor* src_tensor,
-                               phi::DenseTensor* dst_tensor,
-                               const std::string& prefix) {
-      if (src_tensor) {
-        int64_t first_dim = 1;
-        dst_tensor->ShareDataWith(*src_tensor);
-        auto dims_ = src_tensor->dims();
-        for (int i = 0; i < dims_.size() - 3; ++i) {
-          first_dim *= dims_[i];
-        }
-        auto dims_rank = dims_.size();
-        dst_tensor->Resize({first_dim,
-                            dims_[dims_rank - 3],
-                            dims_[dims_rank - 2],
-                            dims_[dims_rank - 1]});
-        VLOG(6) << prefix << ": " << TensorDebugString(dst_tensor);
-      }
-    };
-    dims_merge_func(src_mask, &temp_mask, "temp_mask");
-    dims_merge_func(nonbatched_bias, &temp_bias, "temp_bias");
+    std::vector<int64_t> temp_mask_dim = GetCompressedDim(src_mask);
+    std::vector<int64_t> temp_bias_dim = GetCompressedDim(nonbatched_bias);
 
     // 4. flash_attn parameter setting.
     InitFlashAttnArguments(config);
@@ -984,8 +965,6 @@ class FlashAttnWithGating {
     if (VLOG_IS_ON(6)) {
       VLOG(6) << TensorDebugString(&cu_seq_q, "cu_seq_q");
       VLOG(6) << TensorDebugString(&cu_seq_k, "cu_seq_k");
-      VLOG(6) << TensorDebugString(&temp_bias, "temp_bias");
-      VLOG(6) << TensorDebugString(&temp_mask, "temp_mask");
       VLOG(6) << TensorDebugString(nonbatched_bias, "nonbatched_bias");
       VLOG(6) << TensorDebugString(src_mask, "src_mask");
       VLOG(6) << TensorDebugString(qkv_transpose_out, "qkv_transpose_out");
@@ -1024,10 +1003,10 @@ class FlashAttnWithGating {
         stream,
         fa_seed_,
         fa_offset_,
-        src_mask ? temp_mask.data() : nullptr,
-        nonbatched_bias ? temp_bias.data() : nullptr,
-        src_mask ? temp_mask.dims().Get() : nullptr,
-        nonbatched_bias ? temp_bias.dims().Get() : nullptr);
+        src_mask ? src_mask->data() : nullptr,
+        nonbatched_bias ? nonbatched_bias->data() : nullptr,
+        src_mask ? temp_mask_dim.data() : nullptr,
+        nonbatched_bias ? temp_bias_dim.data() : nullptr);
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
@@ -1048,7 +1027,7 @@ class FlashAttnWithGating {
         fa_head_size_,
         fa_max_seqlen_q_,
         fa_max_seqlen_k_,
-        fa_dropout_probs_,
+        fa_dropout_prob_,
         fa_softmax_scale_,
         fa_zero_tensors_,
         fa_is_causal_,
@@ -1061,10 +1040,10 @@ class FlashAttnWithGating {
         stream,
         fa_seed_,
         fa_offset_,
-        src_mask ? temp_mask.data() : nullptr,
-        nonbatched_bias ? temp_bias.data() : nullptr,
-        src_mask ? temp_mask.dims().Get() : nullptr,
-        nonbatched_bias ? temp_bias.dims().Get() : nullptr);
+        src_mask ? src_mask->data() : nullptr,
+        nonbatched_bias ? nonbatched_bias->data() : nullptr,
+        src_mask ? temp_mask_dim.data() : nullptr,
+        nonbatched_bias ? temp_bias_dim.data() : nullptr);
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
@@ -1107,7 +1086,7 @@ class FlashAttnWithGating {
     T* q_grad_ptr = qkv_transpose_out_grad.data<T>();
     T* k_grad_ptr = q_grad_ptr + q_size;
     T* v_grad_ptr = k_grad_ptr + q_size;
-    VLOG(5) << WaitWithDebugInfo(dev_ctx_);
+    WaitWithDebugInfo(dev_ctx_);
 
     // 2. Init with cu_seq_q and cu_seq_k for flash_attn.
     phi::DenseTensor cu_seq_q, cu_seq_k;
@@ -1115,29 +1094,12 @@ class FlashAttnWithGating {
                          static_cast<int>(config->seq_len_m);
     int64_t step = static_cast<int64_t>(config->seq_len_r);
     AllocAndInitSeqQK(seq_batch_size, step, &cu_seq_q, &cu_seq_k);
+    const int32_t* cu_seq_q_ptr = cu_seq_q.data<int32_t>();
+    const int32_t* cu_seq_k_ptr = cu_seq_k.data<int32_t>();
 
     // 3. Dealing with mask and bias for flash_attn.
-    phi::DenseTensor temp_mask, temp_bias;
-    auto dims_merge_func = [&](const phi::DenseTensor* src_tensor,
-                               phi::DenseTensor* dst_tensor,
-                               const std::string& prefix) {
-      if (src_tensor) {
-        int64_t first_dim = 1;
-        dst_tensor->ShareDataWith(*src_tensor);
-        auto dims_ = src_tensor->dims();
-        for (int i = 0; i < dims_.size() - 3; ++i) {
-          first_dim *= dims_[i];
-        }
-        auto dims_rank = dims_.size();
-        dst_tensor->Resize({first_dim,
-                            dims_[dims_rank - 3],
-                            dims_[dims_rank - 2],
-                            dims_[dims_rank - 1]});
-        VLOG(6) << TensorDebugString(dst_tensor, prefix);
-      }
-    };
-    dims_merge_func(src_mask, &temp_mask, "temp_mask");
-    dims_merge_func(nonbatched_bias, &temp_bias, "temp_bias");
+    std::vector<int64_t> temp_mask_dim = GetCompressedDim(src_mask);
+    std::vector<int64_t> temp_bias_dim = GetCompressedDim(nonbatched_bias);
 
     // 4. flash_attn parameter setting.
     InitFlashAttnArguments(config);
@@ -1175,8 +1137,8 @@ class FlashAttnWithGating {
         static_cast<void*>(v_grad_ptr),
         nullptr,  // set out to nullptr to calculate workspace size
         static_cast<const void*>(fmha_out_grad->data()),
-        cu_seq_q.data<int32_t>(),
-        cu_seq_k.data<int32_t>(),
+        cu_seq_q_ptr,
+        cu_seq_k_ptr,
         fa_total_q_,
         fa_total_k_,
         fa_batch_size_,
@@ -1198,10 +1160,10 @@ class FlashAttnWithGating {
         stream,
         fa_seed_,
         fa_offset_,
-        src_mask ? temp_mask.data() : nullptr,
-        nonbatched_bias ? temp_bias.data() : nullptr,
-        src_mask ? temp_mask.dims().Get() : nullptr,
-        nonbatched_bias ? temp_bias.dims().Get() : nullptr);
+        src_mask ? src_mask->data() : nullptr,
+        nonbatched_bias ? nonbatched_bias->data() : nullptr,
+        src_mask ? temp_mask_dim.data() : nullptr,
+        nonbatched_bias ? temp_bias_dim.data() : nullptr);
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
@@ -1216,8 +1178,8 @@ class FlashAttnWithGating {
         static_cast<void*>(v_grad_ptr),
         static_cast<const void*>(fmha_out->data()),
         static_cast<const void*>(fmha_out_grad->data()),
-        cu_seq_q.data<int32_t>(),
-        cu_seq_k.data<int32_t>(),
+        cu_seq_q_ptr,
+        cu_seq_k_ptr,
         fa_total_q_,
         fa_total_k_,
         fa_batch_size_,
@@ -1239,10 +1201,10 @@ class FlashAttnWithGating {
         stream,
         fa_seed_,
         fa_offset_,
-        src_mask ? temp_mask.data() : nullptr,
-        nonbatched_bias ? temp_bias.data() : nullptr,
-        src_mask ? temp_mask.dims().Get() : nullptr,
-        nonbatched_bias ? temp_bias.dims().Get() : nullptr);
+        src_mask ? src_mask->data() : nullptr,
+        nonbatched_bias ? nonbatched_bias->data() : nullptr,
+        src_mask ? temp_mask_dim.data() : nullptr,
+        nonbatched_bias ? temp_bias_dim.data() : nullptr);
     if (!succ) {
       PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
     }
@@ -1251,14 +1213,12 @@ class FlashAttnWithGating {
 
     if (nonbatched_bias) {
       // compare block reduce
-      const auto temp_bias_num = temp_bias.numel();
-      const auto bias_d_num = bias_d.numel();
-      auto dbias_first_dim = bias_d_num / temp_bias_num;
+      auto dbias_first_dim = nonbatched_bias->numel() / bias_d.numel();
       bias_d.Resize({dbias_first_dim,
-                     temp_bias.dims()[0],
-                     temp_bias.dims()[1],
-                     temp_bias.dims()[2],
-                     temp_bias.dims()[3]});
+                     temp_bias_dim[0],
+                     temp_bias_dim[1],
+                     temp_bias_dim[2],
+                     temp_bias_dim[3]});
       phi::funcs::ReduceKernel<T, T, kps::AddFunctor, kps::IdentityFunctor<T>>(
           dev_ctx_,
           bias_d,
@@ -1295,6 +1255,23 @@ class FlashAttnWithGating {
             << ", step=" << step << ", end=" << end;
   }
 
+  std::vector<int64_t> GetCompressedDim(const phi::DenseTensor* tensor) {
+    std::vector<int64_t> compressed_dims;
+    if (tensor) {
+      int64_t first_dim = 1;
+      const auto& origin_dims = tensor->dims();
+      auto rank = origin_dims.size();
+      for (int i = 0; i < rank - 3; ++i) {
+        first_dim *= origin_dims[i];
+      }
+      compressed_dims = {first_dim,
+                         origin_dims[rank - 3],
+                         origin_dims[rank - 2],
+                         origin_dims[rank - 1]};
+    }
+    return compressed_dims;
+  }
+
   phi::DenseTensor CreateWorkspace(uint64_t workspace_size) {
     phi::DenseTensor workspace;
     if (workspace_size > 0) {
@@ -1380,7 +1357,7 @@ class FlashAttnWithGating {
   int fa_max_seqlen_k_;
   int fa_num_splits_;
   float fa_softmax_scale_{1.0f};
-  float fa_dropout_prob_(0.0f);
+  float fa_dropout_prob_{0.0f};
   uint64_t fa_seed_{0};
   uint64_t fa_offset_{0};
   bool fa_zero_tensors_{false};

From eda8df7133680ba56e7ab4cc762d265759621a6b Mon Sep 17 00:00:00 2001
From: RuohengMa <120699764+RuohengMa@users.noreply.github.com>
Date: Sat, 6 May 2023 14:11:52 +0800
Subject: [PATCH 186/405] [XPU] substitute new api kernel for combinatorial
 adaptive avg_pool2d_grad kernel (#53528)

---
 paddle/phi/kernels/xpu/pool_grad_kernel.cc | 24 ----------------------
 1 file changed, 24 deletions(-)

diff --git a/paddle/phi/kernels/xpu/pool_grad_kernel.cc b/paddle/phi/kernels/xpu/pool_grad_kernel.cc
index 4eeef936216ef..b03be5dd9449c 100644
--- a/paddle/phi/kernels/xpu/pool_grad_kernel.cc
+++ b/paddle/phi/kernels/xpu/pool_grad_kernel.cc
@@ -112,30 +112,6 @@ void Pool2dGradKernel(const Context& ctx,
           true);
 
     } else if (pooling_type == "avg") {
-      // When output dim is 1 * 1 (1 * 1 * 1 in pool_3d), use scale
-      // and broadcast kernels to get same output, but better performance.
-      if (out_h == 1 && out_w == 1 && std::is_same<T, float>::value) {
-        xpu::ctx_guard RAII_GUARD(ctx.x_context());
-        float scale = 1.0 / (in_h * in_w);
-        float* scaled_dy = RAII_GUARD.alloc_l3_or_gm<float>(n * c);
-        r = xpu::scale(ctx.x_context(),
-                       dout.data<float>(),
-                       scaled_dy,
-                       n * c,
-                       true,
-                       scale,
-                       0.0);
-        PADDLE_ENFORCE_XDNN_SUCCESS(r, "scale");
-
-        r = xpu::broadcast(ctx.x_context(),
-                           scaled_dy,
-                           dx->data<float>(),
-                           {n, c, 1, 1},
-                           {n, c, in_h, in_w});
-        PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast");
-
-        return;
-      }
       r = xpu::adaptive_avg_pool2d_grad<XPUType>(
           ctx.x_context(),
           reinterpret_cast<const XPUType*>(dout.data<T>()),

From 4682c0dcfb41c970f0dd8f5ab03c1261678c2901 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Sat, 6 May 2023 15:01:07 +0800
Subject: [PATCH 187/405] Update commit and fix reduce_dim.

---
 cmake/external/flashattn.cmake                      | 2 +-
 paddle/fluid/operators/fused/fused_gate_attention.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index 625259ce47b62..8cbcdb717223a 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -20,7 +20,7 @@ set(FLASHATTN_PREFIX_DIR ${THIRD_PARTY_PATH}/flashattn)
 set(FLASHATTN_SOURCE_SUBDIR csrc/flash_attn)
 set(FLASHATTN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flashattn)
 set(FLASHATTN_REPOSITORY ${GIT_URL}/JamesLim-sy/flash-attention.git)
-set(FLASHATTN_TAG bedc55b0ebc26776909b2b989401fabf87c9bfa3)
+set(FLASHATTN_TAG 93486d94943605436bb4cb636d9ac4a109d3fcaf)
 
 set(FLASHATTN_INCLUDE_DIR
     "${FLASHATTN_INSTALL_DIR}/include"
diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 8447d27d3032c..8c736e8854c36 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -1213,7 +1213,7 @@ class FlashAttnWithGating {
 
     if (nonbatched_bias) {
       // compare block reduce
-      auto dbias_first_dim = nonbatched_bias->numel() / bias_d.numel();
+      auto dbias_first_dim = bias_d.numel() / nonbatched_bias->numel();
       bias_d.Resize({dbias_first_dim,
                      temp_bias_dim[0],
                      temp_bias_dim[1],

From 99399f32cf6665d6980c97e5fe5a8f06513568a4 Mon Sep 17 00:00:00 2001
From: csy0225 <78470701+csy0225@users.noreply.github.com>
Date: Sat, 6 May 2023 15:15:55 +0800
Subject: [PATCH 188/405] XPU Support external stream (#53334)

---
 .../fluid/inference/api/analysis_predictor.cc |  98 +++++++++++++-
 .../fluid/inference/api/analysis_predictor.h  |   3 +
 paddle/fluid/inference/api/infer_context.cc   |   5 +
 paddle/fluid/inference/api/infer_context.h    |  11 ++
 paddle/fluid/inference/api/paddle_api.h       |   3 +-
 .../fluid/inference/api/resource_manager.cc   | 120 ++++++++++++++++++
 paddle/fluid/inference/api/resource_manager.h |  49 +++++++
 paddle/phi/backends/xpu/xpu_context.cc        |  26 +++-
 paddle/phi/backends/xpu/xpu_context.h         |   9 ++
 .../api/analysis_predictor_tester.cc          |  55 ++++++++
 10 files changed, 373 insertions(+), 6 deletions(-)

diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc
index 30734e38b3e8f..928153dab5e62 100644
--- a/paddle/fluid/inference/api/analysis_predictor.cc
+++ b/paddle/fluid/inference/api/analysis_predictor.cc
@@ -334,6 +334,26 @@ bool AnalysisPredictor::Init(
       InitDeviceContexts();
     }
   }
+#endif
+#if defined(PADDLE_WITH_XPU)
+  if (config_.use_xpu_ && config_.use_external_stream_) {
+    private_context_ = true;
+  }
+  if (private_context_) {
+    if (!status_is_cloned_) {
+      predictor_stream_ = config_.GetExecStream();
+    }
+    // NOTE: If the external_stream equals to global_device_contexts's stream,
+    // then fallback.
+    auto global_stream =
+        static_cast<phi::XPUContext *>(
+            platform::DeviceContextPool::Instance().Get(place_))
+            ->stream();
+    if (predictor_stream_ != global_stream) {
+      InitResourceManager(predictor_stream_);
+      InitDeviceContexts();
+    }
+  }
 #endif
   inference::DisplayMemoryInfo(place_, "Init predictor");
   return true;
@@ -418,6 +438,9 @@ void AnalysisPredictor::InitResourceManager(void *stream) {
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
   predictor_stream_ =
       ResourceManager::Instance().InitGPUResource(place_, stream);
+#elif defined(PADDLE_WITH_XPU)
+  predictor_stream_ =
+      ResourceManager::Instance().InitXPUResource(place_, stream);
 #endif
 }
 
@@ -487,6 +510,32 @@ void AnalysisPredictor::InitDeviceContexts() {
           return std::unique_ptr<phi::DeviceContext>(gpu_context);
         }));
   }
+#endif
+#if defined(PADDLE_WITH_XPU)
+  if (place_.GetType() == phi::AllocationType::XPU) {
+    device_contexts_.emplace(
+        place_, std::async(std::launch::deferred, [=] {
+          auto *xpu_resource =
+              ResourceManager::Instance().GetXPUResource(predictor_stream_);
+          auto &instance = memory::allocation::AllocatorFacade::Instance();
+          auto *xpu_context = new InferXPUContext(place_);
+          xpu_context->SetAllocator(instance.GetAllocator(place_).get());
+          xpu_context->SetGenerator(
+              phi::DefaultXPUGenerator(place_.GetDeviceId()).get());
+          xpu_context->SetHostAllocator(
+              instance.GetAllocator(platform::CPUPlace()).get());
+          xpu_context->SetHostGenerator(phi::DefaultCPUGenerator().get());
+          xpu_context->SetZeroAllocator(
+              instance.GetZeroAllocator(place_).get());
+          xpu_context->SetHostZeroAllocator(
+              instance.GetZeroAllocator(platform::CPUPlace()).get());
+          xpu_context->SetStream(xpu_resource->GetStream());
+          xpu_context->SetDriverVersion(xpu_resource->GetDriverVersion());
+          xpu_context->SetRuntimeVersion(xpu_resource->GetRuntimeVersion());
+          xpu_context->SetXpuVersion(xpu_resource->GetXpuVersion());
+          return std::unique_ptr<phi::DeviceContext>(xpu_context);
+        }));
+  }
 #endif
   // TODO(Inference): Support other backends.
 }
@@ -506,10 +555,14 @@ void *AnalysisPredictor::GetExecStream() const {
 #endif
 #if defined(PADDLE_WITH_XPU)
   if (place_.GetType() == phi::AllocationType::XPU) {
-    paddle::platform::DeviceContextPool &pool =
-        paddle::platform::DeviceContextPool::Instance();
-    return reinterpret_cast<const phi::XPUContext *>(pool.Get(place_))
-        ->stream();
+    if (private_context_) {
+      return predictor_stream_;
+    } else {
+      paddle::platform::DeviceContextPool &pool =
+          paddle::platform::DeviceContextPool::Instance();
+      return reinterpret_cast<const phi::XPUContext *>(pool.Get(place_))
+          ->stream();
+    }
   }
 #endif
   // TODO(inference): Support other backends.
@@ -2050,6 +2103,33 @@ bool AnalysisPredictor::ExpRunWithExternalStream(const gpuStream_t stream) {
 }
 #endif
 
+bool AnalysisPredictor::ExpRunWithExternalStream(void *stream) {
+#if defined(PADDLE_WITH_XPU)
+  if (!private_context_) {
+    PADDLE_THROW(platform::errors::Fatal(
+        "Please use config.SetExecStream to init resources, and then we "
+        "will bind resources to execution stream."));
+  }
+  if (stream != predictor_stream_) {
+    paddle::platform::XPUStreamSync(
+        static_cast<paddle::xpuStream>(predictor_stream_));
+    ResourceManager::Instance().XpuResourceReBindStream(predictor_stream_,
+                                                        stream);
+    predictor_stream_ = stream;
+
+    auto *dev_ctxs = reinterpret_cast<const std::map<
+        phi::Place,
+        std::shared_future<std::unique_ptr<phi::DeviceContext>>> *>(
+        this->GetDeviceContexts());
+    auto *dev_ctx =
+        static_cast<InferXPUContext *>(dev_ctxs->at(place_).get().get());
+    dev_ctx->SetStream(stream);
+  }
+  return ZeroCopyRun();
+#endif
+  return false;
+}
+
 void AnalysisPredictor::CollectShapeRangeInfo() {
   // if use gpu, sync first.
   paddle::platform::DeviceContextPool &pool =
@@ -2413,7 +2493,12 @@ AnalysisPredictor::~AnalysisPredictor() {
   if (predictor_stream_ != nullptr) {
     ResourceManager::Instance().DestroyGPUResource(predictor_stream_);
   }
+#elif defined(PADDLE_WITH_XPU)
+  if (predictor_stream_ != nullptr) {
+    ResourceManager::Instance().DestroyXPUResource(predictor_stream_);
+  }
 #endif
+
   if (place_.GetType() != phi::AllocationType::UNDEFINED) {
     memory::Release(place_);
   }
@@ -2922,6 +3007,11 @@ bool InternalUtils::RunWithExternalStream(paddle_infer::Predictor *p,
 #endif
   return false;
 }
+bool InternalUtils::RunWithExternalStream(paddle_infer::Predictor *p,
+                                          void *stream) {
+  auto pred = dynamic_cast<paddle::AnalysisPredictor *>(p->predictor_.get());
+  return pred->ExpRunWithExternalStream(stream);
+}
 
 void InternalUtils::UpdateConfigInterleaved(paddle_infer::Config *c,
                                             bool with_interleaved) {
diff --git a/paddle/fluid/inference/api/analysis_predictor.h b/paddle/fluid/inference/api/analysis_predictor.h
index 83207a8bfd654..185aadbc6abf9 100644
--- a/paddle/fluid/inference/api/analysis_predictor.h
+++ b/paddle/fluid/inference/api/analysis_predictor.h
@@ -225,6 +225,9 @@ class AnalysisPredictor : public PaddlePredictor {
   bool ExpRunWithExternalStream(const gpuStream_t stream);
 #endif
 
+  // Note: Can only be used under thread_local semantics.
+  bool ExpRunWithExternalStream(void *stream);
+
   ///
   /// \brief Get the execution stream on devices with a concept of stream,
   /// otherwise returns nullptr.
diff --git a/paddle/fluid/inference/api/infer_context.cc b/paddle/fluid/inference/api/infer_context.cc
index 9a76528a48edb..b56adddfa4e78 100644
--- a/paddle/fluid/inference/api/infer_context.cc
+++ b/paddle/fluid/inference/api/infer_context.cc
@@ -22,4 +22,9 @@ InferGPUContext::InferGPUContext(const phi::Place& place)
     : phi::GPUContext(place, false) {}
 #endif
 
+#if defined(PADDLE_WITH_XPU)
+InferXPUContext::InferXPUContext(const phi::Place& place)
+    : phi::XPUContext(place) {}
+#endif
+
 }  // namespace paddle
diff --git a/paddle/fluid/inference/api/infer_context.h b/paddle/fluid/inference/api/infer_context.h
index 5dbcf355a6e27..130fd8c8d4869 100644
--- a/paddle/fluid/inference/api/infer_context.h
+++ b/paddle/fluid/inference/api/infer_context.h
@@ -45,4 +45,15 @@ class InferGPUContext : public phi::GPUContext {
   using phi::GPUContext::SetRuntimeVersion;
 };
 #endif
+
+#if defined(PADDLE_WITH_XPU)
+class InferXPUContext : public phi::XPUContext {
+ public:
+  explicit InferXPUContext(const phi::Place& place);
+  using phi::XPUContext::SetDriverVersion;
+  using phi::XPUContext::SetRuntimeVersion;
+  using phi::XPUContext::SetStream;
+  using phi::XPUContext::SetXpuVersion;
+};
+#endif
 }  // namespace paddle
diff --git a/paddle/fluid/inference/api/paddle_api.h b/paddle/fluid/inference/api/paddle_api.h
index 3a51f91b3afc2..ee15468c9b81e 100644
--- a/paddle/fluid/inference/api/paddle_api.h
+++ b/paddle/fluid/inference/api/paddle_api.h
@@ -480,7 +480,8 @@ class PD_INFER_DECL InternalUtils {
                                     cudaStream_t stream);
   static bool RunWithExternalStream(paddle_infer::Predictor* pred,
                                     hipStream_t stream);
-
+  static bool RunWithExternalStream(paddle_infer::Predictor* pred,
+                                    void* stream);
   static void UpdateConfigInterleaved(paddle_infer::Config* c,
                                       bool with_interleaved);
 
diff --git a/paddle/fluid/inference/api/resource_manager.cc b/paddle/fluid/inference/api/resource_manager.cc
index 49d33d6750abe..0f3612a5e976c 100644
--- a/paddle/fluid/inference/api/resource_manager.cc
+++ b/paddle/fluid/inference/api/resource_manager.cc
@@ -41,6 +41,9 @@
 #include "paddle/phi/backends/dynload/cusparse.h"
 #endif  // PADDLE_WITH_CUDA
 
+#ifdef PADDLE_WITH_XPU
+#include "paddle/phi/backends/xpu/xpu_info.h"
+#endif
 namespace paddle {
 namespace internal {
 
@@ -545,4 +548,121 @@ int ResourceManager::RefCount(void* stream) const {
 }
 #endif
 
+#if defined(PADDLE_WITH_XPU)
+// XPUContextResource
+XPUContextResource::XPUContextResource(const phi::Place& place, void* stream)
+    : place_(place) {
+  InitXPUResource(stream);
+}
+
+XPUContextResource::~XPUContextResource() {}
+
+void XPUContextResource::InitXPUResource(void* stream) {
+  phi::backends::xpu::XPUDeviceGuard guard(place_.device);
+  if (stream) {
+    owned_stream_ = false;
+    stream_ = stream;
+  }
+  InitXpuProperties();
+}
+
+void XPUContextResource::InitXpuProperties() {
+  phi::backends::xpu::XPUDeviceGuard guard(place_.device);
+  driver_version_ = phi::backends::xpu::GetDriverVersion();
+  runtime_version_ = phi::backends::xpu::GetRuntimeVersion();
+  xpu_version_ =
+      static_cast<int>(phi::backends::xpu::get_xpu_version(place_.device));
+}
+void* XPUContextResource::GetStream() const { return stream_; }
+
+int XPUContextResource::GetDriverVersion() const { return driver_version_; }
+
+int XPUContextResource::GetRuntimeVersion() const { return runtime_version_; }
+
+int XPUContextResource::GetXpuVersion() const { return xpu_version_; }
+
+void XPUContextResource::ReBindStream(void* stream) {
+  owned_stream_ = false;
+  stream_ = stream;
+}
+// XPUContextResource End.
+
+// Resource Manager
+void* ResourceManager::InitXPUResource(const phi::Place& place, void* stream) {
+  std::lock_guard<std::mutex> lock_gurad(xpu_mutex_);
+  if (xpu_resources_.count(stream)) {
+    Increase(stream);
+    return stream;
+  } else {
+    std::unique_ptr<XPUContextResource> resource{
+        new XPUContextResource(place, stream)};
+    void* s = resource->GetStream();
+    ref_count_[s] = 1;
+    xpu_resources_.emplace(s, std::move(resource));
+    return s;
+  }
+}
+
+XPUContextResource* ResourceManager::GetXPUResource(void* stream) const {
+  PADDLE_ENFORCE_EQ(xpu_resources_.count(stream),
+                    true,
+                    platform::errors::InvalidArgument(
+                        "The stream[%p] not found in xpu_resources.", stream));
+  return xpu_resources_.at(stream).get();
+}
+
+void ResourceManager::XpuResourceReBindStream(void* old_stream,
+                                              void* new_stream) {
+  PADDLE_ENFORCE_EQ(
+      xpu_resources_.count(old_stream),
+      true,
+      platform::errors::InvalidArgument(
+          "The stream[%p] not found in xpu_resources.", old_stream));
+  auto xpu_resource = std::move(xpu_resources_.at(old_stream));
+  DestroyXPUResource(old_stream);
+  PADDLE_ENFORCE_EQ(
+      ref_count_.count(old_stream),
+      0,
+      platform::errors::Fatal("xpu resources rebind stream failed."));
+
+  xpu_resource->ReBindStream(new_stream);
+  ref_count_[new_stream]++;
+  xpu_resources_.emplace(new_stream, std::move(xpu_resource));
+}
+
+void ResourceManager::DestroyXPUResource(void* stream) {
+  PADDLE_ENFORCE_EQ(xpu_resources_.count(stream),
+                    true,
+                    platform::errors::InvalidArgument(
+                        "The stream[%p] not found in xpu_resources.", stream));
+  Decrease(stream);
+}
+
+void ResourceManager::Decrease(void* stream) {
+  PADDLE_ENFORCE_EQ(ref_count_.count(stream),
+                    true,
+                    platform::errors::InvalidArgument(
+                        "The stream[%p] not found in ref_count.", stream));
+  --ref_count_[stream];
+  if (ref_count_[stream] == 0) {
+    ref_count_.erase(stream);
+    xpu_resources_.erase(stream);
+  }
+}
+
+void ResourceManager::Increase(void* stream) {
+  PADDLE_ENFORCE_EQ(ref_count_.count(stream),
+                    true,
+                    platform::errors::InvalidArgument(
+                        "The stream[%p] not found in ref_count.", stream));
+  ++ref_count_[stream];
+}
+
+int ResourceManager::RefCount(void* stream) const {
+  if (ref_count_.count(stream) == 0) return 0;
+  return ref_count_.at(stream);
+}
+// Resource Manager End.
+
+#endif
 }  // namespace paddle
diff --git a/paddle/fluid/inference/api/resource_manager.h b/paddle/fluid/inference/api/resource_manager.h
index 6655324d305b6..119b5d94dc56c 100644
--- a/paddle/fluid/inference/api/resource_manager.h
+++ b/paddle/fluid/inference/api/resource_manager.h
@@ -134,6 +134,33 @@ class GPUContextResource {
 };
 #endif
 
+#if defined(PADDLE_WITH_XPU)
+class XPUContextResource {
+ public:
+  explicit XPUContextResource(const phi::Place& place, void* stream);
+  ~XPUContextResource();
+  phi::Place Place() const;
+  void* GetStream() const;
+  int GetDriverVersion() const;
+  int GetRuntimeVersion() const;
+  int GetXpuVersion() const;
+  void ReBindStream(void* stream);
+
+ private:
+  void InitXPUResource(void* stream);
+  void InitXpuProperties();
+
+ private:
+  bool owned_stream_{true};
+  void* stream_;
+  phi::Place place_;
+
+  int driver_version_;
+  int runtime_version_;
+  int xpu_version_;
+};  // class XPUContextResource
+#endif
+
 class ResourceManager {
  public:
   ResourceManager() = default;
@@ -173,6 +200,28 @@ class ResourceManager {
       gpu_resources_;
 #endif
 
+// XPU Resource
+#if defined(PADDLE_WITH_XPU)
+
+ public:
+  void* InitXPUResource(const phi::Place& place, void* stream);
+  void DestroyXPUResource(void* stream);
+  XPUContextResource* GetXPUResource(void* stream) const;
+  int RefCount(void* stream) const;
+  void XpuResourceReBindStream(void* old_stream, void* new_stream);
+
+ private:
+  void Decrease(void* stream);
+  void Increase(void* stream);
+
+ private:
+  std::mutex xpu_mutex_;
+  // a stream corresponding to a series of resource.
+  std::map<void* /*stream*/, std::atomic<int>> ref_count_;
+  std::map<void* /*stream*/, std::unique_ptr<XPUContextResource>>
+      xpu_resources_;
+#endif
+
  private:
   DISABLE_COPY_AND_ASSIGN(ResourceManager);
 };
diff --git a/paddle/phi/backends/xpu/xpu_context.cc b/paddle/phi/backends/xpu/xpu_context.cc
index b1d768f8445b8..4c0727088f662 100644
--- a/paddle/phi/backends/xpu/xpu_context.cc
+++ b/paddle/phi/backends/xpu/xpu_context.cc
@@ -76,7 +76,7 @@ struct XPUContext::Impl {
     if (owned_ && context_ != nullptr) {
       backends::xpu::XPUDeviceGuard guard(place_.GetDeviceId());
       xpu_wait(context_->xpu_stream);
-      if (context_->xpu_stream) {
+      if (context_->xpu_stream && stream_owned_) {
         // manually destroy XPUStream here until xpu::api integrates this work
         // into Context dtor
         xpu_stream_destroy(context_->xpu_stream);
@@ -111,6 +111,12 @@ struct XPUContext::Impl {
     return context_->xpu_stream;
   }
 
+  // Set external stream for context
+  void SetStream(void* stream) {
+    stream_owned_ = false;
+    context_->set_stream(static_cast<XPUStream>(stream));
+  }
+
   xpu::Context* GetXContext() const {
     PD_CHECK(context_ != nullptr, "the xpu context is nullptr.");
     return context_;
@@ -179,6 +185,7 @@ struct XPUContext::Impl {
       return;
     }
     PADDLE_ENFORCE_XPU_SUCCESS(xpu_stream_create(&context_->xpu_stream));
+    stream_owned_ = true;
   }
 
   // Methods of XPU Dataloader threads contexts map,
@@ -221,8 +228,11 @@ struct XPUContext::Impl {
   }
 
   bool owned_{false};
+  bool stream_owned_{false};
   Place place_;
   backends::xpu::XPUVersion xpu_version_;
+  int runtime_version_;
+  int driver_version_;
   xpu::Context* context_{nullptr};
   std::unordered_map<uint32_t, xpu::Context*> xdl_context_map_;
 
@@ -246,6 +256,20 @@ const Place& XPUContext::GetPlace() const { return impl_->GetPlace(); }
 
 XPUStream XPUContext::stream() const { return impl_->stream(); }
 
+void XPUContext::SetStream(void* stream) { impl_->SetStream(stream); }
+
+void XPUContext::SetXpuVersion(int version) {
+  impl_->xpu_version_ = static_cast<backends::xpu::XPUVersion>(version);
+}
+
+void XPUContext::SetRuntimeVersion(int version) {
+  impl_->runtime_version_ = version;
+}
+
+void XPUContext::SetDriverVersion(int version) {
+  impl_->driver_version_ = version;
+}
+
 backends::xpu::XPUVersion XPUContext::xpu_version() const {
   return impl_->xpu_version_;
 }
diff --git a/paddle/phi/backends/xpu/xpu_context.h b/paddle/phi/backends/xpu/xpu_context.h
index 349118f33362b..fa4899d9871e4 100644
--- a/paddle/phi/backends/xpu/xpu_context.h
+++ b/paddle/phi/backends/xpu/xpu_context.h
@@ -56,6 +56,9 @@ class XPUContext : public DeviceContext,
   void SetBkclContext(xpu::BKCLContext_t context);
   void CreateStream();
 
+  // For share external stream.
+  void SetStream(void* stream);
+
   // Wait for all operations completion in the stream.
   void Wait() const override;
 
@@ -73,6 +76,12 @@ class XPUContext : public DeviceContext,
 
   void SetL3Cache(int l3_size = 14155776);
 
+  void SetXpuVersion(int version);
+
+  void SetRuntimeVersion(int runtime_version);
+
+  void SetDriverVersion(int driver_version);
+
   Eigen::DefaultDevice* eigen_device() const { return nullptr; }
 
   XPUStream stream() const;
diff --git a/test/cpp/inference/api/analysis_predictor_tester.cc b/test/cpp/inference/api/analysis_predictor_tester.cc
index 5692d7607bba3..e6b5630dccad6 100644
--- a/test/cpp/inference/api/analysis_predictor_tester.cc
+++ b/test/cpp/inference/api/analysis_predictor_tester.cc
@@ -17,6 +17,10 @@
 #if defined(PADDLE_WITH_CUDA)
 #include <cuda_runtime.h>
 #endif
+#if defined(PADDLE_WITH_XPU)
+#include "xpu/runtime.h"
+#include "xpu/xdnn.h"
+#endif
 #include <glog/logging.h>
 #include <gtest/gtest.h>
 
@@ -654,6 +658,57 @@ TEST(Predictor, Streams) {
 }
 #endif
 
+#if defined(PADDLE_WITH_XPU)
+TEST(Predictor, XPUStreams) {
+  // external stream
+  {
+    auto context = baidu::xpu::api::create_context();
+    xpu_stream_create(&context->xpu_stream);
+
+    Config config;
+    config.SetModel(FLAGS_dirname);
+    config.EnableXpu();
+    config.SetExecStream(static_cast<void*>(context->xpu_stream));
+    CHECK_EQ(config.external_stream_enabled(), true);
+
+    auto predictor = CreatePredictor(config);
+    auto stream = predictor->GetExecStream();
+    CHECK_EQ(static_cast<void*>(context->xpu_stream), stream);
+    CHECK_NOTNULL(paddle::ResourceManager::Instance().GetXPUResource(stream));
+    CHECK_EQ(paddle::ResourceManager::Instance().RefCount(stream), 1);
+  }
+
+  // 2 predictor on 2 stream
+  {
+    auto context1 = baidu::xpu::api::create_context();
+    xpu_stream_create(&context1->xpu_stream);
+
+    Config config;
+    config.SetModel(FLAGS_dirname);
+    config.EnableXpu();
+    config.SetExecStream(static_cast<void*>(context1->xpu_stream));
+    auto predictor = CreatePredictor(config);
+    auto stream1 = predictor->GetExecStream();
+    CHECK_NOTNULL(paddle::ResourceManager::Instance().GetXPUResource(stream1));
+    CHECK_EQ(paddle::ResourceManager::Instance().RefCount(stream1), 1);
+
+    auto context2 = baidu::xpu::api::create_context();
+    xpu_stream_create(&context2->xpu_stream);
+
+    Config config2;
+    config2.SetModel(FLAGS_dirname);
+    config2.EnableXpu();
+    config2.SetExecStream(static_cast<void*>(context2->xpu_stream));
+    auto predictor2 = CreatePredictor(config2);
+    auto stream2 = predictor2->GetExecStream();
+    CHECK_NOTNULL(paddle::ResourceManager::Instance().GetXPUResource(stream2));
+    CHECK_EQ(paddle::ResourceManager::Instance().RefCount(stream2), 1);
+
+    CHECK_NE(stream1, stream2);
+  }
+}
+#endif
+
 TEST(AnalysisPredictor, OutputHookFunc) {
   auto hookfunc = [](const std::string& type,
                      const std::string& var_name,

From b7295120b0e78b293cd7ae29706e21769d06a3cc Mon Sep 17 00:00:00 2001
From: Yiqun Liu <Xreki@users.noreply.github.com>
Date: Sat, 6 May 2023 15:20:39 +0800
Subject: [PATCH 189/405] Add fused_gate_attention API. (#53432)

* Add fused_gate_attention API.

* Implement FusedDropout API.

* Fix doc and add unittest.

* Skip for non-gpu device.

* Add unittest.
---
 .../tests/unittests/test_dropout_nd_op.py     |  30 +++-
 .../unittests/test_fused_gate_attention_op.py |  93 ++++++++++
 python/paddle/incubate/nn/__init__.py         |   1 +
 .../paddle/incubate/nn/functional/__init__.py |   1 +
 .../nn/functional/fused_gate_attention.py     | 165 ++++++++++++++++++
 .../incubate/nn/layer/fused_dropout_nd.py     | 142 +++++++++++++++
 6 files changed, 430 insertions(+), 2 deletions(-)
 create mode 100644 python/paddle/incubate/nn/functional/fused_gate_attention.py
 create mode 100644 python/paddle/incubate/nn/layer/fused_dropout_nd.py

diff --git a/python/paddle/fluid/tests/unittests/test_dropout_nd_op.py b/python/paddle/fluid/tests/unittests/test_dropout_nd_op.py
index 7a36599a580f3..e2d5ae8685cc9 100644
--- a/python/paddle/fluid/tests/unittests/test_dropout_nd_op.py
+++ b/python/paddle/fluid/tests/unittests/test_dropout_nd_op.py
@@ -176,6 +176,7 @@ def test_check_grad_normal(self):
 
 class TestDropoutNdAPI(unittest.TestCase):
     def setUp(self):
+        paddle.seed(123)
         np.random.seed(123)
         self.places = [fluid.CPUPlace()]
         if core.is_compiled_with_cuda():
@@ -187,11 +188,36 @@ def test_dygraph(self):
             with fluid.dygraph.guard(place):
                 in_np = np.random.random([4, 32, 16]).astype("float32")
                 input = paddle.to_tensor(in_np)
-                res1 = dropout_nd(x=input, p=0.0, axis=[0, 1])
-                res2 = dropout_nd(x=input, p=0.5, axis=[0, 1])
+                dropout_1 = paddle.incubate.nn.FusedDropout(p=0.0, axis=[0, 1])
+                dropout_2 = paddle.incubate.nn.FusedDropout(p=0.5, axis=[0, 1])
+                print(dropout_1)
+                print(dropout_2)
+                res1 = dropout_1(input)
+                res2 = dropout_2(input)
             np.testing.assert_allclose(res1.numpy(), in_np, rtol=1e-05)
         paddle.enable_static()
 
+    def test_error(self):
+        def _run_illegal_type_p():
+            dropout = paddle.incubate.nn.FusedDropout(p="test")
+
+        self.assertRaises(TypeError, _run_illegal_type_p)
+
+        def _run_illegal_value_p():
+            dropout = paddle.incubate.nn.FusedDropout(p=2)
+
+        self.assertRaises(ValueError, _run_illegal_value_p)
+
+        def _run_illegal_mode():
+            dropout = paddle.incubate.nn.FusedDropout(p=0.5, mode="test")
+
+        self.assertRaises(ValueError, _run_illegal_mode)
+
+        def _run_illegal_type_axis():
+            dropout = paddle.incubate.nn.FusedDropout(p=0.5, axis="test")
+
+        self.assertRaises(TypeError, _run_illegal_type_axis)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py b/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py
index 70face3ce5b67..d9bc88d4b2c6d 100644
--- a/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py
+++ b/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py
@@ -28,6 +28,7 @@
 from test_sparse_attention_op import get_cuda_version
 
 import paddle
+import paddle.incubate.nn.functional as F
 from paddle import _legacy_C_ops, nn
 from paddle.fluid import core
 
@@ -425,5 +426,97 @@ def config(self):
         self.batch_size = 2
 
 
+class TestFusedGateAttentionApi(unittest.TestCase):
+    def setUp(self):
+        self.has_gating = True
+        self.batch_size = 2
+        self.msa_len = 3
+        self.res_len = 2
+        self.q_dim = 4
+        self.num_heads = 2
+        self.head_dim = 4
+        self.m_size = self.res_len
+        self.kv_dim = self.q_dim
+        self.out_dim = self.q_dim
+        self.merge_qkv = self.q_dim == self.kv_dim
+
+        self.query_shape = [
+            self.batch_size,
+            self.msa_len,
+            self.res_len,
+            self.q_dim,
+        ]
+        self.qkv_weight_shape = [3, self.num_heads, self.head_dim, self.q_dim]
+
+        self.attn_mask_shape = [
+            self.batch_size,
+            self.msa_len,
+            1,
+            1,
+            self.m_size,
+        ]
+        self.nonbatched_bias_shape = [
+            self.batch_size,
+            1,
+            self.num_heads,
+            self.res_len,
+            self.m_size,
+        ]
+
+        self.gating_w_shape = [self.q_dim, self.num_heads, self.head_dim]
+        self.gating_b_shape = [self.num_heads, self.head_dim]
+
+        self.output_w_shape = [self.num_heads, self.head_dim, self.out_dim]
+        self.output_b_shape = [self.out_dim]
+
+        self.out_shape = [
+            self.batch_size,
+            self.msa_len,
+            self.res_len,
+            self.out_dim,
+        ]
+
+    def test_api(self):
+        if not core.is_compiled_with_cuda():
+            pass
+
+        query = paddle.rand(shape=self.query_shape, dtype="float32")
+        qkv_weight = paddle.rand(shape=self.qkv_weight_shape, dtype="float32")
+
+        attn_mask = paddle.rand(shape=self.attn_mask_shape, dtype="float32")
+        nonbatched_bias = paddle.rand(
+            shape=self.nonbatched_bias_shape, dtype="float32"
+        )
+
+        gate_linear_weight = paddle.rand(
+            shape=self.gating_w_shape, dtype="float32"
+        )
+        gate_linear_bias = paddle.rand(
+            shape=self.gating_b_shape, dtype="float32"
+        )
+
+        out_linear_weight = paddle.rand(
+            shape=self.output_w_shape, dtype="float32"
+        )
+        out_linear_bias = paddle.rand(
+            shape=self.output_b_shape, dtype="float32"
+        )
+
+        output = F.fused_gate_attention(
+            query=query,
+            qkv_weight=qkv_weight,
+            gate_linear_weight=gate_linear_weight,
+            gate_linear_bias=gate_linear_bias,
+            out_linear_weight=out_linear_weight,
+            out_linear_bias=out_linear_bias,
+            nonbatched_bias=nonbatched_bias,
+            attn_mask=attn_mask,
+            has_gating=True,
+            merge_qkv=True,
+        )
+        print(f"output.shape={output.shape}")
+        self.assertEqual(output.shape, self.out_shape)
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/python/paddle/incubate/nn/__init__.py b/python/paddle/incubate/nn/__init__.py
index 3b6869f88c628..c663d6248feb0 100644
--- a/python/paddle/incubate/nn/__init__.py
+++ b/python/paddle/incubate/nn/__init__.py
@@ -22,6 +22,7 @@
 )  # noqa: F401
 from .layer.fused_ec_moe import FusedEcMoe  # noqa: F401
 from .layer.fused_dropout_add import FusedDropoutAdd  # noqa: F401
+from .layer.fused_dropout_nd import FusedDropout  # noqa: F401
 
 __all__ = [  # noqa
     'FusedMultiHeadAttention',
diff --git a/python/paddle/incubate/nn/functional/__init__.py b/python/paddle/incubate/nn/functional/__init__.py
index e5d17294329d4..ccccadd284e9e 100644
--- a/python/paddle/incubate/nn/functional/__init__.py
+++ b/python/paddle/incubate/nn/functional/__init__.py
@@ -19,6 +19,7 @@
 from .fused_transformer import fused_bias_dropout_residual_layer_norm
 from .fused_ec_moe import fused_ec_moe
 from .fused_dropout_add import fused_dropout_add
+from .fused_gate_attention import fused_gate_attention
 
 
 __all__ = [
diff --git a/python/paddle/incubate/nn/functional/fused_gate_attention.py b/python/paddle/incubate/nn/functional/fused_gate_attention.py
new file mode 100644
index 0000000000000..13833683449b5
--- /dev/null
+++ b/python/paddle/incubate/nn/functional/fused_gate_attention.py
@@ -0,0 +1,165 @@
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle import _legacy_C_ops
+from paddle.fluid.framework import _non_static_mode
+
+
+def fused_gate_attention(
+    query,
+    key=None,
+    query_weight=None,
+    key_weight=None,
+    value_weight=None,
+    qkv_weight=None,
+    gate_linear_weight=None,
+    gate_linear_bias=None,
+    out_linear_weight=None,
+    out_linear_bias=None,
+    nonbatched_bias=None,
+    attn_mask=None,
+    has_gating=True,
+    merge_qkv=True,
+    use_flash_attn=False,
+):
+    r"""
+    Attention mapps queries and a set of key-value pairs to outputs, and
+    Gate Attention performs multiple parallel attention to jointly attending
+    to information from different representation subspaces. This API only
+    support self_attention. The pseudo code is as follows:
+
+    .. code-block:: python
+
+        c = c ** (-0.5)
+        q = paddle.einsum('nbqa,ahc->nbqhc', q_data, query_w) * c
+        k = paddle.einsum('nbka,ahc->nbkhc', m_data, key_w)
+        v = paddle.einsum('nbka,ahc->nbkhc', m_data, value_w)
+        logits = paddle.einsum('nbqhc,nbkhc->nbhqk', q, k) + bias
+
+        if nonbatched_bias is not None:
+            logits += paddle.unsqueeze(nonbatched_bias, axis=1)
+
+        weights = paddle.nn.functional.softmax(logits)
+        weighted_avg = paddle.einsum('nbhqk,nbkhc->nbqhc', weights, v)
+
+        if has_gating:
+            gate_values = paddle.einsum('nbqc,chv->nbqhv', q_data, gating_w) + gating_b
+            gate_values = paddle.nn.functional.sigmoid(gate_values)
+            weighted_avg *= gate_values
+
+        output = paddle.einsum('nbqhc,hco->nbqo', weighted_avg, output_w) + output_b
+
+
+    Args:
+        query (Tensor): The input query tensor. The shape is [batch_size, msa_len, res_len, q_dim].
+        key (Tensor, optional): The input key tensor, which can be set when
+                                merge_qkv is False. The shape is [batch_size, msa_len, m_size, kv_dim].
+        query_weight (Tensor, optional): The weight of query linear, which
+                                         should be set when input key is not None. The shape is [q_dim, num_heads, head_dim].
+        key_weight (Tensor, optional): The weight of key linear, which should
+                                       be set when input key is not None. The shape is [kv_dim, num_heads, head_dim].
+        value_weight (Tensor, optional): The weight of value linear, which should
+                                         be set when input key is not None. The shape is [kv_dim, num_heads, head_dim].
+        qkv_weight (Tensor, optional): The weight of qkv linear, which should
+                                       be set when merge_qkv is True. The shape is [3, num_heads, head_dim, q_dim].
+        gate_linear_weight (Tensor, optional): The weight of gating linear,
+                                       which should be set when has_gating is True. The shape is [q_dim, num_heads, head_dim].
+        gate_linear_bias (Tensor, optional): The bias of gating linear, which
+                                             should be set when has_gating is True. The shape is [num_heads, head_dim]. Default None.
+        out_linear_weight (Tensor, optional): The weight of output linear. The shape is [num_heads, head_dim, q_dim].
+        out_linear_bias (Tensor): The bias of output linear, the shape is [q_dim]. Default None.
+        nonbatched_bias (Tensor, optional): The extra bias. The shape is [batch_size, 1, num_heads, res_len, m_size]. Default None.
+        attn_mask (Tensor, optional):  The attention mask. The shape is [batch_size, msa_len, 1, 1, res_len]. Default None.
+        has_gating (bool, optional): Whether has the gating linear. Default True.
+        merge_qkv (bool, optional): Whether has the gating linear. Default True.
+
+    Returns:
+        Tensor: The output Tensor, the data type and shape is same as `query`.
+
+    Examples:
+
+        .. code-block:: python
+
+            # required: gpu
+            import paddle
+            import paddle.incubate.nn.functional as F
+
+            # batch_size = 2
+            # msa_len = 4
+            # res_len = 2
+            # q_dim = 4
+            # num_heads = 8
+            # head_dim = 4
+            # m_size = res_len (when merge_qkv is True)
+
+            # query: [batch_size, msa_len, res_len, q_dim]
+            query = paddle.rand(shape=[2, 4, 2, 4], dtype="float32")
+
+            # qkv_weight:  [3, n_heads, head_dim, q_dim]
+            qkv_weight = paddle.rand(shape=[3, 8, 4, 4], dtype="float32")
+
+            # nonbatched_bias: [batch_size, 1, num_heads, res_len, m_size]
+            nonbatched_bias = paddle.rand(shape=[2, 1, 8, 2, 2], dtype="float32")
+
+            # attn_mask: [batch_size, msa_len, 1, 1, m_size]
+            attn_mask = paddle.rand(shape=[2, 4, 1, 1, 2], dtype="float32")
+
+            # gate_linear_weight: [q_dim, num_heads, head_dim]
+            gate_linear_weight = paddle.rand(shape=[4, 8, 4], dtype="float32")
+            # gate_bias: [num_heads, head_dim]
+            gate_linear_bias = paddle.rand(shape=[8, 4], dtype="float32")
+
+            # out_linear_weight: [num_heads, head_dim, q_dim]
+            out_linear_weight = paddle.rand(shape=[8, 4, 4], dtype="float32")
+            # out_linear_bias: [q_dim]
+            out_linear_bias = paddle.rand(shape=[4], dtype="float32")
+
+            # output: [batch_size, msa_len, res_len, q_dim]
+            output = F.fused_gate_attention(
+                query=query,
+                qkv_weight=qkv_weight,
+                gate_linear_weight=gate_linear_weight,
+                gate_linear_bias=gate_linear_bias,
+                out_linear_weight=out_linear_weight,
+                out_linear_bias=out_linear_bias,
+                nonbatched_bias=nonbatched_bias,
+                attn_mask=attn_mask,
+                has_gating=True,
+                merge_qkv=True)
+            print(output.shape)
+            # [2, 4, 2, 4]
+
+    """
+    if _non_static_mode():
+        _, _, _, _, _, _, _, out = _legacy_C_ops.fused_gate_attention(
+            query,
+            key,
+            query_weight,
+            key_weight,
+            value_weight,
+            qkv_weight,
+            nonbatched_bias,
+            attn_mask,
+            gate_linear_weight,
+            gate_linear_bias,
+            out_linear_weight,
+            out_linear_bias,
+            'has_gating',
+            has_gating,
+            'merge_qkv',
+            merge_qkv,
+            "use_flash_attn",
+            use_flash_attn,
+        )
+        return out
diff --git a/python/paddle/incubate/nn/layer/fused_dropout_nd.py b/python/paddle/incubate/nn/layer/fused_dropout_nd.py
new file mode 100644
index 0000000000000..b8aecf0c3978a
--- /dev/null
+++ b/python/paddle/incubate/nn/layer/fused_dropout_nd.py
@@ -0,0 +1,142 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+from paddle import _legacy_C_ops
+from paddle.fluid.framework import _non_static_mode
+
+
+class FusedDropout(paddle.nn.Layer):
+    r"""
+    Dropout is a regularization technique for reducing overfitting by preventing
+    neuron co-adaption during training as described in the paper:
+    `Improving neural networks by preventing co-adaptation of feature detectors <https://arxiv.org/abs/1207.0580>`_
+    The dropout operator randomly sets the outputs of some units to zero, while upscale others
+    according to the given dropout probability.
+
+    It is an optimized implementation for ``paddle.nn.Dropout``.
+
+    In dygraph mode, please use ``eval()`` to switch to evaluation mode, where dropout is disabled.
+
+    Parameters:
+        p (float|int, optional): Probability of setting units to zero. Default: 0.5
+        axis (int|list|tuple, optional): The axis along which the dropout is performed. Default: None.
+        mode(str, optional): ['upscale_in_train'(default) | 'downscale_in_infer']
+
+                               1. upscale_in_train (default), upscale the output at training time
+
+                                  - train: :math:`out = input \times \frac{mask}{(1.0 - p)}`
+                                  - inference: :math:`out = input`
+
+                               2. downscale_in_infer, downscale the output at inference
+
+                                  - train: :math:`out = input \times mask`
+                                  - inference: :math:`out = input \times (1.0 - p)`
+        name (str, optional): Name for the operation, Default: None. For more information, please refer to :ref:`api_guide_Name`.
+
+    Shape:
+        - input: N-D tensor.
+        - output: N-D tensor, the same shape as input.
+
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+
+            x = paddle.to_tensor([[1, 2, 3], [4, 5, 6]], dtype="float32")
+            m = paddle.incubate.nn.FusedDropout(p=0.5)
+
+            y_train = m(x)
+            print(y_train)
+            # Tensor(shape=[2, 3], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        [[2., 0., 6.],
+            #         [0., 0., 0.]])
+
+            m.eval()  # switch the model to test phase
+            y_test = m(x)
+            print(y_test)
+            # Tensor(shape=[2, 3], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        [[1., 2., 3.],
+            #         [4., 5., 6.]])
+    """
+
+    def __init__(self, p=0.5, axis=None, mode="upscale_in_train", name=None):
+        super().__init__()
+
+        if not isinstance(p, (float, int)):
+            raise TypeError("p argument should be a number")
+        if p < 0 or p > 1:
+            raise ValueError("p argument should between 0 and 1")
+
+        mode = (
+            'downgrade_in_infer' if mode == 'downscale_in_infer' else mode
+        )  # semantic transfer
+        if mode not in ('downscale_in_infer', 'upscale_in_train'):
+            raise ValueError(
+                "mode argument should be 'downscale_in_infer' or 'upscale_in_train'"
+            )
+
+        if axis and not isinstance(axis, (int, list, tuple)):
+            raise TypeError("datatype of axis argument should be int or list")
+
+        self.p = p
+        self.mode = mode
+        self.name = name
+
+        self.axis = None
+        if axis is not None:
+            self.axis = [axis] if isinstance(axis, int) else list(axis)
+
+    def forward(self, input):
+        # fast return for p == 0
+        if self.p == 0:
+            return input
+
+        if self.axis is not None and _non_static_mode():
+            seed = None
+            if paddle.static.default_main_program().random_seed != 0:
+                seed = paddle.static.default_main_program().random_seed
+
+            out, mask = _legacy_C_ops.dropout_nd(
+                input,
+                'dropout_prob',
+                self.p,
+                'is_test',
+                not self.training,
+                'fix_seed',
+                seed is not None,
+                'seed',
+                seed if seed is not None else 0,
+                'dropout_implementation',
+                self.mode,
+                'axis',
+                self.axis,
+            )
+        else:
+            out = paddle.nn.functional.dropout(
+                input,
+                p=self.p,
+                axis=self.axis,
+                training=self.training,
+                mode=self.mode,
+                name=self.name,
+            )
+        return out
+
+    def extra_repr(self):
+        name_str = f', name={self.name}' if self.name else ''
+        return 'p={}, axis={}, mode={}{}'.format(
+            self.p, self.axis, self.mode, name_str
+        )

From dd2860e4b7a994e5963d631a08573227cd04b99e Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Sat, 6 May 2023 15:28:14 +0800
Subject: [PATCH 190/405] API support use_flash_attn.

---
 python/paddle/incubate/nn/functional/fused_gate_attention.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/python/paddle/incubate/nn/functional/fused_gate_attention.py b/python/paddle/incubate/nn/functional/fused_gate_attention.py
index 13833683449b5..78f4abac82396 100644
--- a/python/paddle/incubate/nn/functional/fused_gate_attention.py
+++ b/python/paddle/incubate/nn/functional/fused_gate_attention.py
@@ -83,6 +83,7 @@ def fused_gate_attention(
         attn_mask (Tensor, optional):  The attention mask. The shape is [batch_size, msa_len, 1, 1, res_len]. Default None.
         has_gating (bool, optional): Whether has the gating linear. Default True.
         merge_qkv (bool, optional): Whether has the gating linear. Default True.
+        use_flash_attn (bool, optional): Whether use flash-attention to speedup. Default False.
 
     Returns:
         Tensor: The output Tensor, the data type and shape is same as `query`.
@@ -142,7 +143,7 @@ def fused_gate_attention(
 
     """
     if _non_static_mode():
-        _, _, _, _, _, _, _, out = _legacy_C_ops.fused_gate_attention(
+        _, _, _, _, _, _, _, _, out = _legacy_C_ops.fused_gate_attention(
             query,
             key,
             query_weight,

From d91d758d82870bd762b7954a5f5d1e6e548133e5 Mon Sep 17 00:00:00 2001
From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com>
Date: Sat, 6 May 2023 17:10:59 +0800
Subject: [PATCH 191/405] [IR] OpTrait & OpInterface & OpInfo (#52846)

* add OpTrait OpInterface ValueIterator TypeList

* refine code

* refine code

* refine code

* add opinfo

* add typeid copy constructor

* add trait interface construct method for opinfo_impl

* add trait interface construct method for opinfo_impl

* add trait interface construct method for opinfo_impl

* add trait interface construct method for opinfo_impl

* add trait interface construct method for opinfo_impl

* add create

* add member func for opinfo

* fix compile bug

* add op interface in ircontext

* fix compile bug

* fix compile bug

* refine code

* fix compile bug

* add ut

* refine ut

* refine code of opinfo_impl

* delete unused code

* add dyncast for operation

* refine comment

* refine opinfo_impl

* delete unused code

* refine code by comment

* refine code

* refine code

* refine code for registerOp

* refine opfin create

* refine code of search method of ircontext

* refine op attribute

* change opinfo_map key from type_id to string
---
 paddle/ir/builtin_attribute_storage.cc |  23 +-
 paddle/ir/dialect.cc                   |   4 +
 paddle/ir/dialect.h                    |  57 +++--
 paddle/ir/ir_context.cc                |  58 ++++-
 paddle/ir/ir_context.h                 |  25 ++-
 paddle/ir/op_base.h                    |  50 ++++-
 paddle/ir/op_info.h                    |  67 ++++++
 paddle/ir/op_info_impl.h               | 282 +++++++++++++++++++++++++
 paddle/ir/operation.cc                 |   9 +-
 paddle/ir/operation.h                  |  57 ++++-
 paddle/ir/type_id.h                    |   9 +
 paddle/ir/utils.h                      |  95 +++++++++
 paddle/ir/value.cc                     |   6 +
 paddle/ir/value.h                      |  41 ++++
 test/cpp/ir/CMakeLists.txt             |   1 +
 test/cpp/ir/ir_op_test.cc              | 153 ++++++++++++++
 test/cpp/ir/ir_value_test.cc           |  34 ++-
 17 files changed, 898 insertions(+), 73 deletions(-)
 create mode 100644 paddle/ir/op_info.h
 create mode 100644 paddle/ir/op_info_impl.h
 create mode 100644 test/cpp/ir/ir_op_test.cc

diff --git a/paddle/ir/builtin_attribute_storage.cc b/paddle/ir/builtin_attribute_storage.cc
index c7feacae4d64a..3f785d20c9b92 100644
--- a/paddle/ir/builtin_attribute_storage.cc
+++ b/paddle/ir/builtin_attribute_storage.cc
@@ -59,19 +59,16 @@ DictionaryAttributeStorage::ParamKey DictionaryAttributeStorage::GetAsKey()
 }
 
 Attribute DictionaryAttributeStorage::GetValue(const StrAttribute &name) const {
-  if (size_ > 0) {
-    size_t left = 0;
-    size_t right = size_ - 1;
-    size_t mid = 0;
-    while (left <= right) {
-      mid = (left + right) / 2;
-      if (data_[mid].name() == name) {
-        return data_[mid].value();
-      } else if (data_[mid].name() < name) {
-        left = mid + 1;
-      } else {
-        right = mid - 1;
-      }
+  size_t left = 0;
+  size_t right = size_;
+  while (left < right) {
+    size_t mid = left + (right - left) / 2;
+    if (data_[mid].name() == name) {
+      return data_[mid].value();
+    } else if (data_[mid].name() < name) {
+      left = mid + 1;
+    } else {
+      right = mid;
     }
   }
   return nullptr;
diff --git a/paddle/ir/dialect.cc b/paddle/ir/dialect.cc
index cda00baadc3cb..cdc171557148a 100644
--- a/paddle/ir/dialect.cc
+++ b/paddle/ir/dialect.cc
@@ -31,4 +31,8 @@ void Dialect::RegisterAttribute(ir::AbstractAttribute &&abstract_attribute) {
   this->ir_context()->RegisterAbstractAttribute(
       new_abstract_attribute->type_id(), new_abstract_attribute);
 }
+
+void Dialect::RegisterOp(const std::string &name, OpInfoImpl *op_info) {
+  this->ir_context()->RegisterOpInfo(name, op_info);
+}
 }  // namespace ir
diff --git a/paddle/ir/dialect.h b/paddle/ir/dialect.h
index 40a5a823e3dff..10adc6b3e2e0b 100644
--- a/paddle/ir/dialect.h
+++ b/paddle/ir/dialect.h
@@ -16,6 +16,7 @@
 
 #include "paddle/ir/attribute_base.h"
 #include "paddle/ir/ir_context.h"
+#include "paddle/ir/op_info_impl.h"
 #include "paddle/ir/type_base.h"
 
 namespace ir {
@@ -45,17 +46,19 @@ class Dialect {
     (void)std::initializer_list<int>{0, (RegisterType<Args>(), 0)...};
   }
 
-  ///
-  /// \brief Register type of class T.
-  ///
   template <typename T>
   void RegisterType() {
     VLOG(4) << "Type registered into Dialect. --->";
-    ir::AbstractType *abstract_type =
-        new ir::AbstractType(std::move(ir::AbstractType::get<T>(*this)));
-    this->ir_context()->RegisterAbstractType(ir::TypeId::get<T>(),
-                                             abstract_type);
-    ir::TypeManager::RegisterType<T>(this->ir_context());
+    // if (this->ir_context()->registed_abstract_type().count(
+    //         ir::TypeId::get<T>()) == 0) {
+    if (this->ir_context()->GetRegisteredAbstractType(ir::TypeId::get<T>()) ==
+        nullptr) {
+      ir::AbstractType *abstract_type =
+          new ir::AbstractType(std::move(ir::AbstractType::get<T>(*this)));
+      this->ir_context()->RegisterAbstractType(ir::TypeId::get<T>(),
+                                               abstract_type);
+      ir::TypeManager::RegisterType<T>(this->ir_context());
+    }
     VLOG(4) << "----------------------------------";
   }
 
@@ -78,24 +81,42 @@ class Dialect {
     (void)std::initializer_list<int>{0, (RegisterAttribute<Args>(), 0)...};
   }
 
-  ///
-  /// \brief Register attribute of class T.
-  ///
   template <typename T>
   void RegisterAttribute() {
     VLOG(4) << "Attribute registered into Dialect. --->";
-    ir::AbstractAttribute *abstract_attribute = new ir::AbstractAttribute(
-        std::move(ir::AbstractAttribute::get<T>(*this)));
-    this->ir_context()->RegisterAbstractAttribute(ir::TypeId::get<T>(),
-                                                  abstract_attribute);
-    ir::AttributeManager::RegisterAttribute<T>(this->ir_context());
+    if (this->ir_context()->GetRegisteredAbstractAttribute(
+            ir::TypeId::get<T>()) == nullptr) {
+      ir::AbstractAttribute *abstract_attribute = new ir::AbstractAttribute(
+          std::move(ir::AbstractAttribute::get<T>(*this)));
+      this->ir_context()->RegisterAbstractAttribute(ir::TypeId::get<T>(),
+                                                    abstract_attribute);
+      ir::AttributeManager::RegisterAttribute<T>(this->ir_context());
+    }
     VLOG(4) << "----------------------------------";
   }
 
+  void RegisterAttribute(ir::AbstractAttribute &&abstract_attribute);
+
   ///
-  /// \brief Register abstract_attribute into context.
+  /// \brief Register Operation methods.
   ///
-  void RegisterAttribute(ir::AbstractAttribute &&abstract_attribute);
+  template <typename... Args>
+  void RegisterOps() {
+    (void)std::initializer_list<int>{0, (RegisterOp<Args>(), 0)...};
+  }
+
+  template <typename ConcertOp>
+  void RegisterOp() {
+    std::string name = this->name() + "." + std::string(ConcertOp::name());
+    VLOG(4) << "Op " << name << " registered into Dialect. --->";
+    if (this->ir_context()->GetRegisteredOpInfo(name) == nullptr) {
+      ir::OpInfoImpl *op_info = ir::OpInfoImpl::create<ConcertOp>(this);
+      this->ir_context()->RegisterOpInfo(name, op_info);
+    }
+    VLOG(4) << "----------------------------------";
+  }
+
+  void RegisterOp(const std::string &name, OpInfoImpl *op_info);
 
  private:
   std::string name_;
diff --git a/paddle/ir/ir_context.cc b/paddle/ir/ir_context.cc
index 17098ccb5e556..d172b748cddae 100644
--- a/paddle/ir/ir_context.cc
+++ b/paddle/ir/ir_context.cc
@@ -20,6 +20,7 @@
 #include "paddle/ir/builtin_dialect.h"
 #include "paddle/ir/builtin_type.h"
 #include "paddle/ir/dialect.h"
+#include "paddle/ir/op_info_impl.h"
 #include "paddle/ir/spin_lock.h"
 #include "paddle/ir/type_base.h"
 
@@ -46,6 +47,11 @@ class IrContextImpl {
       delete dialect_map.second;
     }
     registed_dialect_.clear();
+
+    for (auto &op_map : registed_op_infos_) {
+      op_map.second->destroy();
+    }
+    registed_op_infos_.clear();
   }
 
   void RegisterAbstractType(ir::TypeId type_id, AbstractType *abstract_type) {
@@ -93,6 +99,25 @@ class IrContextImpl {
     return nullptr;
   }
 
+  void RegisterOpInfo(const std::string &name, OpInfoImpl *opinfo) {
+    std::lock_guard<ir::SpinLock> guard(registed_op_infos_lock_);
+    VLOG(4) << "Register an operation of: [Name=" << name
+            << ", OpInfoImpl ptr=" << opinfo << "].";
+    registed_op_infos_.emplace(name, opinfo);
+  }
+
+  OpInfoImpl *GetOpInfo(const std::string &name) {
+    std::lock_guard<ir::SpinLock> guard(registed_op_infos_lock_);
+    auto iter = registed_op_infos_.find(name);
+    if (iter != registed_op_infos_.end()) {
+      VLOG(4) << "Fonund a cached operation of: [name=" << name
+              << ", OpInfoImpl ptr=" << iter->second << "].";
+      return iter->second;
+    }
+    LOG(WARNING) << "No cache found operation of: [Name=" << name << "].";
+    return nullptr;
+  }
+
   void RegisterDialect(std::string name, Dialect *dialect) {
     std::lock_guard<ir::SpinLock> guard(registed_dialect_lock_);
     VLOG(4) << "Register a dialect of: [name=" << name
@@ -135,6 +160,10 @@ class IrContextImpl {
   std::unordered_map<std::string, Dialect *> registed_dialect_;
   ir::SpinLock registed_dialect_lock_;
 
+  // The Op registered in the context.
+  std::unordered_map<std::string, OpInfoImpl *> registed_op_infos_;
+  ir::SpinLock registed_op_infos_lock_;
+
   ir::SpinLock destructor_lock_;
 };
 
@@ -165,9 +194,12 @@ StorageManager &IrContext::type_storage_manager() {
   return impl().registed_type_storage_manager_;
 }
 
-std::unordered_map<TypeId, AbstractType *>
-    &IrContext::registed_abstracted_type() {
-  return impl().registed_abstract_types_;
+AbstractType *IrContext::GetRegisteredAbstractType(TypeId id) {
+  auto search = impl().registed_abstract_types_.find(id);
+  if (search != impl().registed_abstract_types_.end()) {
+    return search->second;
+  }
+  return nullptr;
 }
 
 void IrContext::RegisterAbstractAttribute(
@@ -179,9 +211,12 @@ StorageManager &IrContext::attribute_storage_manager() {
   return impl().registed_attribute_storage_manager_;
 }
 
-std::unordered_map<TypeId, AbstractAttribute *>
-    &IrContext::registed_abstracted_attribute() {
-  return impl().registed_abstract_attributes_;
+AbstractAttribute *IrContext::GetRegisteredAbstractAttribute(TypeId id) {
+  auto search = impl().registed_abstract_attributes_.find(id);
+  if (search != impl().registed_abstract_attributes_.end()) {
+    return search->second;
+  }
+  return nullptr;
 }
 
 Dialect *IrContext::GetOrRegisterDialect(
@@ -216,6 +251,17 @@ Dialect *IrContext::GetRegisteredDialect(const std::string &dialect_name) {
   return nullptr;
 }
 
+OpInfoImpl *IrContext::GetRegisteredOpInfo(const std::string &name) {
+  OpInfoImpl *rtn = impl().GetOpInfo(name);
+  return rtn ? rtn : nullptr;
+}
+
+void IrContext::RegisterOpInfo(const std::string &name, OpInfoImpl *opinfo) {
+  if (impl().GetOpInfo(name) == nullptr) {
+    impl().RegisterOpInfo(name, opinfo);
+  }
+}
+
 const AbstractType &AbstractType::lookup(TypeId type_id, IrContext *ctx) {
   auto &impl = ctx->impl();
   AbstractType *abstract_type = impl.GetAbstractType(type_id);
diff --git a/paddle/ir/ir_context.h b/paddle/ir/ir_context.h
index f7512344888f6..efa03dd569536 100644
--- a/paddle/ir/ir_context.h
+++ b/paddle/ir/ir_context.h
@@ -26,6 +26,7 @@ class AbstractType;
 class AbstractAttribute;
 class TypeId;
 class Dialect;
+class OpInfoImpl;
 
 ///
 /// \brief IrContext is a global parameterless class used to store and manage
@@ -47,7 +48,7 @@ class IrContext {
   IrContextImpl &impl() { return *impl_; }
 
   ///
-  /// \brief Register an AbstractType to IrContext
+  /// \brief Register an AbstractType to IrContext.
   ///
   /// \param type_id The type id of the AbstractType.
   /// \param abstract_type AbstractType* provided by user.
@@ -64,13 +65,9 @@ class IrContext {
   StorageManager &type_storage_manager();
 
   ///
-  /// \brief Returns the storage uniquer used for constructing TypeStorage
-  /// instances.
-  ///
-  /// \return The storage uniquer used for constructing TypeStorage
-  /// instances.
+  /// \brief Get registered AbstractType from IrContext.
   ///
-  std::unordered_map<TypeId, AbstractType *> &registed_abstracted_type();
+  AbstractType *GetRegisteredAbstractType(TypeId id);
 
   ///
   /// \brief Register an AbstractAttribute to IrContext
@@ -91,14 +88,16 @@ class IrContext {
   StorageManager &attribute_storage_manager();
 
   ///
-  /// \brief Returns the storage uniquer used for constructing AttributeStorage
-  /// instances.
+  /// \brief Get registered AbstractAttribute from IrContext.
   ///
-  /// \return The storage uniquer used for constructing AttributeStorage
-  /// instances.
+  AbstractAttribute *GetRegisteredAbstractAttribute(TypeId id);
+
+  ///
+  /// \brief Get or register operaiton.
   ///
-  std::unordered_map<TypeId, AbstractAttribute *>
-      &registed_abstracted_attribute();
+  void RegisterOpInfo(const std::string &name, OpInfoImpl *opinfo);
+
+  OpInfoImpl *GetRegisteredOpInfo(const std::string &name);
 
   ///
   /// \brief Get the dialect of the DialectT class in the context, ff not found,
diff --git a/paddle/ir/op_base.h b/paddle/ir/op_base.h
index 38ff4002c6b2b..23318f5e69c47 100644
--- a/paddle/ir/op_base.h
+++ b/paddle/ir/op_base.h
@@ -15,23 +15,59 @@
 #pragma once
 
 #include "paddle/ir/operation.h"
+#include "paddle/ir/utils.h"
 
 namespace ir {
 class OpBase {
  public:
-  Operation *operation() { return operation_; }
+  explicit OpBase(const Operation *operation) : operation_(operation) {}
 
-  explicit operator bool() { return operation() != nullptr; }
+  const Operation *operation() const { return operation_; }
 
-  operator Operation *() const { return operation_; }
+  explicit operator bool() const { return operation() != nullptr; }
 
-  Operation *operator->() const { return operation_; }
+  operator const Operation *() const { return operation_; }
 
- protected:
-  explicit OpBase(Operation *operation) : operation_(operation) {}
+  const Operation *operator->() const { return operation_; }
 
  private:
-  Operation *operation_;
+  const Operation *operation_;  // Not owned
+};
+
+///
+/// \brief OpTrait
+///
+template <class ConcreteTrait>
+class OpTraitBase : public OpBase {
+ public:
+  explicit OpTraitBase(const Operation *op) : OpBase(op) {}
+
+  static TypeId GetTraitId() { return TypeId::get<ConcreteTrait>(); }
+};
+
+///
+/// \brief OpInterface
+///
+template <typename ConcreteInterface>
+class OpInterfaceBase : public OpBase {
+ public:
+  // explicit OpInterfaceBase(Operation *op) : OpBase(op) {}
+
+  explicit OpInterfaceBase(const Operation *op) : OpBase(op) {}
+
+  static TypeId GetInterfaceId() { return TypeId::get<ConcreteInterface>(); }
+};
+
+template <typename ConcreteOp, class... TraitOrInterface>
+class Op : public OpBase {
+ public:
+  using OpBase::OpBase;
+
+  using TraitList =
+      typename Filter<OpTraitBase, std::tuple<TraitOrInterface...>>::Type;
+
+  using InterfaceList =
+      typename Filter<OpInterfaceBase, std::tuple<TraitOrInterface...>>::Type;
 };
 
 }  // namespace ir
diff --git a/paddle/ir/op_info.h b/paddle/ir/op_info.h
new file mode 100644
index 0000000000000..2828f59c2942a
--- /dev/null
+++ b/paddle/ir/op_info.h
@@ -0,0 +1,67 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <functional>
+
+#include "paddle/ir/op_info_impl.h"
+
+namespace ir {
+class OpInfo {
+ public:
+  constexpr OpInfo() = default;
+
+  OpInfo(const OpInfoImpl *impl) : impl_(impl) {}  // NOLINT
+
+  OpInfo(const OpInfo &other) = default;
+
+  OpInfo &operator=(const OpInfo &other) = default;
+
+  bool operator==(OpInfo other) const { return impl_ == other.impl_; }
+
+  bool operator!=(OpInfo other) const { return impl_ != other.impl_; }
+
+  explicit operator bool() const { return impl_; }
+
+  bool operator!() const { return impl_ == nullptr; }
+
+  const OpInfoImpl *impl() const { return impl_; }
+
+  template <typename Trait>
+  bool HasTrait() const {
+    return impl_->HasTrait<Trait>();
+  }
+
+  template <typename Interface>
+  bool HasInterface() const {
+    return impl_->HasInterface<Interface>();
+  }
+
+  friend struct std::hash<OpInfo>;
+
+ private:
+  const OpInfoImpl *impl_{nullptr};  // not owned
+};
+
+}  // namespace ir
+
+namespace std {
+template <>
+struct hash<ir::OpInfo> {
+  std::size_t operator()(const ir::OpInfo &obj) const {
+    return std::hash<const ir::OpInfoImpl *>()(obj.impl_);
+  }
+};
+}  // namespace std
diff --git a/paddle/ir/op_info_impl.h b/paddle/ir/op_info_impl.h
new file mode 100644
index 0000000000000..78ca586d9d209
--- /dev/null
+++ b/paddle/ir/op_info_impl.h
@@ -0,0 +1,282 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <algorithm>
+#include <cstring>
+#include <initializer_list>
+#include <utility>
+
+#include "paddle/ir/builtin_attribute.h"
+// #include "paddle/ir/ir_context.h"
+#include "paddle/ir/type.h"
+
+namespace ir {
+class Dialect;
+///
+/// \brief Tool template class for construct interfaces or Traits.
+///
+template <typename ConcreteOp, typename... Args>
+class ConstructInterfacesOrTraits {
+ public:
+  /// Construct method for interfaces.
+  static std::pair<TypeId, void *> *interface(
+      std::pair<TypeId, void *> *p_interface) {
+    (void)std::initializer_list<int>{
+        0, (PlacementConstrctInterface<Args>(p_interface), 0)...};
+    return p_interface;
+  }
+
+  /// Construct method for traits.
+  static TypeId *trait(TypeId *p_trait) {
+    (void)std::initializer_list<int>{
+        0, (PlacementConstrctTrait<Args>(p_trait), 0)...};
+    return p_trait;
+  }
+
+ private:
+  /// Placement new interface.
+  template <typename T>
+  static void PlacementConstrctInterface(
+      std::pair<TypeId, void *> *&p_interface) {  // NOLINT
+    new (&(p_interface->first)) TypeId(ir::TypeId::get<T>());
+    p_interface->second =
+        malloc(sizeof(typename T::template Model<ConcreteOp>));
+    new (p_interface->second) typename T::template Model<ConcreteOp>();
+    VLOG(4) << "New a interface: id[" << p_interface->first.storage()
+            << "], interface[" << p_interface->second << "].";
+    ++p_interface;
+  }
+
+  /// Placement new trait.
+  template <typename T>
+  static void PlacementConstrctTrait(ir::TypeId *&p_trait) {  // NOLINT
+    new (p_trait) TypeId(ir::TypeId::get<T>());
+    VLOG(4) << "New a trait: id[" << (*p_trait).storage() << "].";
+    ++p_trait;
+  }
+};
+
+/// Specialized for tuple type.
+template <typename ConcreteOp, typename... Args>
+class ConstructInterfacesOrTraits<ConcreteOp, std::tuple<Args...>> {
+ public:
+  /// Construct method for interfaces.
+  static std::pair<TypeId, void *> *interface(
+      std::pair<TypeId, void *> *p_interface) {
+    return ConstructInterfacesOrTraits<ConcreteOp, Args...>::interface(
+        p_interface);
+  }
+
+  /// Construct method for traits.
+  static TypeId *trait(TypeId *p_trait) {
+    return ConstructInterfacesOrTraits<ConcreteOp, Args...>::trait(p_trait);
+  }
+};
+
+///
+/// \brief OpInfoImpl class.
+///
+class OpInfoImpl {
+ public:
+  ///
+  /// \brief Construct and Deconstruct OpInfoImpl. The memory layout of
+  /// OpInfoImpl is: std::pair<TypeId, void *>... | TypeId... | OpInfoImpl
+  ///
+  template <typename ConcreteOp>
+  static OpInfoImpl *create(ir::Dialect *dialect) {
+    // (1) Malloc memory for interfaces, traits, opinfo_impl.
+    size_t interfaces_num =
+        std::tuple_size<typename ConcreteOp::InterfaceList>::value;
+    size_t traits_num = std::tuple_size<typename ConcreteOp::TraitList>::value;
+    size_t attributes_num = ConcreteOp::attributes_num();
+    VLOG(4) << "Create OpInfoImpl with: " << interfaces_num << " interfaces, "
+            << traits_num << " traits, " << attributes_num << " attributes.";
+    size_t base_size = sizeof(std::pair<ir::TypeId, void *>) * interfaces_num +
+                       sizeof(ir::TypeId) * traits_num + sizeof(OpInfoImpl);
+    void *base_ptr = malloc(base_size);
+    VLOG(4) << "Malloc " << base_size << " Bytes at " << base_ptr;
+
+    // (2) Construct interfaces and sort by TypeId.
+    std::pair<ir::TypeId, void *> *p_first_interface = nullptr;
+    if (interfaces_num > 0) {
+      p_first_interface =
+          reinterpret_cast<std::pair<ir::TypeId, void *> *>(base_ptr);
+      VLOG(4) << "Construct interfaces at " << p_first_interface << " ......";
+      ConstructInterfacesOrTraits<
+          ConcreteOp,
+          typename ConcreteOp::InterfaceList>::interface(p_first_interface);
+      std::sort(p_first_interface, p_first_interface + interfaces_num);
+      base_ptr = reinterpret_cast<void *>(p_first_interface + interfaces_num);
+    }
+
+    // (3) Construct traits and sort by TypeId.
+    ir::TypeId *p_first_trait = nullptr;
+    if (traits_num > 0) {
+      p_first_trait = reinterpret_cast<ir::TypeId *>(base_ptr);
+      VLOG(4) << "Construct traits at " << p_first_trait << " ......";
+      ConstructInterfacesOrTraits<ConcreteOp, typename ConcreteOp::TraitList>::
+          trait(p_first_trait);
+      std::sort(p_first_trait, p_first_trait + traits_num);
+      base_ptr = reinterpret_cast<void *>(p_first_trait + traits_num);
+    }
+
+    // (4) Construct opinfo_impl.
+    OpInfoImpl *p_opinfo_impl = reinterpret_cast<OpInfoImpl *>(base_ptr);
+    VLOG(4) << "Construct op_info_impl at " << p_opinfo_impl << " ......";
+    OpInfoImpl *op_info =
+        new (p_opinfo_impl) OpInfoImpl(interfaces_num,
+                                       traits_num,
+                                       ConcreteOp::attributes_name_,
+                                       attributes_num,
+                                       ir::TypeId::get<ConcreteOp>(),
+                                       ConcreteOp::name(),
+                                       dialect);
+    return op_info;
+  }
+
+  void destroy() {
+    VLOG(4) << "Destroy op_info impl at " << this;
+    // (1) free interfaces
+    void *base_ptr = reinterpret_cast<void *>(
+        reinterpret_cast<char *>(this) - sizeof(ir::TypeId) * num_traits_ -
+        sizeof(std::pair<ir::TypeId, void *>) * num_interfaces_);
+    if (num_interfaces_ > 0) {
+      std::pair<ir::TypeId, void *> *p_first_interface =
+          reinterpret_cast<std::pair<ir::TypeId, void *> *>(base_ptr);
+      for (size_t i = 0; i < num_interfaces_; i++) {
+        free((p_first_interface + i)->second);
+      }
+    }
+    // (2) free memeory
+    VLOG(4) << "Free base_ptr " << base_ptr;
+    free(base_ptr);
+  }
+
+  ///
+  /// \brief Search methods for Trait or Interface.
+  ///
+  template <typename Trait>
+  bool HasTrait() const {
+    return HasTrait(TypeId::get<Trait>());
+  }
+
+  bool HasTrait(TypeId trait_id) const {
+    if (num_traits_ > 0) {
+      TypeId *p_first_trait = reinterpret_cast<TypeId *>(
+          reinterpret_cast<char *>(const_cast<OpInfoImpl *>(this)) -
+          sizeof(ir::TypeId) * num_traits_);
+      return std::binary_search(
+          p_first_trait, p_first_trait + num_traits_, trait_id);
+    }
+    return false;
+  }
+
+  template <typename Interface>
+  bool HasInterface() const {
+    return HasInterface(TypeId::get<Interface>());
+  }
+
+  bool HasInterface(TypeId interface_id) const {
+    if (num_interfaces_ > 0) {
+      std::pair<ir::TypeId, void *> *p_first_interface =
+          reinterpret_cast<std::pair<ir::TypeId, void *> *>(
+              reinterpret_cast<char *>(const_cast<OpInfoImpl *>(this)) -
+              sizeof(ir::TypeId) * num_traits_ -
+              sizeof(std::pair<ir::TypeId, void *>) * num_interfaces_);
+      return std::binary_search(p_first_interface,
+                                p_first_interface + num_interfaces_,
+                                std::make_pair(interface_id, nullptr),
+                                CompareInterface);
+    }
+    return false;
+  }
+
+  template <typename Interface>
+  typename Interface::Concept *GetInterfaceImpl() const {
+    if (num_interfaces_ > 0) {
+      ir::TypeId interface_id = ir::TypeId::get<Interface>();
+      std::pair<ir::TypeId, void *> *p_first_interface =
+          reinterpret_cast<std::pair<ir::TypeId, void *> *>(
+              reinterpret_cast<char *>(const_cast<OpInfoImpl *>(this)) -
+              sizeof(ir::TypeId) * num_traits_ -
+              sizeof(std::pair<ir::TypeId, void *>) * num_interfaces_);
+      size_t left = 0;
+      size_t right = num_interfaces_;
+      while (left < right) {
+        size_t mid = left + (right - left) / 2;
+        if ((p_first_interface + mid)->first == interface_id) {
+          return reinterpret_cast<typename Interface::Concept *>(
+              (p_first_interface + mid)->second);
+        } else if ((p_first_interface + mid)->first < interface_id) {
+          left = mid + 1;
+        } else {
+          right = mid;
+        }
+      }
+    }
+    return nullptr;
+  }
+
+  ir::TypeId id() const { return op_id_; }
+
+  const char *name() const { return op_name_; }
+
+  ir::Dialect *dialect() const { return dialect_; }
+
+ private:
+  OpInfoImpl(uint32_t num_interfaces,
+             uint32_t num_traits,
+             const char **p_attributes,
+             uint32_t num_attributes,
+             TypeId op_id,
+             const char *op_name,
+             ir::Dialect *dialect)
+      : num_interfaces_(num_interfaces),
+        num_traits_(num_traits),
+        p_attributes_(p_attributes),
+        num_attributes_(num_attributes),
+        op_id_(op_id),
+        op_name_(op_name),
+        dialect_(dialect) {}
+
+  static bool CompareInterface(const std::pair<ir::TypeId, void *> &a,
+                               const std::pair<ir::TypeId, void *> &b) {
+    return a.first < b.first;
+  }
+
+  /// Interface will be recorded by std::pair<TypeId, void*>.
+  uint32_t num_interfaces_ = 0;
+
+  /// Trait will be recorded by TypeId.
+  uint32_t num_traits_ = 0;
+
+  /// Attributes array address.
+  const char **p_attributes_{nullptr};
+
+  /// The number of attributes for this Op.
+  uint32_t num_attributes_ = 0;
+
+  /// The TypeId of this Op.
+  TypeId op_id_;
+
+  /// The name of this Op.
+  const char *op_name_;
+
+  /// The dialect of this Op belong to.
+  ir::Dialect *dialect_;
+};
+
+}  // namespace ir
diff --git a/paddle/ir/operation.cc b/paddle/ir/operation.cc
index e9d727f1b5fb3..9222ee7e1afbb 100644
--- a/paddle/ir/operation.cc
+++ b/paddle/ir/operation.cc
@@ -21,7 +21,8 @@ namespace ir {
 // OpInlineResult, Operation, Operand.
 Operation *Operation::create(const std::vector<ir::OpResult> &inputs,
                              const std::vector<ir::Type> &output_types,
-                             ir::DictionaryAttribute attribute) {
+                             ir::DictionaryAttribute attribute,
+                             ir::OpInfo op_info) {
   // 1. Calculate the required memory size for OpResults + Operation +
   // OpOperands.
   uint32_t num_results = output_types.size();
@@ -52,7 +53,7 @@ Operation *Operation::create(const std::vector<ir::OpResult> &inputs,
   }
   // 3.2. Construct Operation.
   Operation *op =
-      new (base_ptr) Operation(num_results, num_operands, attribute);
+      new (base_ptr) Operation(num_results, num_operands, attribute, op_info);
   base_ptr += sizeof(Operation);
   // 3.3. Construct OpOperands.
   if ((reinterpret_cast<uintptr_t>(base_ptr) & 0x7) != 0) {
@@ -116,13 +117,15 @@ void Operation::destroy() {
 
 Operation::Operation(uint32_t num_results,
                      uint32_t num_operands,
-                     ir::DictionaryAttribute attribute) {
+                     ir::DictionaryAttribute attribute,
+                     ir::OpInfo op_info) {
   if (!attribute) {
     throw("unexpected null attribute dictionary");
   }
   num_results_ = num_results;
   num_operands_ = num_operands;
   attribute_ = attribute;
+  op_info_ = op_info;
 }
 
 ir::OpResult Operation::GetResultByIndex(uint32_t index) {
diff --git a/paddle/ir/operation.h b/paddle/ir/operation.h
index 924dcafb73dfc..a51043ad5687e 100644
--- a/paddle/ir/operation.h
+++ b/paddle/ir/operation.h
@@ -15,10 +15,15 @@
 #pragma once
 
 #include "paddle/ir/builtin_attribute.h"
+#include "paddle/ir/op_info.h"
 #include "paddle/ir/type.h"
 #include "paddle/ir/value_impl.h"
 
 namespace ir {
+template <class ConcreteTrait>
+class OpTraitBase;
+template <typename ConcreteInterface>
+class OpInterfaceBase;
 
 class alignas(8) Operation final {
  public:
@@ -28,7 +33,8 @@ class alignas(8) Operation final {
   ///
   static Operation *create(const std::vector<ir::OpResult> &inputs,
                            const std::vector<ir::Type> &output_types,
-                           ir::DictionaryAttribute attribute);
+                           ir::DictionaryAttribute attribute,
+                           ir::OpInfo op_info);
 
   void destroy();
 
@@ -36,19 +42,60 @@ class alignas(8) Operation final {
 
   std::string print();
 
-  ir::DictionaryAttribute attribute() { return attribute_; }
+  ir::DictionaryAttribute attribute() const { return attribute_; }
 
-  uint32_t num_results() { return num_results_; }
+  ir::OpInfo op_info() const { return op_info_; }
 
-  uint32_t num_operands() { return num_operands_; }
+  uint32_t num_results() const { return num_results_; }
+
+  uint32_t num_operands() const { return num_operands_; }
+
+  template <typename T>
+  T dyn_cast() const {
+    return CastUtil<T>::call(this);
+  }
+
+  template <typename Trait>
+  bool HasTrait() const {
+    return op_info_.HasTrait<Trait>();
+  }
+
+  template <typename Interface>
+  bool HasInterface() const {
+    return op_info_.HasInterface<Interface>();
+  }
 
  private:
   Operation(uint32_t num_results,
             uint32_t num_operands,
-            ir::DictionaryAttribute attribute);
+            ir::DictionaryAttribute attribute,
+            ir::OpInfo op_info);
+
+  template <typename T, typename Enabler = void>
+  struct CastUtil {
+    static T call(const Operation *op) {
+      throw("Can't dyn_cast to T, T should be a Trait or Interface");
+    }
+  };
+  template <typename T>
+  struct CastUtil<T,
+                  typename std::enable_if<
+                      std::is_base_of<OpTraitBase<T>, T>::value>::type> {
+    static T call(const Operation *op) { return T(op); }
+  };
+  template <typename T>
+  struct CastUtil<T,
+                  typename std::enable_if<
+                      std::is_base_of<OpInterfaceBase<T>, T>::value>::type> {
+    static T call(const Operation *op) {
+      return T(op, op->op_info_.impl()->GetInterfaceImpl<T>());
+    }
+  };
 
   ir::DictionaryAttribute attribute_;
 
+  ir::OpInfo op_info_;
+
   uint32_t num_results_ = 0;
 
   uint32_t num_operands_ = 0;
diff --git a/paddle/ir/type_id.h b/paddle/ir/type_id.h
index b7a2dcd362d01..33cd09b5d4a78 100644
--- a/paddle/ir/type_id.h
+++ b/paddle/ir/type_id.h
@@ -45,6 +45,12 @@ class TypeId {
     return TypeId(&instance);
   }
 
+  TypeId(const TypeId &other) = default;
+
+  TypeId &operator=(const TypeId &other) = default;
+
+  const Storage *storage() const { return storage_; }
+
   ///
   /// \brief Comparison operations.
   ///
@@ -54,6 +60,9 @@ class TypeId {
   inline bool operator!=(const TypeId &other) const {
     return !(*this == other);
   }
+  inline bool operator<(const TypeId &other) const {
+    return storage_ < other.storage_;
+  }
 
   ///
   /// \brief Enable hashing TypeId instances.
diff --git a/paddle/ir/utils.h b/paddle/ir/utils.h
index b4dd00281e159..f4316a7e57e44 100644
--- a/paddle/ir/utils.h
+++ b/paddle/ir/utils.h
@@ -17,12 +17,107 @@
 #include <cassert>
 #include <cstdint>
 #include <cstdlib>
+#include <tuple>
+#include <type_traits>
 
 namespace ir {
+///
+/// \brief Equivalent to boost::hash_combine.
+///
 std::size_t hash_combine(std::size_t lhs, std::size_t rhs);
 
+///
+/// \brief Aligned malloc and free functions.
+///
 void *aligned_malloc(size_t size, size_t alignment);
 
 void aligned_free(void *mem_ptr);
 
+///
+/// \brief Some template methods for manipulating std::tuple.
+///
+/// (1) Pop front element from Tuple
+template <typename Tuple>
+struct PopFrontT;
+
+template <typename Head, typename... Tail>
+struct PopFrontT<std::tuple<Head, Tail...>> {
+ public:
+  using Type = std::tuple<Tail...>;
+};
+
+template <typename Tuple>
+using PopFront = typename PopFrontT<Tuple>::Type;
+
+/// (2) Push front element to Tuple
+template <typename NewElement, typename Tuple>
+struct PushFrontT;
+
+template <typename NewElement, typename... Elements>
+struct PushFrontT<NewElement, std::tuple<Elements...>> {
+ public:
+  using Type = std::tuple<NewElement, Elements...>;
+};
+
+template <typename NewElement, typename... Elements>
+struct PushFrontT<std::tuple<NewElement>, std::tuple<Elements...>> {
+ public:
+  using Type = std::tuple<NewElement, Elements...>;
+};
+
+template <typename NewElement, typename Tuple>
+using PushFront = typename PushFrontT<NewElement, Tuple>::Type;
+
+/// (3) IsEmpty
+template <typename Tuple>
+struct IsEmpty {
+  static constexpr bool value = false;
+};
+
+template <>
+struct IsEmpty<std::tuple<>> {
+  static constexpr bool value = true;
+};
+
+/// (4) IfThenElseT
+template <bool COND, typename TrueT, typename FalseT>
+struct IfThenElseT {
+  using Type = TrueT;
+};
+
+template <typename TrueT, typename FalseT>
+struct IfThenElseT<false, TrueT, FalseT> {
+  using Type = FalseT;
+};
+
+template <bool COND, typename TrueT, typename FalseT>
+using IfThenElse = typename IfThenElseT<COND, TrueT, FalseT>::Type;
+
+/// (5) Filter out all types inherited from BaseT from the tuple.
+template <template <typename> class BaseT,
+          typename Tuple,
+          bool Empty = IsEmpty<Tuple>::value>
+struct Filter;
+
+template <template <typename> class BaseT, typename Tuple>
+struct Filter<BaseT, Tuple, false> {
+ private:
+  using Matched =
+      IfThenElse<std::is_base_of<BaseT<std::tuple_element_t<0, Tuple>>,
+                                 std::tuple_element_t<0, Tuple>>::value,
+                 std::tuple<std::tuple_element_t<0, Tuple>>,
+                 std::tuple<>>;
+  using Rest = typename Filter<BaseT, PopFront<Tuple>>::Type;
+
+ public:
+  using Type =
+      IfThenElse<IsEmpty<Matched>::value, Rest, PushFront<Matched, Rest>>;
+};
+
+// basis case:
+template <template <typename> class BaseT, typename Tuple>
+struct Filter<BaseT, Tuple, true> {
+  using Type = std::tuple<>;
+};
+
 }  // namespace ir
diff --git a/paddle/ir/value.cc b/paddle/ir/value.cc
index f5ecc41018bcf..2f1d5e5616f41 100644
--- a/paddle/ir/value.cc
+++ b/paddle/ir/value.cc
@@ -75,6 +75,12 @@ Operation *Value::GetDefiningOp() const {
 
 std::string Value::print_ud_chain() { return impl_->print_ud_chain(); }
 
+Value::use_iterator Value::begin() const {
+  return ir::OpOperand(impl_->first_use());
+}
+
+Value::use_iterator Value::end() const { return Value::use_iterator(); }
+
 // OpResult
 bool OpResult::classof(Value value) {
   return ir::isa<detail::OpResultImpl>(value.impl());
diff --git a/paddle/ir/value.h b/paddle/ir/value.h
index 3d197182cd6e9..14b2c11ad9e29 100644
--- a/paddle/ir/value.h
+++ b/paddle/ir/value.h
@@ -56,6 +56,38 @@ class OpOperand {
   detail::OpOperandImpl *impl_{nullptr};
 };
 
+///
+/// \brief Value Iterator
+///
+template <typename OperandType>
+class ValueUseIterator {
+ public:
+  ValueUseIterator(OperandType use = nullptr) : current_(use) {}  // NOLINT
+
+  bool operator==(const ValueUseIterator<OperandType> &rhs) const {
+    return current_ == rhs.current_;
+  }
+  ir::Operation *owner() const { return current_.impl()->owner(); }
+
+  OperandType get() const { return current_; }
+
+  OperandType operator*() const { return get(); }
+
+  ValueUseIterator<OperandType> &operator++() {
+    current_ = current_.impl()->next_use();
+    return *this;
+  }
+
+  ValueUseIterator<OperandType> operator++(int) {
+    ValueUseIterator<OperandType> tmp = *this;
+    ++*(this);
+    return tmp;
+  }
+
+ protected:
+  OperandType current_;
+};
+
 ///
 /// \brief Value class represents the SSA value in the IR system. This class
 /// only provides interfaces, for specific implementation, see Impl class.
@@ -96,6 +128,15 @@ class Value {
 
   std::string print_ud_chain();
 
+  ///
+  /// \brief Provide iterator interface to access Value use chain.
+  ///
+  using use_iterator = ValueUseIterator<OpOperand>;
+
+  use_iterator begin() const;
+
+  use_iterator end() const;
+
   friend struct std::hash<Value>;
 
  protected:
diff --git a/test/cpp/ir/CMakeLists.txt b/test/cpp/ir/CMakeLists.txt
index 48feb56c2e270..9d1d9b9d42b75 100644
--- a/test/cpp/ir/CMakeLists.txt
+++ b/test/cpp/ir/CMakeLists.txt
@@ -2,4 +2,5 @@ if(WITH_NEWIR)
   cc_test_old(type_test SRCS type_test.cc DEPS new_ir gtest)
   cc_test_old(ir_attribute_test SRCS ir_attribute_test.cc DEPS new_ir gtest)
   cc_test_old(ir_value_test SRCS ir_value_test.cc DEPS new_ir gtest)
+  cc_test_old(ir_op_test SRCS ir_op_test.cc DEPS new_ir gtest)
 endif()
diff --git a/test/cpp/ir/ir_op_test.cc b/test/cpp/ir/ir_op_test.cc
new file mode 100644
index 0000000000000..342eb5330e504
--- /dev/null
+++ b/test/cpp/ir/ir_op_test.cc
@@ -0,0 +1,153 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include "paddle/ir/builtin_type.h"
+#include "paddle/ir/dialect.h"
+#include "paddle/ir/ir_context.h"
+#include "paddle/ir/op_base.h"
+
+/// \brief Define built-in Trait, derived from OpTraitBase.
+class ReadOnlyTrait : public ir::OpTraitBase<ReadOnlyTrait> {
+ public:
+  explicit ReadOnlyTrait(const ir::Operation *op)
+      : ir::OpTraitBase<ReadOnlyTrait>(op) {}
+};
+
+/// \brief Define built-in Interface, derived from OpInterfaceBase. Concepts and
+/// Models need to be defined within the class. Concept defines abstract
+/// interface functions, and Model is a template class that defines the specific
+/// implementation of interface functions based on template parameters.
+class InferShapeInterface : public ir::OpInterfaceBase<InferShapeInterface> {
+ public:
+  struct Concept {
+    explicit Concept(void (*infer_shape)(const ir::Operation *))
+        : infer_shape_(infer_shape) {}
+    void (*infer_shape_)(const ir::Operation *);
+  };
+
+  template <class ConcreteOp>
+  struct Model : public Concept {
+    static void InferShape(const ir::Operation *op) {
+      ConcreteOp concret_op = ConcreteOp(op);
+      if (concret_op == nullptr) throw("concret_op is nullptr");
+      concret_op.InferShape();
+    }
+
+    Model() : Concept(InferShape) {
+      if (sizeof(Model) != sizeof(Concept)) {
+        throw("sizeof(Model) != sizeof(Concept)");
+      }
+    }
+  };
+
+  InferShapeInterface(const ir::Operation *op, Concept *impl)
+      : ir::OpInterfaceBase<InferShapeInterface>(op), impl_(impl) {}
+
+  void InferShape() { impl_->infer_shape_(operation()); }
+
+ private:
+  Concept *impl_;
+};
+
+// Define op1.
+class Operation1 : public ir::Op<Operation1> {
+ public:
+  using Op::Op;
+  static const char *name() { return "Operation1"; }
+  static const char *attributes_name_[];
+  static uint32_t attributes_num() { return 2; }
+};
+const char *Operation1::attributes_name_[] = {"op1_attr1", "op1_attr2"};
+
+// Define op2.
+class Operation2
+    : public ir::Op<Operation2, ReadOnlyTrait, InferShapeInterface> {
+ public:
+  using Op::Op;
+  static const char *name() { return "Operation2"; }
+  static const char *attributes_name_[];
+  static uint32_t attributes_num() { return 2; }
+  static void InferShape() {
+    std::cout << "This is op2's InferShape interface." << std::endl;
+  }
+};
+const char *Operation2::attributes_name_[] = {"op2_attr1", "op2_attr2"};
+
+// Define a dialect, op1 and op2 will be registered by this dialect.
+class TestDialect : public ir::Dialect {
+ public:
+  explicit TestDialect(ir::IrContext *context)
+      : ir::Dialect(name(), context, ir::TypeId::get<TestDialect>()) {
+    initialize();
+  }
+  static const char *name() { return "op_test"; }
+
+ private:
+  void initialize() { RegisterOps<Operation1, Operation2>(); }
+};
+
+ir::DictionaryAttribute CreateAttribute(std::string attribute_name,
+                                        std::string attribute) {
+  ir::IrContext *ctx = ir::IrContext::Instance();
+  ir::StrAttribute attr_name = ir::StrAttribute::get(ctx, attribute_name);
+  ir::Attribute attr_value = ir::StrAttribute::get(ctx, attribute);
+  std::map<ir::StrAttribute, ir::Attribute> named_attr;
+  named_attr.insert(
+      std::pair<ir::StrAttribute, ir::Attribute>(attr_name, attr_value));
+  return ir::DictionaryAttribute::get(ctx, named_attr);
+}
+
+TEST(op_test, op_test) {
+  // (1) Register Dialect, Operation1, Operation2 into IrContext.
+  ir::IrContext *ctx = ir::IrContext::Instance();
+  ir::Dialect *test_dialect = ctx->GetOrRegisterDialect<TestDialect>();
+  std::cout << test_dialect << std::endl;
+
+  // (2) Get registered operations.
+  std::string op1_name =
+      test_dialect->name() + "." + std::string(Operation1::name());
+  ir::OpInfoImpl *op1_info = ctx->GetRegisteredOpInfo(op1_name);
+  EXPECT_EQ(op1_info != nullptr, true);
+  std::string op2_name =
+      test_dialect->name() + "." + std::string(Operation2::name());
+  ir::OpInfoImpl *op2_info = ctx->GetRegisteredOpInfo(op2_name);
+  EXPECT_EQ(op2_info != nullptr, true);
+
+  EXPECT_EQ(op1_info->HasTrait<ReadOnlyTrait>(), false);
+  EXPECT_EQ(op1_info->HasInterface<InferShapeInterface>(), false);
+  EXPECT_EQ(op2_info->HasTrait<ReadOnlyTrait>(), true);
+  EXPECT_EQ(op2_info->HasInterface<InferShapeInterface>(), true);
+
+  // (3) Test uses for op.
+  std::vector<ir::OpResult> op_inputs = {};
+  std::vector<ir::Type> op_output_types = {ir::Float32Type::get(ctx)};
+  ir::Operation *op =
+      ir::Operation::create(op_inputs,
+                            op_output_types,
+                            CreateAttribute("op1_name", "op1_attr"),
+                            op2_info);
+
+  if (op->HasTrait<ReadOnlyTrait>()) {
+    ReadOnlyTrait trait = op->dyn_cast<ReadOnlyTrait>();
+    EXPECT_EQ(trait.operation(), op);
+  }
+  if (op->HasInterface<InferShapeInterface>()) {
+    InferShapeInterface interface = op->dyn_cast<InferShapeInterface>();
+    interface.InferShape();
+  }
+
+  op->destroy();
+}
diff --git a/test/cpp/ir/ir_value_test.cc b/test/cpp/ir/ir_value_test.cc
index c04e7c35128f4..102619905fe7b 100644
--- a/test/cpp/ir/ir_value_test.cc
+++ b/test/cpp/ir/ir_value_test.cc
@@ -40,21 +40,30 @@ TEST(value_test, value_test) {
   // 1. Construct OP1: a = OP1()
   std::vector<ir::OpResult> op1_inputs = {};
   std::vector<ir::Type> op1_output_types = {ir::Float32Type::get(ctx)};
-  ir::Operation *op1 = ir::Operation::create(
-      op1_inputs, op1_output_types, CreateAttribute("op1_name", "op1_attr"));
+  ir::Operation *op1 =
+      ir::Operation::create(op1_inputs,
+                            op1_output_types,
+                            CreateAttribute("op1_name", "op1_attr"),
+                            nullptr);
   std::cout << op1->print() << std::endl;
   // 2. Construct OP2: b = OP2();
   std::vector<ir::OpResult> op2_inputs = {};
   std::vector<ir::Type> op2_output_types = {ir::Float32Type::get(ctx)};
-  ir::Operation *op2 = ir::Operation::create(
-      op2_inputs, op2_output_types, CreateAttribute("op2_name", "op2_attr"));
+  ir::Operation *op2 =
+      ir::Operation::create(op2_inputs,
+                            op2_output_types,
+                            CreateAttribute("op2_name", "op2_attr"),
+                            nullptr);
   std::cout << op2->print() << std::endl;
   // 3. Construct OP3: c = OP3(a, b);
   std::vector<ir::OpResult> op3_inputs = {op1->GetResultByIndex(0),
                                           op2->GetResultByIndex(0)};
   std::vector<ir::Type> op3_output_types = {ir::Float32Type::get(ctx)};
-  ir::Operation *op3 = ir::Operation::create(
-      op3_inputs, op3_output_types, CreateAttribute("op3_name", "op3_attr"));
+  ir::Operation *op3 =
+      ir::Operation::create(op3_inputs,
+                            op3_output_types,
+                            CreateAttribute("op3_name", "op3_attr"),
+                            nullptr);
   std::cout << op3->print() << std::endl;
   // 4. Construct OP4: d, e, f, g, h, i, j = OP4(a, c);
   std::vector<ir::OpResult> op4_inputs = {op1->GetResultByIndex(0),
@@ -63,8 +72,11 @@ TEST(value_test, value_test) {
   for (size_t i = 0; i < 7; i++) {
     op4_output_types.push_back(ir::Float32Type::get(ctx));
   }
-  ir::Operation *op4 = ir::Operation::create(
-      op4_inputs, op4_output_types, CreateAttribute("op4_name", "op4_attr"));
+  ir::Operation *op4 =
+      ir::Operation::create(op4_inputs,
+                            op4_output_types,
+                            CreateAttribute("op4_name", "op4_attr"),
+                            nullptr);
   std::cout << op4->print() << std::endl;
 
   // Test 1:
@@ -86,6 +98,12 @@ TEST(value_test, value_test) {
   EXPECT_EQ(op4_first_input->next_use(), op3_first_input);
   EXPECT_EQ(op3_first_input->next_use(), nullptr);
 
+  // Test 3: Value iterator
+  ir::Value::use_iterator iter = op1->GetResultByIndex(0).begin();
+  EXPECT_EQ(iter.owner(), op4);
+  ++iter;
+  EXPECT_EQ(iter.owner(), op3);
+
   // destroy
   std::cout << op1->GetResultByIndex(0).print_ud_chain() << std::endl;
   op4->destroy();

From f5476daddbe0b81d22958b89d60ec38d8c69e987 Mon Sep 17 00:00:00 2001
From: umiswing <umiswing@foxmail.com>
Date: Sat, 6 May 2023 17:23:04 +0800
Subject: [PATCH 192/405] Use copy_if_different to avoid recompilation of
 generated cutlass (#53531)

kernels.
---
 cmake/external/cutlass.cmake                  | 27 +++++++++++----
 .../gather_gemm_scatter_manifest.py           | 33 ++++++++++++++-----
 .../gather_gemm_scatter_operation.py          | 18 ++--------
 .../kernels/sparse/gpu/gather_gemm_scatter.h  |  2 +-
 4 files changed, 47 insertions(+), 33 deletions(-)

diff --git a/cmake/external/cutlass.cmake b/cmake/external/cutlass.cmake
index 0b8f813c872e7..9417dcbcdaadf 100644
--- a/cmake/external/cutlass.cmake
+++ b/cmake/external/cutlass.cmake
@@ -42,20 +42,33 @@ ExternalProject_Add(
   INSTALL_COMMAND ""
   TEST_COMMAND "")
 
+set(gemm_operations_file
+    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/generated/gemm/all_gemm_operations.h
+)
+set(configurations_file
+    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/generated/gemm/configurations.h
+)
+set(tmp_gemm_operations_file ${gemm_operations_file}.tmp)
+set(tmp_configurations_file ${configurations_file}.tmp)
+
 add_custom_target(
   cutlass_codegen
-  COMMAND
-    rm -rf
-    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/build
-  COMMAND
-    mkdir -p
-    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/build/generated/gemm
   COMMAND
     ${PYTHON_EXECUTABLE} -B
     ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_generator.py
     "${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/tools/library/scripts/"
-    "${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/build"
+    "${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator"
     "${CMAKE_CUDA_COMPILER_VERSION}"
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_gemm_operations_file}
+          ${gemm_operations_file}
+  COMMAND
+    ${CMAKE_COMMAND} -E echo
+    "copy_if_different ${tmp_gemm_operations_file} to ${gemm_operations_file}"
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_configurations_file}
+          ${configurations_file}
+  COMMAND
+    ${CMAKE_COMMAND} -E echo
+    "copy_if_different ${tmp_configurations_file} to ${configurations_file}"
   VERBATIM)
 
 add_library(cutlass INTERFACE)
diff --git a/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_manifest.py b/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_manifest.py
index 92c07a655b8b7..c3ddd22777957 100644
--- a/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_manifest.py
+++ b/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_manifest.py
@@ -29,6 +29,13 @@ def __init__(self, generated_path, kind, args):
             OperationKind.Gemm: EmitGatherGemmScatterConfigurationLibrary
         }
         self.header_template = "#pragma once\n#ifdef PADDLE_WITH_CUTLASS\n#include \"paddle/phi/kernels/sparse/gpu/cutlass_generator/common.h\"\n"
+        self.configuration_header_template = """
+/*
+  Generated by gemm_operation.py - Do not edit.
+*/
+#pragma once
+#ifdef PADDLE_WITH_CUTLASS
+"""
         self.entry_template = ""
         self.configuration_prototype_template = ""
         self.configuration_template = ""
@@ -58,11 +65,23 @@ def __enter__(self):
 
         self.top_level_path = os.path.join(
             self.operation_path,
-            "all_%s_operations.h" % OperationKindNames[self.kind],
+            "all_%s_operations.h.tmp" % OperationKindNames[self.kind],
         )
 
+        self.configuration_path = os.path.join(
+            self.operation_path, "configurations.h.tmp"
+        ).replace('\\', '/')
+
         self.top_level_file = open(self.top_level_path, "w")
         self.top_level_file.write(self.header_template)
+        self.top_level_file.write(
+            '#include "' + self.operation_path + '/' + 'configurations.h"\n'
+        )
+
+        self.configuration_file = open(self.configuration_path, "w")
+        self.configuration_file.write(self.configuration_header_template)
+        self.configuration_file.write(self.namespace_template)
+        self.configuration_file.close()
 
         self.source_files = [
             self.top_level_path,
@@ -105,14 +124,6 @@ def emit(self, configuration_name, operations):
                 + "<>>,"
             )
 
-        self.top_level_file.write(
-            '#include "'
-            + self.operation_path
-            + '/'
-            + configuration_name
-            + '.h"\n'
-        )
-
     def __exit__(self, exception_type, exception_value, traceback):
         self.top_level_file.write(
             SubstituteTemplate(
@@ -137,6 +148,10 @@ def __exit__(self, exception_type, exception_value, traceback):
         self.top_level_file.write(self.epilogue_template)
         self.top_level_file.close()
 
+        self.configuration_file = open(self.configuration_path, "a")
+        self.configuration_file.write(self.epilogue_template)
+        self.configuration_file.close()
+
 
 class GatherGemmScatterManifest(Manifest):
     def emit(self, target=GeneratorTarget.Library):
diff --git a/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_operation.py b/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_operation.py
index 41b5eeab6a91d..8cfa2c3373b7b 100644
--- a/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_operation.py
+++ b/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_operation.py
@@ -207,7 +207,7 @@ class EmitGatherGemmScatterConfigurationLibrary(EmitGemmConfigurationLibrary):
     def __init__(self, operation_path, configuration_name):
         self.configuration_name = configuration_name
         self.configuration_path = os.path.join(
-            operation_path, "%s.h" % configuration_name
+            operation_path, "configurations.h.tmp"
         ).replace('\\', '/')
 
         self.instance_emitter = {
@@ -246,9 +246,7 @@ def __init__(self, operation_path, configuration_name):
 """
 
     def __enter__(self):
-        self.configuration_file = open(self.configuration_path, "w")
-        self.configuration_file.write(self.header_template)
-        self.configuration_file.write(self.separator)
+        self.configuration_file = open(self.configuration_path, "a")
 
         self.includes = collections.OrderedDict([])
         self.instance_definitions = []
@@ -259,22 +257,10 @@ def __enter__(self):
 
     def __exit__(self, exception_type, exception_value, traceback):
 
-        # Write includes
-        for incl, _ in self.includes.items():
-            include_statement = "#include \"%s\"\n" % incl
-            self.configuration_file.write(include_statement)
-
-        self.configuration_file.write(self.separator)
-        self.configuration_file.write(self.namespace_template)
-
         # Write instance definitions in top-level namespace
         for instance_definition in self.instance_definitions:
             self.configuration_file.write(instance_definition)
 
-        for instance_wrapper in self.instance_wrappers:
-            self.configuration_file.write(instance_wrapper)
-
-        self.configuration_file.write(self.epilogue_template)
         self.configuration_file.close()
 
 
diff --git a/paddle/phi/kernels/sparse/gpu/gather_gemm_scatter.h b/paddle/phi/kernels/sparse/gpu/gather_gemm_scatter.h
index 92e07c1e4d5ed..7d8ad345cf43a 100644
--- a/paddle/phi/kernels/sparse/gpu/gather_gemm_scatter.h
+++ b/paddle/phi/kernels/sparse/gpu/gather_gemm_scatter.h
@@ -18,7 +18,7 @@
 #include "paddle/phi/backends/gpu/gpu_context.h"
 #include "paddle/phi/common/data_type.h"
 #include "paddle/phi/kernels/autotune/auto_tune_base.h"
-#include "paddle/phi/kernels/sparse/gpu/cutlass_generator/build/generated/gemm/all_gemm_operations.h"
+#include "paddle/phi/kernels/sparse/gpu/cutlass_generator/generated/gemm/all_gemm_operations.h"
 
 namespace phi {
 namespace sparse {

From a5a0e8fef76aa436b2d7b1c80bf221f663707121 Mon Sep 17 00:00:00 2001
From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com>
Date: Sat, 6 May 2023 17:27:51 +0800
Subject: [PATCH 193/405] =?UTF-8?q?=E3=80=90prim=E3=80=91Elementwise=20dou?=
 =?UTF-8?q?ble=20grad=20(#53014)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add mul doubel grad

* add sub_double_grad

* add add sub high test

* add mutiply test

* modify other unsqueeze

* delete api.yaml

* only for make ci run

* midify unsqueeze

* modify unsqueeze

* tmp

* modify operants gen

* review modify

* modify review

* debug

* debug

* modify ci cross boundary

* delete log
---
 paddle/fluid/prim/api/manual_prim/utils/utils.h | 17 ++++++++++++-----
 test/prim/prim/vjp/test_comp_high_grad.py       |  2 +-
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/paddle/fluid/prim/api/manual_prim/utils/utils.h b/paddle/fluid/prim/api/manual_prim/utils/utils.h
index d72da6461da75..d37a50c21a8e7 100644
--- a/paddle/fluid/prim/api/manual_prim/utils/utils.h
+++ b/paddle/fluid/prim/api/manual_prim/utils/utils.h
@@ -115,17 +115,24 @@ static std::vector<DST_T> unsafe_vector_cast(const std::vector<SRC_T>& src) {
 }
 
 // This fucction compute unsqueeze dims for reshape to replace unsqueeze.
-static std::vector<int> get_unsqueeze_dims(const Tensor& origin,
-                                           const IntArray& axis) {
+static std::vector<int64_t> get_unsqueeze_dims(
+    const Tensor& origin, const std::vector<int64_t>& axis) {
   auto origin_dims = origin.shape();
   auto total_shape_size = origin_dims.size() + axis.size();
-  std::vector<int> result;
-  int j = 0, k = 0;
+  std::vector<int64_t> result;
+  size_t j = 0, k = 0;
   for (size_t i = 0; i < total_shape_size; ++i) {
-    if (axis[j] == int64_t(i)) {
+    if (j < axis.size() && axis[j] == int64_t(i)) {
       result.push_back(1);
       j++;
     } else {
+      PADDLE_ENFORCE_LT(
+          k,
+          origin_dims.size(),
+          platform::errors::OutOfRange("Your index [%lu] exceeds the number of "
+                                       "elements in origin_dims[%lu].",
+                                       k,
+                                       origin_dims.size()));
       result.push_back(origin_dims[k]);
       k++;
     }
diff --git a/test/prim/prim/vjp/test_comp_high_grad.py b/test/prim/prim/vjp/test_comp_high_grad.py
index 77fa38684d906..76283528e2404 100644
--- a/test/prim/prim/vjp/test_comp_high_grad.py
+++ b/test/prim/prim/vjp/test_comp_high_grad.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

From 1d8c82b6cae2111f4b4a432a62ebbb6b45eb20e0 Mon Sep 17 00:00:00 2001
From: Kai Song <50285351+USTCKAY@users.noreply.github.com>
Date: Sat, 6 May 2023 17:43:40 +0800
Subject: [PATCH 194/405] fix strided_slice ut (#53553)

* fix strided_slice ut

* remove check_dygraph
---
 .../fluid/tests/unittests/test_strided_slice_op.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/test_strided_slice_op.py b/python/paddle/fluid/tests/unittests/test_strided_slice_op.py
index 117321dbd3ed5..304066687814f 100644
--- a/python/paddle/fluid/tests/unittests/test_strided_slice_op.py
+++ b/python/paddle/fluid/tests/unittests/test_strided_slice_op.py
@@ -554,10 +554,10 @@ def test_check_grad_normal(self):
 class TestStridedSliceAPI(unittest.TestCase):
     def test_1(self):
         input = np.random.random([3, 4, 5, 6]).astype("float64")
-        minus_1 = paddle.tensor.fill_constant([1], "int32", -1)
-        minus_3 = paddle.tensor.fill_constant([1], "int32", -3)
+        minus_1 = paddle.tensor.fill_constant([], "int32", -1)
+        minus_3 = paddle.tensor.fill_constant([], "int32", -3)
         starts = paddle.static.data(name='starts', shape=[3], dtype='int32')
-        ends = paddle.static.data(name='ends', shape=[3], dtype='int64')
+        ends = paddle.static.data(name='ends', shape=[3], dtype='int32')
         strides = paddle.static.data(name='strides', shape=[3], dtype='int32')
 
         x = paddle.static.data(
@@ -1032,10 +1032,10 @@ def setUp(self):
         }
 
     def test_check_output(self):
-        self.check_output(check_eager=True)
+        self.check_output()
 
     def test_check_grad(self):
-        self.check_grad({'Input'}, 'Out', check_eager=True)
+        self.check_grad({'Input'}, 'Out')
 
     def initTestCase(self):
         self.input = np.random.rand(100)
@@ -1069,10 +1069,10 @@ def setUp(self):
         }
 
     def test_check_output(self):
-        self.check_output(check_eager=True)
+        self.check_output()
 
     def test_check_grad(self):
-        self.check_grad({'Input'}, 'Out', check_eager=True)
+        self.check_grad({'Input'}, 'Out')
 
     def initTestCase(self):
         self.input = np.random.rand(100)

From ca174eaf469ecf80e9d513aabfa2d69407582c9b Mon Sep 17 00:00:00 2001
From: xiaoluomi <49263480+xiaoluomi@users.noreply.github.com>
Date: Sat, 6 May 2023 18:23:45 +0800
Subject: [PATCH 195/405] fix conv1d_transpose insert quant node bug (#53320)

---
 .../post_training_quantization.py             | 28 +++++++++++++++++--
 .../static/quantization/quantization_pass.py  | 25 +++++++++++++----
 2 files changed, 44 insertions(+), 9 deletions(-)

diff --git a/python/paddle/static/quantization/post_training_quantization.py b/python/paddle/static/quantization/post_training_quantization.py
index af3f35f977def..f80b293a12d4b 100644
--- a/python/paddle/static/quantization/post_training_quantization.py
+++ b/python/paddle/static/quantization/post_training_quantization.py
@@ -684,15 +684,37 @@ def collect_var_name(var_name_list, persistable_var_names, op_type):
                             op._set_attr("op_namescope", "skip_quant")
 
                 op_type = op.type
+                # skip quant form simular conv1d_transpose
+                if op_type == 'conv2d_transpose':
+                    in_name = op.input("Filter")[0]
+                    for _op in self._program.blocks[block_id].ops:
+                        var_name = utils._get_op_output_var_names(_op)
+                        if in_name in var_name:
+                            for name in utils._get_op_input_var_names(_op):
+                                if name not in persistable_var_names:
+                                    op._set_attr("op_namescope", "skip_quant")
+                                    _op._set_attr("op_namescope", "skip_quant")
                 if self._is_full_quantize and op_type not in list(
                     SUPPORT_QUANTIZATION_OP_DICT.keys()
                 ):
                     _logger.warning(
                         op_type + " is not supported for quantization."
                     )
-                is_conv1d_quant = (op_type == "unsqueeze2") and (
-                    utils._get_op_input_var_names(op)[0]
-                    in persistable_var_names
+                conv1d_persistable_var_names = []
+                for opname in persistable_var_names:
+                    if 'conv1d' in opname:
+                        conv1d_persistable_var_names.append(opname)
+
+                is_conv1d_quant = (
+                    (op_type == "unsqueeze2")
+                    and (
+                        utils._get_op_input_var_names(op)[0]
+                        in conv1d_persistable_var_names
+                    )
+                    and (
+                        utils._get_op_input_var_names(op)[0]
+                        in conv1d_persistable_var_names
+                    )
                 )
                 # For quantized ops, sample inputs and outputs
                 if (
diff --git a/python/paddle/static/quantization/quantization_pass.py b/python/paddle/static/quantization/quantization_pass.py
index 8f68e796c459a..ba49516f25e1f 100644
--- a/python/paddle/static/quantization/quantization_pass.py
+++ b/python/paddle/static/quantization/quantization_pass.py
@@ -2469,6 +2469,7 @@ def __init__(
         self._act_preprocess_func = act_preprocess_func
         self._optimizer = optimizer_func
         self._exe = executor
+        self._conv1dtranspose_flag = False
         quant_type = [
             'abs_max',
             'channel_wise_abs_max',
@@ -2597,11 +2598,15 @@ def _transform_forward(self, graph, op):
                         'trans_y'
                     )
                     op_type = op_type + '_trans_y' if trans_y else op_type
-                    quant_axis = (
-                        1
-                        if op_type in utils._channelwise_quant_axis1_ops
-                        else 0
-                    )
+                    if self._conv1dtranspose_flag:
+                        quant_axis = 1
+                        self._conv1dtranspose_flag = False
+                    else:
+                        quant_axis = (
+                            1
+                            if op.name() in utils._channelwise_quant_axis1_ops
+                            else 0
+                        )
                 insert_quant_pass = InsertQuantizeLinear(
                     self._place,
                     self._scope,
@@ -2657,7 +2662,11 @@ def _quant_conv1d(self, graph, op):
         for _op in graph.all_op_nodes():
             var_names = utils._get_op_output_var_names(_op)
             if conv_weight_var_name in var_names and self._has_weight(_op):
-                weight_scale_node = self._transform_forward(graph, _op)
+                if op.name() == 'conv2d_transpose':
+                    if not self._is_skip_quant(graph, _op):
+                        weight_scale_node = self._transform_forward(graph, _op)
+                else:
+                    weight_scale_node = self._transform_forward(graph, _op)
         # insert qdq before conv2d
         for var_node in op.inputs:
             quant_bits = (
@@ -2677,6 +2686,10 @@ def _quant_conv1d(self, graph, op):
                 quant_axis = (
                     1 if op.name() in utils._channelwise_quant_axis1_ops else 0
                 )
+                if 'unsqueeze2' in utils._channelwise_quant_axis1_ops:
+                    utils._channelwise_quant_axis1_ops.remove('unsqueeze2')
+            if self._is_skip_quant(graph, op):
+                return
             insert_quant_pass = InsertQuantizeLinear(
                 self._place,
                 self._scope,

From 08b44e67b3ca8b0f40ef8150bd5ddae635108022 Mon Sep 17 00:00:00 2001
From: Yuanle Liu <yuanlehome@163.com>
Date: Sat, 6 May 2023 20:24:33 +0800
Subject: [PATCH 196/405] [inference][trt] add lookup_table op trt converter,
 use trt gather layer (#53554)

* add lookup_table op trt converter

* update
---
 .../inference/tensorrt/convert/CMakeLists.txt |  1 +
 .../convert/fused_lookup_tables_op.cc         |  4 +-
 .../tensorrt/convert/lookup_table_op.cc       | 47 +++++++++++++++++++
 3 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 paddle/fluid/inference/tensorrt/convert/lookup_table_op.cc

diff --git a/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt b/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt
index 13c0137c7d895..d09c344ca1643 100755
--- a/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt
+++ b/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt
@@ -100,6 +100,7 @@ list(
   skip_merge_layernorm_op.cc
   generic_and_custom_plugin_creater.cc
   fused_lookup_tables_op.cc
+  lookup_table_op.cc
   elementwiseadd_transpose_op.cc
   skip_groupnorm_act_op.cc
   preln_groupnorm_act_op.cc
diff --git a/paddle/fluid/inference/tensorrt/convert/fused_lookup_tables_op.cc b/paddle/fluid/inference/tensorrt/convert/fused_lookup_tables_op.cc
index 27afced4125f8..aaeea2d725809 100644
--- a/paddle/fluid/inference/tensorrt/convert/fused_lookup_tables_op.cc
+++ b/paddle/fluid/inference/tensorrt/convert/fused_lookup_tables_op.cc
@@ -110,4 +110,6 @@ class FusedLookupTablesOpConverter : public OpConverter {
 }  // namespace inference
 }  // namespace paddle
 
-REGISTER_TRT_OP_CONVERTER(lookup_table, FusedLookupTablesOpConverter);
+// NOTE(liuyuanle): We will remove the implementation here later. Ref to
+// tensorrt/convert/lookup_table_op.cc.
+// REGISTER_TRT_OP_CONVERTER(lookup_table, FusedLookupTablesOpConverter);
diff --git a/paddle/fluid/inference/tensorrt/convert/lookup_table_op.cc b/paddle/fluid/inference/tensorrt/convert/lookup_table_op.cc
new file mode 100644
index 0000000000000..e517a00ec9448
--- /dev/null
+++ b/paddle/fluid/inference/tensorrt/convert/lookup_table_op.cc
@@ -0,0 +1,47 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/fluid/inference/tensorrt/convert/op_converter.h"
+
+namespace paddle {
+namespace inference {
+namespace tensorrt {
+
+class LookupTableOpConverter : public OpConverter {
+ public:
+  void operator()(const framework::proto::OpDesc& op,
+                  const framework::Scope& scope,
+                  bool test_mode) override {
+    framework::OpDesc op_desc(op, nullptr);
+    VLOG(3)
+        << "convert lookup_table(lookup_table_v2) op to TensorRT IGatherLayer";
+
+    auto ids_name = op_desc.Input("Ids").front();
+    auto w_name = op_desc.Input("W").front();
+    auto out_name = op_desc.Output("Out").front();
+
+    auto* ids_tensor = engine_->GetITensor(ids_name);
+    auto* w_tensor = engine_->GetITensor(w_name);
+
+    auto* gather_layer =
+        TRT_ENGINE_ADD_LAYER(engine_, Gather, *w_tensor, *ids_tensor, 0);
+    RreplenishLayerAndOutput(gather_layer, "gather", {out_name}, test_mode);
+  }
+};
+
+}  // namespace tensorrt
+}  // namespace inference
+}  // namespace paddle
+
+REGISTER_TRT_OP_CONVERTER(lookup_table, LookupTableOpConverter);

From b65e93269744efed630e409507a489cee37e6125 Mon Sep 17 00:00:00 2001
From: Yiqun Liu <Xreki@users.noreply.github.com>
Date: Sat, 6 May 2023 21:38:37 +0800
Subject: [PATCH 197/405] Add PADDLE_THROW in take_along_axis kernel when the
 datatype of index is wrong. (#53556)

---
 paddle/phi/kernels/gpu/take_along_axis_grad_kernel.cu | 10 +++++-----
 paddle/phi/kernels/gpu/take_along_axis_kernel.cu      | 10 +++++-----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/paddle/phi/kernels/gpu/take_along_axis_grad_kernel.cu b/paddle/phi/kernels/gpu/take_along_axis_grad_kernel.cu
index f38ab6416693f..6cea759283673 100644
--- a/paddle/phi/kernels/gpu/take_along_axis_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/take_along_axis_grad_kernel.cu
@@ -30,11 +30,6 @@ void TakeAlongAxisGradKernel(const Context& dev_ctx,
                              const DenseTensor& out_grad,
                              int axis,
                              DenseTensor* x_grad) {
-  PADDLE_ENFORCE_EQ(
-      dev_ctx.GetPlace().GetType() == phi::AllocationType::GPU,
-      true,
-      errors::PreconditionNotMet("This kernel only runs on GPU."));
-
   // We need to know the shape of input matrix to determine the shape of grad
   // matrix of input.
   x_grad->Resize(x.dims());
@@ -55,6 +50,11 @@ void TakeAlongAxisGradKernel(const Context& dev_ctx,
   } else if (index_type == DataType::INT64) {
     phi::funcs::gpu_scatter_add_kernel<T, int64_t>(
         *x_grad, axis, index, out_grad, dev_ctx);
+  } else {
+    PADDLE_THROW(
+        phi::errors::InvalidArgument("The data type of input index is expected "
+                                     "to be int32 or int64, but recieved %s.",
+                                     phi::DataTypeToString(index_type)));
   }
 }
 
diff --git a/paddle/phi/kernels/gpu/take_along_axis_kernel.cu b/paddle/phi/kernels/gpu/take_along_axis_kernel.cu
index 6ad828f2680b3..ba4c6ba27e682 100644
--- a/paddle/phi/kernels/gpu/take_along_axis_kernel.cu
+++ b/paddle/phi/kernels/gpu/take_along_axis_kernel.cu
@@ -28,11 +28,6 @@ void TakeAlongAxisKernel(const Context& dev_ctx,
                          const DenseTensor& index,
                          int axis,
                          DenseTensor* out) {
-  PADDLE_ENFORCE_EQ(
-      dev_ctx.GetPlace().GetType() == phi::AllocationType::GPU,
-      true,
-      errors::PreconditionNotMet("This kernel only runs on GPU device."));
-
   out->Resize(index.dims());
   dev_ctx.template Alloc<T>(out);
 
@@ -41,6 +36,11 @@ void TakeAlongAxisKernel(const Context& dev_ctx,
     phi::funcs::gpu_gather_kernel<T, int32_t>(x, axis, index, *out, dev_ctx);
   } else if (index_type == DataType::INT64) {
     phi::funcs::gpu_gather_kernel<T, int64_t>(x, axis, index, *out, dev_ctx);
+  } else {
+    PADDLE_THROW(
+        phi::errors::InvalidArgument("The data type of input index is expected "
+                                     "to be int32 or int64, but recieved %s.",
+                                     phi::DataTypeToString(index_type)));
   }
 }
 

From fe807308dcfe5631e97696c1e229fdf5a79dc0f8 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Mon, 8 May 2023 09:19:13 +0800
Subject: [PATCH 198/405] Fix compiling error on CI.

---
 .../operators/fused/fused_gate_attention.h    | 61 ++++---------------
 1 file changed, 13 insertions(+), 48 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 8c736e8854c36..1aa22751d3e38 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -14,7 +14,10 @@ limitations under the License. */
 
 #pragma once
 
+#ifdef PADDLE_WITH_FLASHATTN
 #include "paddle/phi/backends/dynload/flashattn.h"
+#endif
+
 #include "paddle/phi/kernels/arange_kernel.h"
 #include "paddle/phi/kernels/funcs/broadcast_function.h"
 #include "paddle/phi/kernels/funcs/elementwise_functor.h"
@@ -23,10 +26,6 @@ limitations under the License. */
 #include "paddle/phi/kernels/funcs/transpose_function.cu.h"
 #include "paddle/phi/kernels/gpudnn/softmax_gpudnn.h"
 
-#ifdef PADDLE_WITH_FLASHATTN
-#include "paddle/phi/kernels/flash_attn_kernel.h"
-#endif
-
 namespace paddle {
 namespace operators {
 
@@ -829,50 +828,6 @@ class FMHAGateRef {
   }
 
  private:
-  void ComputeBatchedGEMM(const phi::DDim& a_dims,
-                          const phi::DDim& b_dims,
-                          const T* a_ptr,
-                          const T* b_ptr,
-                          T* c_ptr,
-                          bool trans_a,
-                          bool trans_b,
-                          int64_t m,
-                          int64_t n,
-                          int64_t k,
-                          int64_t batch_size,
-                          T alpha = static_cast<T>(1.0),
-                          T beta = static_cast<T>(0.0)) {
-    int64_t stride_a = m * k;
-    int64_t stride_b = k * n;
-    int64_t stride_out = m * n;
-
-    phi::funcs::MatmulPlanner matmul_planner(
-        vectorize(a_dims),
-        vectorize(b_dims),
-        trans_a,
-        trans_b,
-        phi::CppTypeToDataType<T>::Type(),
-        phi::funcs::MatmulFusedType::kMatmul);
-
-    using blaslt = phi::funcs::MatmulWithCublasLt<T>;
-    blaslt::RunWithBatch(dev_ctx_,
-                         a_ptr,
-                         b_ptr,
-                         c_ptr,
-                         m,
-                         n,
-                         k,
-                         trans_a,
-                         trans_b,
-                         batch_size,
-                         stride_a,
-                         stride_b,
-                         stride_out,
-                         &matmul_planner,
-                         alpha,
-                         beta);
-  }
-
   void ComputeBatchedGEMM(const T* a_ptr,
                           const T* b_ptr,
                           T* c_ptr,
@@ -922,6 +877,7 @@ class FlashAttnWithGating {
                       phi::DenseTensor* fmha_out,
                       phi::DenseTensor* gate_out,
                       GateAttentionConfig<T>* config) {
+#ifdef PADDLE_WITH_FLASHATTN
     bool is_bf16 =
         qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
     TypeDebugInfo<T>();
@@ -1053,6 +1009,10 @@ class FlashAttnWithGating {
     if (config->has_gating) {
       gate_out->Resize(config->gate_out_dims);
     }
+#else
+    PADDLE_THROW(phi::errors::Unimplemented(
+        "FlashAttention is unsupported, please set use_flash_attn to false."));
+#endif
   }
 
   void ComputeBackward(const phi::DenseTensor* qkv_transpose_out,
@@ -1064,6 +1024,7 @@ class FlashAttnWithGating {
                        phi::DenseTensor* src_mask_grad,
                        phi::DenseTensor* nonbatched_bias_grad,
                        GateAttentionGradConfig<T>* config) {
+#ifdef PADDLE_WITH_FLASHATTN
     bool is_bf16 =
         qkv_transpose_out->dtype() == DataType::BFLOAT16 ? true : false;
     TypeDebugInfo<T>();
@@ -1229,6 +1190,10 @@ class FlashAttnWithGating {
 
     phi::DenseTensor* qkv_out_grad = config->GetQKVOutGrad();
     ComputeQKVTransposeBackward(qkv_transpose_out_grad, qkv_out_grad);
+#else
+    PADDLE_THROW(phi::errors::Unimplemented(
+        "FlashAttention is unsupported, please set use_flash_attn to false."));
+#endif
   }
 
  private:

From a2991538600b5cafe9e2452a00fd31a79062c251 Mon Sep 17 00:00:00 2001
From: umiswing <umiswing@foxmail.com>
Date: Mon, 8 May 2023 10:13:53 +0800
Subject: [PATCH 199/405] [cutlass] Avoid rewrite generated kernels in
 recompilation. (#53575)

---
 cmake/external/cutlass.cmake                         | 12 ++++++++----
 .../gather_gemm_scatter_manifest.py                  |  3 ++-
 paddle/phi/kernels/sparse/gpu/gather_gemm_scatter.h  |  2 +-
 3 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/cmake/external/cutlass.cmake b/cmake/external/cutlass.cmake
index 9417dcbcdaadf..ae7bf6ce6fa45 100644
--- a/cmake/external/cutlass.cmake
+++ b/cmake/external/cutlass.cmake
@@ -42,14 +42,18 @@ ExternalProject_Add(
   INSTALL_COMMAND ""
   TEST_COMMAND "")
 
+set(tmp_gemm_operations_file
+    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/generated/gemm/all_gemm_operations.h.tmp
+)
+set(tmp_configurations_file
+    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/generated/gemm/configurations.h.tmp
+)
 set(gemm_operations_file
-    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/generated/gemm/all_gemm_operations.h
+    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/all_gemm_operations.h
 )
 set(configurations_file
-    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/generated/gemm/configurations.h
+    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/configurations.h
 )
-set(tmp_gemm_operations_file ${gemm_operations_file}.tmp)
-set(tmp_configurations_file ${configurations_file}.tmp)
 
 add_custom_target(
   cutlass_codegen
diff --git a/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_manifest.py b/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_manifest.py
index c3ddd22777957..ac0b3baf32587 100644
--- a/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_manifest.py
+++ b/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_manifest.py
@@ -75,7 +75,8 @@ def __enter__(self):
         self.top_level_file = open(self.top_level_path, "w")
         self.top_level_file.write(self.header_template)
         self.top_level_file.write(
-            '#include "' + self.operation_path + '/' + 'configurations.h"\n'
+            '#include "'
+            + 'paddle/phi/kernels/sparse/gpu/cutlass_generator/configurations.h"\n'
         )
 
         self.configuration_file = open(self.configuration_path, "w")
diff --git a/paddle/phi/kernels/sparse/gpu/gather_gemm_scatter.h b/paddle/phi/kernels/sparse/gpu/gather_gemm_scatter.h
index 7d8ad345cf43a..73b7c3f3003b8 100644
--- a/paddle/phi/kernels/sparse/gpu/gather_gemm_scatter.h
+++ b/paddle/phi/kernels/sparse/gpu/gather_gemm_scatter.h
@@ -18,7 +18,7 @@
 #include "paddle/phi/backends/gpu/gpu_context.h"
 #include "paddle/phi/common/data_type.h"
 #include "paddle/phi/kernels/autotune/auto_tune_base.h"
-#include "paddle/phi/kernels/sparse/gpu/cutlass_generator/generated/gemm/all_gemm_operations.h"
+#include "paddle/phi/kernels/sparse/gpu/cutlass_generator/all_gemm_operations.h"
 
 namespace phi {
 namespace sparse {

From 65c6ed17a84f6a5800ad0d6fd3ac3f1df7b2a982 Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Mon, 8 May 2023 10:33:55 +0800
Subject: [PATCH 200/405] fix tag commit id with newest

---
 cmake/external/flashattn.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index 29c0b8efb426c..8cbcdb717223a 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -20,7 +20,7 @@ set(FLASHATTN_PREFIX_DIR ${THIRD_PARTY_PATH}/flashattn)
 set(FLASHATTN_SOURCE_SUBDIR csrc/flash_attn)
 set(FLASHATTN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flashattn)
 set(FLASHATTN_REPOSITORY ${GIT_URL}/JamesLim-sy/flash-attention.git)
-set(FLASHATTN_TAG 782a98a1fc02fb36d07129a97750430e4fdf2022)
+set(FLASHATTN_TAG 93486d94943605436bb4cb636d9ac4a109d3fcaf)
 
 set(FLASHATTN_INCLUDE_DIR
     "${FLASHATTN_INSTALL_DIR}/include"

From acefdeb70d940d951387ee9c3e668d8c03f66772 Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Mon, 8 May 2023 10:54:01 +0800
Subject: [PATCH 201/405] Fix typos, test=document_fix (#53540)

---
 paddle/phi/backends/device_base.cc                   | 2 +-
 paddle/phi/core/compat/get_kerneltype_forvar_utils.h | 4 ++--
 paddle/phi/core/compat/op_utils.h                    | 2 +-
 paddle/phi/kernels/funcs/blas/blas.h                 | 2 +-
 paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h       | 6 +++---
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/paddle/phi/backends/device_base.cc b/paddle/phi/backends/device_base.cc
index 351b4ebd51bc6..2a1bf8b07437c 100644
--- a/paddle/phi/backends/device_base.cc
+++ b/paddle/phi/backends/device_base.cc
@@ -124,7 +124,7 @@ bool DeviceInterface::QueryEvent(size_t dev_id, const event::Event* event) {
   return true;
 }
 
-// memery manage
+// memory manage
 void DeviceInterface::MemoryCopyH2D(size_t dev_id,
                                     void* dst,
                                     const void* src,
diff --git a/paddle/phi/core/compat/get_kerneltype_forvar_utils.h b/paddle/phi/core/compat/get_kerneltype_forvar_utils.h
index 48348ce43f906..6f3798af93742 100644
--- a/paddle/phi/core/compat/get_kerneltype_forvar_utils.h
+++ b/paddle/phi/core/compat/get_kerneltype_forvar_utils.h
@@ -22,7 +22,7 @@ class KernelKey;
 class DenseTensor;
 /**
  * Note: GetKernelTypeForVarContext is currently designed for oneDNN kernel when
- * the related memeber function 'GetKernelTypeForVar' is special. It is
+ * the related member function 'GetKernelTypeForVar' is special. It is
  * possible to leverage to other vendor libraries in the future.
  */
 class GetKernelTypeForVarContext {
@@ -47,7 +47,7 @@ class GetKernelTypeForVarContext {
 
  private:
   const KernelKey* kernel_key_;  // not owned
-  // Use AttributeMap in namespace 'phi' to avoid depending 'fuild'
+  // Use AttributeMap in namespace 'phi' to avoid depending 'fluid'
   const AttributeMap* attrs_;  // not owned
   std::string* var_name_;      // not owned
   DenseTensor* tensor_;        // not owned
diff --git a/paddle/phi/core/compat/op_utils.h b/paddle/phi/core/compat/op_utils.h
index 7e3d10bd56a70..f3e594eae11f7 100644
--- a/paddle/phi/core/compat/op_utils.h
+++ b/paddle/phi/core/compat/op_utils.h
@@ -31,7 +31,7 @@ const static std::string deprecated_kernel_name = "deprecated";  // NOLINT
 
 const std::unordered_set<std::string> standard_kernel_suffixs({
     "sr",  // SelectedRows kernel
-    "raw"  // fallback kernel of origfinal fluid op
+    "raw"  // fallback kernel of original fluid op
 });
 
 /**
diff --git a/paddle/phi/kernels/funcs/blas/blas.h b/paddle/phi/kernels/funcs/blas/blas.h
index 9e970cf1b549a..140eca890480f 100644
--- a/paddle/phi/kernels/funcs/blas/blas.h
+++ b/paddle/phi/kernels/funcs/blas/blas.h
@@ -38,7 +38,7 @@ namespace funcs {
  * if Mat A is [BatchSize, H, W], Mat B is [BatchSize, H, W]. It will be a
  * `batch_size` times of GEMM. The batched GEMM could be faster base on the
  * implementation of the blas library. The batch size could be zero. If any
- * matrix of `matmul` has a batch size, the will be a batched GEMM, too. e.g.,
+ * matrix of `matmul` has a batch size, there will be a batched GEMM, too. e.g.,
  * Mat A is [BatchSize, H1, W2], and Mat B [H2, W2], The result matrix wil be
  * [BatchSize, H1, W2]
  *
diff --git a/paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h b/paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h
index 1bc409bd0dfb5..ee3709e967d89 100644
--- a/paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h
+++ b/paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h
@@ -39,7 +39,7 @@ namespace funcs {
 // While kMatmul, kMatmulGrad, kMatmulGradWithoutBias share the same
 // enum value, but if all elements for MatmulPlanner->GetKey() is same,
 // no matter forward or backward, they could share the same descriptor
-// cache, in that the descritpor is for decription of matmul operation.
+// cache, in that the descriptor is for description of matmul operation.
 enum MatmulFusedType {
   kMatmul = CUBLASLT_EPILOGUE_DEFAULT,
   kMatmulGrad = CUBLASLT_EPILOGUE_DEFAULT,
@@ -216,7 +216,7 @@ struct MatmulDescriptor {
     cudaDataType_t scale_type = phi::backends::gpu::ToCudaDataType<MT>();
     cublasComputeType_t compute_type = GetCudaComputeType<T>();
 
-    // Create operation desciriptor; see cublasLtMatmulDescAttributes_t for
+    // Create operation descriptor; see cublasLtMatmulDescAttributes_t for
     // details about defaults; just need to set the transforms for A and B
     PADDLE_ENFORCE_GPU_SUCCESS(
         dynload::cublasLtMatmulDescCreate(&op_desc, compute_type, scale_type));
@@ -787,7 +787,7 @@ struct LinearGradWithCublasLt : public CublasLtBase<T> {
   }
 };
 #else
-// A void structure just for successfully complile.
+// A void structure just for successfully compile.
 struct MatmulPlanner {};
 #endif  // (PADDLE_WITH_CUDA) && CUDA_VERSION >= 11060
 

From 3fd2e7659dd754d01115c5071e6988cfa5707551 Mon Sep 17 00:00:00 2001
From: LyndonKong <kongrui.chn@outlook.com>
Date: Mon, 8 May 2023 10:58:18 +0800
Subject: [PATCH 202/405] Fix sample code of poisson_nll_loss (#53551)

---
 python/paddle/nn/functional/loss.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py
index b105d3d7fc28e..ae7b8813a0e3e 100644
--- a/python/paddle/nn/functional/loss.py
+++ b/python/paddle/nn/functional/loss.py
@@ -1541,7 +1541,7 @@ def poisson_nll_loss(
 
             input = paddle.randn([5, 2], dtype=paddle.float32)
             label = paddle.randn([5, 2], dtype=paddle.float32)
-            loss = F.poisson_nll_loss(input, label, log_input=True, reduction='None')
+            loss = F.poisson_nll_loss(input, label, log_input=True, reduction='none')
             print(loss)
             loss = F.poisson_nll_loss(input, label, reduction='mean')
             print(loss)

From 2bf61284890b43c8f4e27643690a5e407c250879 Mon Sep 17 00:00:00 2001
From: Zhang Ting <zhangting_2017@163.com>
Date: Mon, 8 May 2023 11:04:25 +0800
Subject: [PATCH 203/405] [AMP] fix static promote (#53439)

---
 python/paddle/static/amp/fp16_lists.py        | 68 +++++++++----------
 python/paddle/static/amp/fp16_utils.py        | 20 ++++--
 test/amp/test_amp_list.py                     |  7 +-
 test/amp/test_amp_promote.py                  |  1 -
 test/amp/test_model_cast_to_bf16.py           |  8 ++-
 .../contrib/test_image_classification_fp16.py | 35 ++++++++--
 6 files changed, 87 insertions(+), 52 deletions(-)

diff --git a/python/paddle/static/amp/fp16_lists.py b/python/paddle/static/amp/fp16_lists.py
index 6e0a4a5254ccd..79abcc5e6c522 100644
--- a/python/paddle/static/amp/fp16_lists.py
+++ b/python/paddle/static/amp/fp16_lists.py
@@ -23,11 +23,15 @@
 )
 
 # lookup_table fp16 is slower than fp32, though fp16 is supported.
-_extra_unsupported_list = {
+_extra_black_list = {
     'lookup_table',
     'lookup_table_v2',
     'scatter',
-    'scatter_grad',
+    'linear_interp_v2',
+    'nearest_interp_v2',
+    'bilinear_interp_v2',
+    'bicubic_interp_v2',
+    'trilinear_interp_v2',
 }
 
 
@@ -118,8 +122,7 @@ def _get_sys_unsupported_list(dtype):
 def _get_unsupported_list(dtype):
     # The set of ops that don't support fp16 calculation
     _, _sys_unsupported_list = _get_sys_unsupported_list(dtype)
-    unsupported_list = _extra_unsupported_list | _sys_unsupported_list
-    return unsupported_list
+    return _sys_unsupported_list
 
 
 # The three sets listed below are changed dynamiclly. They don't contain all
@@ -145,6 +148,32 @@ def _get_white_list(dtype):
     return white_list_for_dtype
 
 
+# The set of ops that support fp16 calculation and are considered numerically-
+# dangerous and whose effects may also be observed in downstream ops.
+black_list = {
+    'exp',
+    'square',
+    'log',
+    'mean',
+    'sum',
+    'cos_sim',
+    'softmax',
+    'softmax_with_cross_entropy',
+    'sigmoid_cross_entropy_with_logits',
+    'c_softmax_with_cross_entropy',
+    'cross_entropy',
+    'cross_entropy2',
+    # default fp32 can avoid return inf when the sum value large than 65504
+    'reduce_sum',
+}
+
+
+def _get_black_list():
+    _black_list = copy.copy(black_list)
+    _black_list = _black_list | _extra_black_list
+    return _black_list
+
+
 class AutoMixedPrecisionLists:
     """
     AutoMixedPrecisionLists is a class for black/white list. It can update
@@ -170,7 +199,7 @@ def __init__(
         self._custom_white_list = custom_white_list
         self._custom_black_list = custom_black_list
         self.white_list = copy.copy(_get_white_list(self.amp_dtype))
-        self.black_list = copy.copy(black_list)
+        self.black_list = copy.copy(_get_black_list())
         self.gray_list = copy.copy(gray_list)
         self.unsupported_list = copy.copy(_get_unsupported_list(self.amp_dtype))
         self.black_varnames = copy.copy(custom_black_varnames)
@@ -196,8 +225,6 @@ def _update_list(self):
                 elif op_name in self.gray_list:
                     self.gray_list.remove(op_name)
                 self.white_list.add(op_name)
-                if op_name in _extra_unsupported_list:
-                    self.unsupported_list.remove(op_name)
         if self._custom_black_list:
             for op_name in self._custom_black_list:
                 if op_name in self.white_list:
@@ -217,33 +244,6 @@ def _update_list(self):
             )
 
 
-# The set of ops that support fp16 calculation and are considered numerically-
-# dangerous and whose effects may also be observed in downstream ops.
-black_list = {
-    'exp',
-    'square',
-    'log',
-    'mean',
-    'sum',
-    'cos_sim',
-    'softmax',
-    'softmax_with_cross_entropy',
-    'sigmoid_cross_entropy_with_logits',
-    'c_softmax_with_cross_entropy',
-    'cross_entropy',
-    'cross_entropy2',
-    # fp16 is slower than fp32, though fp16 is supported.
-    'lookup_table',
-    'lookup_table_v2',
-    'linear_interp_v2',
-    'nearest_interp_v2',
-    'bilinear_interp_v2',
-    'bicubic_interp_v2',
-    'trilinear_interp_v2',
-    # default fp32 can avoid return inf when the sum value large than 65504
-    'reduce_sum',
-}
-
 # This set contains two types of ops. All ops supported fp16 calculation. One
 # of two types is considered numerically-safe, but may be made unsafe by an
 # upstream blacklist op. Another type do not have numerically-significant
diff --git a/python/paddle/static/amp/fp16_utils.py b/python/paddle/static/amp/fp16_utils.py
index 740930769cb99..c2f8a12d33b46 100644
--- a/python/paddle/static/amp/fp16_utils.py
+++ b/python/paddle/static/amp/fp16_utils.py
@@ -425,15 +425,22 @@ def set_var_dst_dtype(
 
 
 def set_param_dtype(program, dtype, amp_lists, use_fp16_guard, level):
-    if level == "O1":
-        return
     keep_fp32_var_names = set()
+    if level == "O1":
+        return keep_fp32_var_names
     all_parameters = []
     for block in program.blocks:
         all_parameters.extend(block.all_parameters())
         ops = block.ops
         for op in ops:
-            if op_need_keep_fp32(op, amp_lists, use_fp16_guard):
+            # Currently, lookup_table is in black_list and unsupport_list, it's weight will be
+            # set to fp32 in setp 1 of cast_model_tp_fp16. But the weight may be used as matmul's
+            # input in transformer, so the weight is also in to_fp16_var_names.
+            # TODO(zhangting2020): consider fix auto_parallel_fp16 and remove lookup_table
+            # from black_list and unsupport_list.
+            if op in ['lookup_table', 'lookup_table_v2']:
+                continue
+            if _need_keep_fp32(op, amp_lists.unsupported_list, use_fp16_guard):
                 for in_name in op.input_names:
                     keep_fp32_var_names = keep_fp32_var_names.union(
                         op.input(in_name)
@@ -451,6 +458,7 @@ def set_param_dtype(program, dtype, amp_lists, use_fp16_guard, level):
         if param.name not in keep_fp32_var_names:
             _logger.debug(f"-- set param {param.name} to {dtype} --.")
             param.desc.set_dtype(dtype)
+    return keep_fp32_var_names
 
 
 def op_need_keep_fp32(op, amp_lists, use_fp16_guard):
@@ -607,15 +615,17 @@ def cast_model_to_fp16(
     keep_fp32_ops = set()
     keep_fp16_ops = set()
     to_fp16_var_names = set()
+    keep_fp32_var_names = set()
 
     # step 1: set params dtype.
-    set_param_dtype(
+    fp32_var_names = set_param_dtype(
         program,
         dtype=dest_type,
         amp_lists=amp_lists,
         use_fp16_guard=use_fp16_guard,
         level=level,
     )
+    keep_fp32_var_names = keep_fp32_var_names.union(fp32_var_names)
 
     def need_process(op):
         need_process = True
@@ -719,6 +729,8 @@ def need_process(op):
             idx += num_cast_ops + 1
     _logger.debug("---- after cast model to fp16 ----")
     _logger.debug(program)
+
+    to_fp16_var_names.difference_update(keep_fp32_var_names)
     return to_fp16_var_names
 
 
diff --git a/test/amp/test_amp_list.py b/test/amp/test_amp_list.py
index 9b0bf5129c36d..e61aa8281eccc 100644
--- a/test/amp/test_amp_list.py
+++ b/test/amp/test_amp_list.py
@@ -50,9 +50,10 @@ def test_static(self):
         self.check_if_op_not_in_list(
             self.custom_white_list, amp_list.black_list
         )
-        self.check_if_op_not_in_list(
-            self.custom_white_list, amp_list.unsupported_list
-        )
+        if paddle.amp.is_float16_supported():
+            self.check_if_op_not_in_list(
+                self.custom_white_list, amp_list.black_list
+            )
 
     def test_eager(self):
         if not paddle.amp.is_float16_supported():
diff --git a/test/amp/test_amp_promote.py b/test/amp/test_amp_promote.py
index 8aa5d71a79ef0..e75799b39c332 100644
--- a/test/amp/test_amp_promote.py
+++ b/test/amp/test_amp_promote.py
@@ -48,7 +48,6 @@ def check_promote_results(
 
         max_iters = 2
         x_fp32 = np.random.random(size=[1, 1, 6, 6]).astype("float32")
-        print(main_program)
         losses_o1 = self.run_program(
             main_program,
             startup_program,
diff --git a/test/amp/test_model_cast_to_bf16.py b/test/amp/test_model_cast_to_bf16.py
index 3002b623b18af..7e4de2630d4a5 100644
--- a/test/amp/test_model_cast_to_bf16.py
+++ b/test/amp/test_model_cast_to_bf16.py
@@ -265,18 +265,20 @@ def test_amp_bf16_o2(self):
 
         amp.debugging.collect_operator_stats(main_program)
         op_stats_list = amp.debugging._get_op_stats_list(main_program)
+        expected_fp32_calls = {"lookup_table_v2": 1}
         expected_bf16_calls = {
             "matmul_v2": 1,
             "elementwise_add": 1,
             "dropout": 1,
             "lookup_table_v2": 0,
-            "squared_l2_norm": 2,
-            "adamw": 2,
+            "squared_l2_norm": 3,
+            "adamw": 3,
         }
         self._check_optimizer(
             main_program,
             expected_bf16_calls["matmul_v2"]
-            + expected_bf16_calls["elementwise_add"],
+            + expected_bf16_calls["elementwise_add"]
+            + expected_fp32_calls["lookup_table_v2"],
         )
         self._check_op_calls(op_stats_list[0], expected_bf16_calls)
 
diff --git a/test/contrib/test_image_classification_fp16.py b/test/contrib/test_image_classification_fp16.py
index fb1bafdc8613d..0fc98c4792d22 100644
--- a/test/contrib/test_image_classification_fp16.py
+++ b/test/contrib/test_image_classification_fp16.py
@@ -318,7 +318,10 @@ def test_amp_lists(self):
             copy.copy(paddle.static.amp.fp16_lists.white_list)
             | paddle.static.amp.fp16_lists._only_supported_fp16_list
         )
-        black_list = copy.copy(paddle.static.amp.fp16_lists.black_list)
+        black_list = copy.copy(
+            paddle.static.amp.fp16_lists.black_list
+            | paddle.static.amp.fp16_lists._extra_black_list
+        )
         gray_list = copy.copy(paddle.static.amp.fp16_lists.gray_list)
 
         amp_lists = paddle.static.amp.AutoMixedPrecisionLists()
@@ -331,7 +334,10 @@ def test_amp_lists_1(self):
             copy.copy(paddle.static.amp.fp16_lists.white_list)
             | paddle.static.amp.fp16_lists._only_supported_fp16_list
         )
-        black_list = copy.copy(paddle.static.amp.fp16_lists.black_list)
+        black_list = copy.copy(
+            paddle.static.amp.fp16_lists.black_list
+            | paddle.static.amp.fp16_lists._extra_black_list
+        )
         gray_list = copy.copy(paddle.static.amp.fp16_lists.gray_list)
 
         # 1. w={'exp}, b=None
@@ -348,7 +354,10 @@ def test_amp_lists_2(self):
             copy.copy(paddle.static.amp.fp16_lists.white_list)
             | paddle.static.amp.fp16_lists._only_supported_fp16_list
         )
-        black_list = copy.copy(paddle.static.amp.fp16_lists.black_list)
+        black_list = copy.copy(
+            paddle.static.amp.fp16_lists.black_list
+            | paddle.static.amp.fp16_lists._extra_black_list
+        )
         gray_list = copy.copy(paddle.static.amp.fp16_lists.gray_list)
 
         # 2. w={'tanh'}, b=None
@@ -365,7 +374,10 @@ def test_amp_lists_3(self):
             copy.copy(paddle.static.amp.fp16_lists.white_list)
             | paddle.static.amp.fp16_lists._only_supported_fp16_list
         )
-        black_list = copy.copy(paddle.static.amp.fp16_lists.black_list)
+        black_list = copy.copy(
+            paddle.static.amp.fp16_lists.black_list
+            | paddle.static.amp.fp16_lists._extra_black_list
+        )
         gray_list = copy.copy(paddle.static.amp.fp16_lists.gray_list)
 
         # 3. w={'lstm'}, b=None
@@ -381,7 +393,10 @@ def test_amp_lists_4(self):
             copy.copy(paddle.static.amp.fp16_lists.white_list)
             | paddle.static.amp.fp16_lists._only_supported_fp16_list
         )
-        black_list = copy.copy(paddle.static.amp.fp16_lists.black_list)
+        black_list = copy.copy(
+            paddle.static.amp.fp16_lists.black_list
+            | paddle.static.amp.fp16_lists._extra_black_list
+        )
         gray_list = copy.copy(paddle.static.amp.fp16_lists.gray_list)
 
         # 4. w=None, b={'conv2d'}
@@ -400,7 +415,10 @@ def test_amp_lists_5(self):
             copy.copy(paddle.static.amp.fp16_lists.white_list)
             | paddle.static.amp.fp16_lists._only_supported_fp16_list
         )
-        black_list = copy.copy(paddle.static.amp.fp16_lists.black_list)
+        black_list = copy.copy(
+            paddle.static.amp.fp16_lists.black_list
+            | paddle.static.amp.fp16_lists._extra_black_list
+        )
         gray_list = copy.copy(paddle.static.amp.fp16_lists.gray_list)
 
         # 5. w=None, b={'tanh'}
@@ -419,7 +437,10 @@ def test_amp_lists_6(self):
             copy.copy(paddle.static.amp.fp16_lists.white_list)
             | paddle.static.amp.fp16_lists._only_supported_fp16_list
         )
-        black_list = copy.copy(paddle.static.amp.fp16_lists.black_list)
+        black_list = copy.copy(
+            paddle.static.amp.fp16_lists.black_list
+            | paddle.static.amp.fp16_lists._extra_black_list
+        )
         gray_list = copy.copy(paddle.static.amp.fp16_lists.gray_list)
 
         # 6. w=None, b={'lstm'}

From fca8595eed79fe32198dce9c0c414911289fa3d4 Mon Sep 17 00:00:00 2001
From: Yuanle Liu <yuanlehome@163.com>
Date: Mon, 8 May 2023 11:08:49 +0800
Subject: [PATCH 204/405] [Paddle-TRT] add generic plugin for
 lookup_table_v2(embedding) op (#53539)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add embedding generic plugin， not enabled
---
 .../generic_and_custom_plugin_creater.cc      | 16 +++-
 .../tensorrt/dynamic_shape_infermeta.cc       | 19 +++++
 paddle/fluid/inference/tensorrt/op_teller.cc  |  3 +-
 .../tensorrt/plugin/generic_plugin.cu         | 28 ++++++-
 .../tensorrt/plugin/generic_plugin.h          | 31 +++----
 .../tensorrt/plugin_arg_mapping_context.cc    | 81 ++++++++++++-------
 .../tensorrt/plugin_arg_mapping_context.h     |  9 ++-
 .../tensorrt/test_arg_mapping_context.cc      | 13 +--
 8 files changed, 137 insertions(+), 63 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc b/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc
index ad01ddff4ce04..a889fc9fa3147 100644
--- a/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc
+++ b/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc
@@ -17,13 +17,13 @@ limitations under the License. */
 #include "paddle/fluid/inference/tensorrt/plugin/generic_plugin.h"
 #include "paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.h"
 #include "paddle/phi/api/ext/op_meta_info.h"
+#include "paddle/phi/core/enforce.h"
+#include "paddle/phi/core/errors.h"
 
 namespace paddle {
 namespace inference {
 namespace tensorrt {
-/*
- * Stack converter from fluid to tensorRT.
- */
+
 class CustomPluginCreater : public OpConverter {
  public:
   void operator()(const framework::proto::OpDesc &op,
@@ -164,6 +164,8 @@ class GenericPluginCreater : public OpConverter {
                   const framework::Scope &scope,
                   bool test_mode) override {
     framework::OpDesc op_desc(op, nullptr);
+    VLOG(3) << "convert " << op_desc.Type() << " op to generic pluign layer";
+
     CHECK(block_);
     const framework::BlockDesc block_desc(
         nullptr, const_cast<framework::proto::BlockDesc *>(block_));
@@ -181,6 +183,14 @@ class GenericPluginCreater : public OpConverter {
       phi_kernel_signature =
           phi::DefaultKernelSignatureMap::Instance().Get(op_desc.Type());
     }
+    VLOG(3) << phi_kernel_signature;
+    PADDLE_ENFORCE_EQ(
+        phi_kernel_signature.input_names.empty() ||
+            phi_kernel_signature.output_names.empty(),
+        false,
+        platform::errors::PreconditionNotMet(
+            "The %s op's kernel signature (inputs and output) should not null.",
+            op_desc.Type()));
 
     bool with_fp16 = engine_->WithFp16() && !engine_->disable_trt_plugin_fp16();
 
diff --git a/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta.cc b/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta.cc
index fe0c4e0d18ef4..69458e240b846 100644
--- a/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta.cc
+++ b/paddle/fluid/inference/tensorrt/dynamic_shape_infermeta.cc
@@ -417,6 +417,24 @@ nvinfer1::DimsExprs GridSamplerInferMeta(
   return output;
 }
 
+nvinfer1::DimsExprs LookupTableV2InferMeta(
+    int output_index,
+    const nvinfer1::DimsExprs* inputs,
+    int nb_inputs,
+    nvinfer1::IExprBuilder& expr_builder,  // NOLINT
+    const framework::OpDesc& op_desc) {
+  const auto x_dims = inputs[0];
+  const auto weight_dims = inputs[1];
+
+  nvinfer1::DimsExprs output;
+  output.nbDims = x_dims.nbDims + 1;
+  for (int i = 0; i < x_dims.nbDims; ++i) {
+    output.d[i] = x_dims.d[i];
+  }
+  output.d[x_dims.nbDims] = weight_dims.d[1];
+  return output;
+}
+
 PD_REGISTER_DYNAMIC_INFER_META_FN(gather_nd, GatherNdInferMeta);
 PD_REGISTER_DYNAMIC_INFER_META_FN(yolo_box, YoloBoxInferMeta);
 PD_REGISTER_DYNAMIC_INFER_META_FN(instance_norm, InstanceNormInferMeta);
@@ -427,6 +445,7 @@ PD_REGISTER_DYNAMIC_INFER_META_FN(moe, MoeInferMeta);
 PD_REGISTER_DYNAMIC_INFER_META_FN(pad3d, Pad3dInferMeta);
 PD_REGISTER_DYNAMIC_INFER_META_FN(grid_sampler, GridSamplerInferMeta);
 PD_REGISTER_DYNAMIC_INFER_META_FN(p_norm, PNormInferMeta);
+
 }  // namespace tensorrt
 }  // namespace inference
 }  // namespace paddle
diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index 0aea348566bff..beb289b420f2d 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -2865,10 +2865,10 @@ struct SimpleOpTypeSetTeller : public Teller {
       "logsigmoid",
       "preln_layernorm_shift_partition",
       "lookup_table",
+      "lookup_table_v2",
       "trans_layernorm",
       "merge_layernorm",
       "skip_merge_layernorm",
-      "lookup_table_v2",
       "expand_v2",
       "expand_as_v2",
       "fuse_eleadd_transpose",
@@ -3143,6 +3143,7 @@ OpTeller::OpTeller() {
   tellers_.emplace_back(new tensorrt::GenericPluginTeller);
   tellers_.emplace_back(new tensorrt::CustomPluginTeller);
 }
+
 }  // namespace tensorrt
 }  // namespace inference
 }  // namespace paddle
diff --git a/paddle/fluid/inference/tensorrt/plugin/generic_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/generic_plugin.cu
index f9bbb27958676..04645d7a0efaf 100644
--- a/paddle/fluid/inference/tensorrt/plugin/generic_plugin.cu
+++ b/paddle/fluid/inference/tensorrt/plugin/generic_plugin.cu
@@ -18,6 +18,7 @@
 #include "paddle/fluid/framework/phi_utils.h"
 #include "paddle/fluid/inference/tensorrt/dynamic_shape_infermeta_registry.h"
 #include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/common/data_type.h"
 #include "paddle/phi/core/compat/op_utils.h"
 #include "paddle/phi/core/kernel_context.h"
 #include "paddle/phi/core/kernel_factory.h"
@@ -354,6 +355,19 @@ bool GenericPlugin::supportsFormatCombination(
     if (pos == 3)
       return in_out[0].type == in_out[pos].type &&
              in_out[0].format == in_out[pos].format;
+  } else if (op_desc_.Type() == "lookup_table_v2") {
+    if (pos == 0)
+      return (in_out[pos].type == nvinfer1::DataType::kINT32 &&
+              (in_out[pos].format == nvinfer1::TensorFormat::kLINEAR));
+    if (pos == 1)
+      return (in_out[pos].type == nvinfer1::DataType::kFLOAT) ||
+             ((isFp16Supported() &&
+               in_out[pos].type == nvinfer1::DataType::kHALF)) &&
+                 (in_out[pos].format == nvinfer1::TensorFormat::kLINEAR);
+    // output
+    if (pos == 2)
+      return in_out[1].type == in_out[pos].type &&
+             in_out[1].format == in_out[pos].format;
   } else {
     return (in_out[pos].type == nvinfer1::DataType::kFLOAT ||
             (isFp16Supported() &&
@@ -367,6 +381,9 @@ nvinfer1::DataType GenericPlugin::getOutputDataType(
     int index,
     const nvinfer1::DataType* input_types,
     int nb_inputs) const TRT_NOEXCEPT {
+  if (op_desc_.Type() == "lookup_table_v2") {
+    return input_types[1];
+  }
   return input_types[0];
 }
 
@@ -472,7 +489,7 @@ int GenericPlugin::enqueue(const nvinfer1::PluginTensorDesc* input_desc,
                            cudaStream_t stream) TRT_NOEXCEPT {
   platform::CUDAPlace place(platform::GetCurrentDeviceId());
 
-  // [TODO]now generic plugin do not support INT8 precision
+  // TODO(inference): generic plugin do not support INT8 precision now.
   auto protoType2PhiType =
       [&](int proto_type,
           nvinfer1::DataType nv_dtype) -> std::pair<phi::DataType, int> {
@@ -503,8 +520,13 @@ int GenericPlugin::enqueue(const nvinfer1::PluginTensorDesc* input_desc,
     }
   };
 
+  nvinfer1::DataType data_type;
   // input
-  auto data_type = input_desc[0].type;
+  if (op_desc_.Type() == "lookup_table_v2") {
+    data_type = input_desc[1].type;
+  } else {
+    data_type = input_desc[0].type;
+  }
   CHECK((data_type == nvinfer1::DataType::kFLOAT) ||
         (data_type == nvinfer1::DataType::kHALF));
 
@@ -555,8 +577,6 @@ int GenericPlugin::enqueue(const nvinfer1::PluginTensorDesc* input_desc,
                             output_numel * data_type_and_size.second,
                             place));
 
-    phi::DenseTensor output_densetonsor(output_alloc, output_meta);
-
     (*dense_tensor_outputs_)[i] =
         std::move(phi::DenseTensor(output_alloc, output_meta));
 
diff --git a/paddle/fluid/inference/tensorrt/plugin/generic_plugin.h b/paddle/fluid/inference/tensorrt/plugin/generic_plugin.h
index 4b78f0d1ed137..e630b5264865a 100644
--- a/paddle/fluid/inference/tensorrt/plugin/generic_plugin.h
+++ b/paddle/fluid/inference/tensorrt/plugin/generic_plugin.h
@@ -15,8 +15,6 @@
 #pragma once
 
 #include <NvInfer.h>
-#include <stdio.h>
-#include <cassert>
 #include <string>
 #include <vector>
 
@@ -84,45 +82,50 @@ class GenericPlugin : public DynamicPluginTensorRT {
   // Shutdown the layer. This is called when the engine is destroyed
   void terminate() TRT_NOEXCEPT override;
 
-  void destroy() TRT_NOEXCEPT{};
+  void destroy() TRT_NOEXCEPT override{};
 
-  size_t getSerializationSize() const TRT_NOEXCEPT {
-    return op_meta_data_.size() + SerializedSize(inputs_data_type_) +
-           SerializedSize(outputs_data_type_) + SerializedSize(with_fp16_);
+  size_t getSerializationSize() const TRT_NOEXCEPT override {
+    size_t sum = 0;
+    sum += SerializedSize(inputs_data_type_);
+    sum += SerializedSize(outputs_data_type_);
+    sum += SerializedSize(with_fp16_);
+    sum += op_meta_data_.size();
+    return sum;
   }
 
-  void serialize(void* buffer) const TRT_NOEXCEPT;
+  void serialize(void* buffer) const TRT_NOEXCEPT override;
 
   // The Func in IPluginV2
-  nvinfer1::IPluginV2DynamicExt* clone() const TRT_NOEXCEPT;
+  nvinfer1::IPluginV2DynamicExt* clone() const TRT_NOEXCEPT override;
 
   nvinfer1::DimsExprs getOutputDimensions(
       int output_index,
       const nvinfer1::DimsExprs* inputs,
       int nb_inputs,
       nvinfer1::IExprBuilder& expr_builder)  // NOLINT
-      TRT_NOEXCEPT;
+      TRT_NOEXCEPT override;
 
   bool supportsFormatCombination(int pos,
                                  const nvinfer1::PluginTensorDesc* in_out,
                                  int nb_inputs,
-                                 int nb_outputs) TRT_NOEXCEPT;
+                                 int nb_outputs) TRT_NOEXCEPT override;
 
   void configurePlugin(const nvinfer1::DynamicPluginTensorDesc* in,
                        int nb_inputs,
                        const nvinfer1::DynamicPluginTensorDesc* out,
-                       int nb_outputs) TRT_NOEXCEPT;
+                       int nb_outputs) TRT_NOEXCEPT override;
 
   int enqueue(const nvinfer1::PluginTensorDesc* input_desc,
               const nvinfer1::PluginTensorDesc* output_desc,
               const void* const* inputs,
               void* const* outputs,
               void* workspace,
-              cudaStream_t stream) TRT_NOEXCEPT;
+              cudaStream_t stream) TRT_NOEXCEPT override;
 
   nvinfer1::DataType getOutputDataType(int index,
                                        const nvinfer1::DataType* input_types,
-                                       int nb_inputs) const TRT_NOEXCEPT;
+                                       int nb_inputs) const
+      TRT_NOEXCEPT override;
 
   bool isFp16Supported() {
     auto half_dtype = nvinfer1::DataType::kHALF;
@@ -146,7 +149,6 @@ class GenericPlugin : public DynamicPluginTensorRT {
   std::vector<phi::DenseTensor>* dense_tensor_outputs_{nullptr};
 
  private:
-  InputOutPutVarInfo in_out_info_;
   std::vector<int> inputs_data_type_;
   std::vector<int> outputs_data_type_;
 };
@@ -166,6 +168,7 @@ class GenericPluginCreator : public TensorRTPluginCreator {
     return new GenericPlugin(serial_data, serial_length);
   }
 };
+
 REGISTER_TRT_PLUGIN_V2(GenericPluginCreator);
 
 }  // namespace plugin
diff --git a/paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.cc b/paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.cc
index 805813cbe153c..26cb5166362b2 100644
--- a/paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.cc
+++ b/paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.cc
@@ -19,7 +19,7 @@ namespace inference {
 namespace tensorrt {
 
 bool PluginArgumentMappingContext::HasInput(const std::string& name) const {
-  auto inputs = op_desc_ptr_->Inputs();
+  auto inputs = op_desc_->Inputs();
   for (auto& i : inputs) {
     if (i.first == name && !i.second.empty()) return true;
   }
@@ -27,7 +27,7 @@ bool PluginArgumentMappingContext::HasInput(const std::string& name) const {
 }
 
 bool PluginArgumentMappingContext::HasOutput(const std::string& name) const {
-  auto outputs = op_desc_ptr_->Outputs();
+  auto outputs = op_desc_->Outputs();
   for (auto& i : outputs) {
     if (i.first == name && !i.second.empty()) return true;
   }
@@ -35,47 +35,44 @@ bool PluginArgumentMappingContext::HasOutput(const std::string& name) const {
 }
 
 bool PluginArgumentMappingContext::HasAttr(const std::string& name) const {
-  return op_desc_ptr_->HasAttr(name);
+  return op_desc_->HasAttr(name);
 }
 
 paddle::any PluginArgumentMappingContext::Attr(
     const std::string& attr_name) const {
-  auto attr_type = op_desc_ptr_->GetAttrType(attr_name);
+  auto attr_type = op_desc_->GetAttrType(attr_name);
   switch (attr_type) {
     case framework::proto::AttrType::INT: {
-      return PADDLE_GET_CONST(int, op_desc_ptr_->GetAttr(attr_name));
+      return PADDLE_GET_CONST(int, op_desc_->GetAttr(attr_name));
       break;
     };
     case framework::proto::AttrType::FLOAT: {
-      return PADDLE_GET_CONST(float, op_desc_ptr_->GetAttr(attr_name));
+      return PADDLE_GET_CONST(float, op_desc_->GetAttr(attr_name));
       break;
     };
     case framework::proto::AttrType::STRING: {
-      return PADDLE_GET_CONST(std::string, op_desc_ptr_->GetAttr(attr_name));
+      return PADDLE_GET_CONST(std::string, op_desc_->GetAttr(attr_name));
       break;
     };
     case framework::proto::AttrType::INTS: {
-      return PADDLE_GET_CONST(std::vector<int>,
-                              op_desc_ptr_->GetAttr(attr_name));
+      return PADDLE_GET_CONST(std::vector<int>, op_desc_->GetAttr(attr_name));
       break;
     };
     case framework::proto::AttrType::FLOATS: {
-      return PADDLE_GET_CONST(std::vector<float>,
-                              op_desc_ptr_->GetAttr(attr_name));
+      return PADDLE_GET_CONST(std::vector<float>, op_desc_->GetAttr(attr_name));
       break;
     };
     case framework::proto::AttrType::STRINGS: {
       return PADDLE_GET_CONST(std::vector<std::string>,
-                              op_desc_ptr_->GetAttr(attr_name));
+                              op_desc_->GetAttr(attr_name));
       break;
     };
     case framework::proto::AttrType::BOOLEAN: {
-      return PADDLE_GET_CONST(bool, op_desc_ptr_->GetAttr(attr_name));
+      return PADDLE_GET_CONST(bool, op_desc_->GetAttr(attr_name));
       break;
     };
     case framework::proto::AttrType::BOOLEANS: {
-      return PADDLE_GET_CONST(std::vector<bool>,
-                              op_desc_ptr_->GetAttr(attr_name));
+      return PADDLE_GET_CONST(std::vector<bool>, op_desc_->GetAttr(attr_name));
       break;
     };
     default: {
@@ -87,54 +84,82 @@ paddle::any PluginArgumentMappingContext::Attr(
 }
 
 size_t PluginArgumentMappingContext::InputSize(const std::string& name) const {
-  return op_desc_ptr_->Inputs().at(name).size();
+  return op_desc_->Inputs().at(name).size();
 }
+
 size_t PluginArgumentMappingContext::OutputSize(const std::string& name) const {
-  return op_desc_ptr_->Outputs().at(name).size();
+  return op_desc_->Outputs().at(name).size();
 }
+
 bool PluginArgumentMappingContext::IsDenseTensorInput(
     const std::string& name) const {
-  return false;
+  return true;
 }
+
 bool PluginArgumentMappingContext::IsDenseTensorInputs(
     const std::string& name) const {
-  return false;
+  return true;
 }
-bool PluginArgumentMappingContext::IsSelectedRowsInput(
+
+bool PluginArgumentMappingContext::IsDenseTensorVectorInput(
     const std::string& name) const {
+  PADDLE_THROW(phi::errors::Unimplemented(
+      "Not supported for input vector of DenseTensor."));
   return false;
 }
-bool PluginArgumentMappingContext::IsSelectedRowsInputs(
+
+bool PluginArgumentMappingContext::IsDenseTensorOutput(
     const std::string& name) const {
-  return false;
+  return true;
 }
-bool PluginArgumentMappingContext::IsSparseCooTensorInput(
+
+bool PluginArgumentMappingContext::IsSelectedRowsInput(
     const std::string& name) const {
+  PADDLE_THROW(
+      phi::errors::Unimplemented("Not supported for input of SelectedRows."));
   return false;
 }
 
-bool PluginArgumentMappingContext::IsSparseCooTensorOutput(
+bool PluginArgumentMappingContext::IsSelectedRowsInputs(
     const std::string& name) const {
+  PADDLE_THROW(
+      phi::errors::Unimplemented("Not supported for inputs of SelectedRows."));
   return false;
 }
 
-bool PluginArgumentMappingContext::IsSparseCsrTensorInput(
+bool PluginArgumentMappingContext::IsSelectedRowsOutput(
     const std::string& name) const {
+  PADDLE_THROW(
+      phi::errors::Unimplemented("Not supported for output of SelectedRows."));
   return false;
 }
-bool PluginArgumentMappingContext::IsDenseTensorVectorInput(
+
+bool PluginArgumentMappingContext::IsSparseCooTensorInput(
     const std::string& name) const {
+  PADDLE_THROW(phi::errors::Unimplemented(
+      "Not supported for input of SparseCooTensor."));
   return false;
 }
 
-bool PluginArgumentMappingContext::IsDenseTensorOutput(
+bool PluginArgumentMappingContext::IsSparseCooTensorOutput(
     const std::string& name) const {
+  PADDLE_THROW(phi::errors::Unimplemented(
+      "Not supported for output of SparseCooTensor."));
   return false;
 }
-bool PluginArgumentMappingContext::IsSelectedRowsOutput(
+
+bool PluginArgumentMappingContext::IsSparseCsrTensorInput(
     const std::string& name) const {
+  PADDLE_THROW(phi::errors::Unimplemented(
+      "Not supported for input of SparseCsrTensor."));
   return false;
 }
+
+bool PluginArgumentMappingContext::IsForInferShape() const {
+  PADDLE_THROW(phi::errors::Unimplemented("Not supported for InferShape."));
+  return false;
+}
+
 }  // namespace tensorrt
 }  // namespace inference
 }  // namespace paddle
diff --git a/paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.h b/paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.h
index c0c30f3ac57b0..7e5aa1ef813d3 100644
--- a/paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.h
+++ b/paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.h
@@ -25,8 +25,8 @@ namespace tensorrt {
 
 class PluginArgumentMappingContext : public ::phi::ArgumentMappingContext {
  public:
-  explicit PluginArgumentMappingContext(framework::OpDesc* op_desc_ptr)
-      : op_desc_ptr_(op_desc_ptr) {}
+  explicit PluginArgumentMappingContext(const framework::OpDesc* op_desc)
+      : op_desc_(op_desc) {}
 
   bool HasInput(const std::string& name) const override;
 
@@ -60,11 +60,12 @@ class PluginArgumentMappingContext : public ::phi::ArgumentMappingContext {
 
   bool IsSelectedRowsOutput(const std::string& name) const override;
 
-  bool IsForInferShape() const override { return false; }
+  bool IsForInferShape() const override;
 
  private:
-  framework::OpDesc* op_desc_ptr_;
+  const framework::OpDesc* op_desc_;
 };
+
 }  // namespace tensorrt
 }  // namespace inference
 }  // namespace paddle
diff --git a/paddle/fluid/inference/tensorrt/test_arg_mapping_context.cc b/paddle/fluid/inference/tensorrt/test_arg_mapping_context.cc
index d514fa0bb1af8..97090518153d1 100644
--- a/paddle/fluid/inference/tensorrt/test_arg_mapping_context.cc
+++ b/paddle/fluid/inference/tensorrt/test_arg_mapping_context.cc
@@ -117,15 +117,10 @@ TEST(ArgMappingContexTest, BasicFunction) {
 
   EXPECT_EQ(context.InputSize("X"), true);
   EXPECT_EQ(context.OutputSize("Out"), true);
-  EXPECT_EQ(context.IsDenseTensorInput("X"), false);
-  EXPECT_EQ(context.IsDenseTensorInputs("X"), false);
-  EXPECT_EQ(context.IsSelectedRowsInput("X"), false);
-  EXPECT_EQ(context.IsDenseTensorVectorInput("X"), false);
-
-  EXPECT_EQ(context.IsDenseTensorOutput("Out"), false);
-  EXPECT_EQ(context.IsSelectedRowsOutput("Out"), false);
-  EXPECT_EQ(context.IsSparseCooTensorOutput("Out"), false);
-  EXPECT_EQ(context.IsForInferShape(), false);
+  EXPECT_EQ(context.IsDenseTensorInput("X"), true);
+  EXPECT_EQ(context.IsDenseTensorInputs("X"), true);
+
+  EXPECT_EQ(context.IsDenseTensorOutput("Out"), true);
 }
 
 }  // namespace tensorrt

From a01b20ddeeee55ecc3d9e799cdf33ba5755de6b6 Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Mon, 8 May 2023 11:17:56 +0800
Subject: [PATCH 205/405] [test]mv fluid op mkldnn to test/cpp/fluid/mkldnn
 (#53458)

---
 paddle/fluid/operators/CMakeLists.txt         |  6 ---
 .../operators/mkldnn/caching_tests.cmake      | 20 -------
 .../operators/mkldnn/inplace_op_tests.cmake   | 12 -----
 .../operators/mkldnn/nhwc_op_tests.cmake      | 18 -------
 test/cpp/fluid/CMakeLists.txt                 |  6 +--
 test/cpp/fluid/mkldnn/CMakeLists.txt          | 52 +++++++++++++++++++
 .../cpp/fluid}/mkldnn/test_mkldnn_caching.cc  |  0
 .../fluid}/mkldnn/test_mkldnn_op_inplace.cc   |  0
 .../cpp/fluid}/mkldnn/test_mkldnn_op_nhwc.cc  |  0
 9 files changed, 55 insertions(+), 59 deletions(-)
 delete mode 100644 paddle/fluid/operators/mkldnn/caching_tests.cmake
 delete mode 100644 paddle/fluid/operators/mkldnn/inplace_op_tests.cmake
 delete mode 100644 paddle/fluid/operators/mkldnn/nhwc_op_tests.cmake
 create mode 100644 test/cpp/fluid/mkldnn/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/mkldnn/test_mkldnn_caching.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/mkldnn/test_mkldnn_op_inplace.cc (100%)
 rename {paddle/fluid/operators => test/cpp/fluid}/mkldnn/test_mkldnn_op_nhwc.cc (100%)

diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt
index a0d89394ece6a..0cefb7e69ace3 100644
--- a/paddle/fluid/operators/CMakeLists.txt
+++ b/paddle/fluid/operators/CMakeLists.txt
@@ -175,12 +175,6 @@ endif()
 
 set(GLOB_OP_LIB ${OP_LIBRARY} CACHE INTERNAL "Global OP library")
 
-if(WITH_MKLDNN)
-include(mkldnn/inplace_op_tests.cmake)
-include(mkldnn/caching_tests.cmake)
-include(mkldnn/nhwc_op_tests.cmake)
-endif()
-
 if(WITH_UNITY_BUILD)
     # Using Unity Build to compile operators, `register_operator` will cause
     # the unity library to lose some symbols.
diff --git a/paddle/fluid/operators/mkldnn/caching_tests.cmake b/paddle/fluid/operators/mkldnn/caching_tests.cmake
deleted file mode 100644
index 49f08622265d0..0000000000000
--- a/paddle/fluid/operators/mkldnn/caching_tests.cmake
+++ /dev/null
@@ -1,20 +0,0 @@
-set(TEST_MKLDNN_CACHING_DEPS
-    op_registry
-    elementwise_mul_op
-    elementwise_add_op
-    activation_op
-    softmax_op
-    conv_op
-    im2col
-    vol2col
-    softmax
-    scope
-    device_context
-    enforce)
-if(WITH_GPU OR WITH_ROCM)
-  set(TEST_MKLDNN_CACHING_DEPS ${TEST_MKLDNN_CACHING_DEPS} depthwise_conv)
-endif()
-cc_test(
-  test_mkldnn_caching
-  SRCS mkldnn/test_mkldnn_caching.cc
-  DEPS ${TEST_MKLDNN_CACHING_DEPS})
diff --git a/paddle/fluid/operators/mkldnn/inplace_op_tests.cmake b/paddle/fluid/operators/mkldnn/inplace_op_tests.cmake
deleted file mode 100644
index 18893e22ec85b..0000000000000
--- a/paddle/fluid/operators/mkldnn/inplace_op_tests.cmake
+++ /dev/null
@@ -1,12 +0,0 @@
-cc_test(
-  test_mkldnn_op_inplace
-  SRCS mkldnn/test_mkldnn_op_inplace.cc
-  DEPS op_registry
-       elementwise_add_op
-       activation_op
-       softmax_op
-       softmax
-       scope
-       device_context
-       enforce
-       executor)
diff --git a/paddle/fluid/operators/mkldnn/nhwc_op_tests.cmake b/paddle/fluid/operators/mkldnn/nhwc_op_tests.cmake
deleted file mode 100644
index bda7ab2a3b3a2..0000000000000
--- a/paddle/fluid/operators/mkldnn/nhwc_op_tests.cmake
+++ /dev/null
@@ -1,18 +0,0 @@
-cc_test_old(
-  test_mkldnn_op_nhwc
-  SRCS
-  mkldnn/test_mkldnn_op_nhwc.cc
-  DEPS
-  op_registry
-  pool_op
-  shape_op
-  crop_op
-  activation_op
-  generated_op
-  pooling
-  transpose_op
-  fused_transpose_op
-  scope
-  device_context
-  enforce
-  executor)
diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt
index 029cf3db9053d..34f45371cca2a 100644
--- a/test/cpp/fluid/CMakeLists.txt
+++ b/test/cpp/fluid/CMakeLists.txt
@@ -11,9 +11,9 @@ if(WITH_LITE)
   add_subdirectory(lite)
 endif()
 add_subdirectory(math)
-# if(WITH_MKLDNN)
-#   add_subdirectory(mkldnn)
-# endif()
+if(WITH_MKLDNN)
+  add_subdirectory(mkldnn)
+endif()
 add_subdirectory(nccl)
 if(WITH_PSCORE)
   add_subdirectory(pscore)
diff --git a/test/cpp/fluid/mkldnn/CMakeLists.txt b/test/cpp/fluid/mkldnn/CMakeLists.txt
new file mode 100644
index 0000000000000..34f3ce8959c68
--- /dev/null
+++ b/test/cpp/fluid/mkldnn/CMakeLists.txt
@@ -0,0 +1,52 @@
+cc_test(
+  test_mkldnn_op_inplace
+  SRCS test_mkldnn_op_inplace.cc
+  DEPS op_registry
+       elementwise_add_op
+       activation_op
+       softmax_op
+       softmax
+       scope
+       device_context
+       enforce
+       executor)
+
+set(TEST_MKLDNN_CACHING_DEPS
+    op_registry
+    elementwise_mul_op
+    elementwise_add_op
+    activation_op
+    softmax_op
+    conv_op
+    im2col
+    vol2col
+    softmax
+    scope
+    device_context
+    enforce)
+if(WITH_GPU OR WITH_ROCM)
+  set(TEST_MKLDNN_CACHING_DEPS ${TEST_MKLDNN_CACHING_DEPS} depthwise_conv)
+endif()
+cc_test(
+  test_mkldnn_caching
+  SRCS test_mkldnn_caching.cc
+  DEPS ${TEST_MKLDNN_CACHING_DEPS})
+
+cc_test_old(
+  test_mkldnn_op_nhwc
+  SRCS
+  test_mkldnn_op_nhwc.cc
+  DEPS
+  op_registry
+  pool_op
+  shape_op
+  crop_op
+  activation_op
+  generated_op
+  pooling
+  transpose_op
+  fused_transpose_op
+  scope
+  device_context
+  enforce
+  executor)
diff --git a/paddle/fluid/operators/mkldnn/test_mkldnn_caching.cc b/test/cpp/fluid/mkldnn/test_mkldnn_caching.cc
similarity index 100%
rename from paddle/fluid/operators/mkldnn/test_mkldnn_caching.cc
rename to test/cpp/fluid/mkldnn/test_mkldnn_caching.cc
diff --git a/paddle/fluid/operators/mkldnn/test_mkldnn_op_inplace.cc b/test/cpp/fluid/mkldnn/test_mkldnn_op_inplace.cc
similarity index 100%
rename from paddle/fluid/operators/mkldnn/test_mkldnn_op_inplace.cc
rename to test/cpp/fluid/mkldnn/test_mkldnn_op_inplace.cc
diff --git a/paddle/fluid/operators/mkldnn/test_mkldnn_op_nhwc.cc b/test/cpp/fluid/mkldnn/test_mkldnn_op_nhwc.cc
similarity index 100%
rename from paddle/fluid/operators/mkldnn/test_mkldnn_op_nhwc.cc
rename to test/cpp/fluid/mkldnn/test_mkldnn_op_nhwc.cc

From 2f5033822196dcea8513fc2116c2b13cc23f3c3c Mon Sep 17 00:00:00 2001
From: Yuanle Liu <yuanlehome@163.com>
Date: Mon, 8 May 2023 11:34:24 +0800
Subject: [PATCH 206/405] add ut for lookup_table op trt converter (#53563)

---
 .../test_trt_convert_lookup_table_v2.py       | 142 ++++++++++++++++++
 test/ir/inference/test_trt_convert_p_norm.py  |   2 +-
 2 files changed, 143 insertions(+), 1 deletion(-)
 create mode 100644 test/ir/inference/test_trt_convert_lookup_table_v2.py

diff --git a/test/ir/inference/test_trt_convert_lookup_table_v2.py b/test/ir/inference/test_trt_convert_lookup_table_v2.py
new file mode 100644
index 0000000000000..60bd6c6f267af
--- /dev/null
+++ b/test/ir/inference/test_trt_convert_lookup_table_v2.py
@@ -0,0 +1,142 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from functools import partial
+from typing import Any, Dict, List
+
+import numpy as np
+from program_config import ProgramConfig, TensorConfig
+from trt_layer_auto_scan_test import TrtLayerAutoScanTest
+
+import paddle.inference as paddle_infer
+
+
+class TrtConvertLookupTableV2Test(TrtLayerAutoScanTest):
+    def sample_program_configs(self):
+        def generate_input1(dims, attrs: List[Dict[str, Any]]):
+            if dims == 1:
+                return np.array([32]).astype(np.int64)
+            elif dims == 2:
+                return np.array([[3, 16, 24], [6, 4, 47]]).astype(np.int64)
+            else:
+                return np.array(
+                    [
+                        [[3, 16, 24], [30, 16, 14], [2, 6, 24]],
+                        [[3, 26, 34], [3, 16, 24], [3, 6, 4]],
+                        [[3, 16, 24], [53, 16, 54], [30, 1, 24]],
+                    ]
+                ).astype(np.int64)
+
+        def generate_input2(dims, attrs: List[Dict[str, Any]]):
+            return np.random.uniform(-1, 1, [64, 4]).astype('float32')
+
+        for dims in [1, 2, 3]:
+            self.dims = dims
+
+            ops_config = [
+                {
+                    "op_type": "lookup_table_v2",
+                    "op_inputs": {"Ids": ["indices"], "W": ["data"]},
+                    "op_outputs": {"Out": ["out_data"]},
+                    "op_attrs": {},
+                }
+            ]
+            ops = self.generate_op_config(ops_config)
+
+            program_config = ProgramConfig(
+                ops=ops,
+                weights={
+                    "data": TensorConfig(
+                        data_gen=partial(generate_input2, {}, {})
+                    )
+                },
+                inputs={
+                    "indices": TensorConfig(
+                        data_gen=partial(generate_input1, dims, {})
+                    )
+                },
+                outputs=["out_data"],
+            )
+
+            yield program_config
+
+    def sample_predictor_configs(
+        self, program_config
+    ) -> (paddle_infer.Config, List[int], float):
+        def generate_dynamic_shape(attrs):
+            if self.dims == 1:
+                self.dynamic_shape.min_input_shape = {
+                    "indices": [1],
+                    "data": [64, 4],
+                }
+                self.dynamic_shape.max_input_shape = {
+                    "indices": [1],
+                    "data": [64, 4],
+                }
+                self.dynamic_shape.opt_input_shape = {
+                    "indices": [1],
+                    "data": [64, 4],
+                }
+            elif self.dims == 2:
+                self.dynamic_shape.min_input_shape = {
+                    "indices": [2, 3],
+                    "data": [64, 4],
+                }
+                self.dynamic_shape.max_input_shape = {
+                    "indices": [2, 3],
+                    "data": [64, 4],
+                }
+                self.dynamic_shape.opt_input_shape = {
+                    "indices": [2, 3],
+                    "data": [64, 4],
+                }
+            else:
+                self.dynamic_shape.min_input_shape = {
+                    "indices": [3, 3, 3],
+                    "data": [64, 4],
+                }
+                self.dynamic_shape.max_input_shape = {
+                    "indices": [3, 3, 3],
+                    "data": [64, 4],
+                }
+                self.dynamic_shape.opt_input_shape = {
+                    "indices": [3, 3, 3],
+                    "data": [64, 4],
+                }
+
+        def generate_trt_nodes_num(attrs, dynamic_shape):
+            return 1, 2
+
+        attrs = [
+            program_config.ops[i].attrs for i in range(len(program_config.ops))
+        ]
+
+        # for dynamic_shape mode
+        generate_dynamic_shape(attrs)
+        self.trt_param.precision = paddle_infer.PrecisionType.Float32
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), 1e-5
+        self.trt_param.precision = paddle_infer.PrecisionType.Half
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), (1e-3, 1e-3)
+
+    def test(self):
+        self.run_test()
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/ir/inference/test_trt_convert_p_norm.py b/test/ir/inference/test_trt_convert_p_norm.py
index 5aa48135a89f6..6fc8e5ec87cb0 100644
--- a/test/ir/inference/test_trt_convert_p_norm.py
+++ b/test/ir/inference/test_trt_convert_p_norm.py
@@ -23,7 +23,7 @@
 import paddle.inference as paddle_infer
 
 
-class TrtConvertCeluTest(TrtLayerAutoScanTest):
+class TrtConvertPNormTest(TrtLayerAutoScanTest):
     def sample_program_configs(self):
         def generate_input1(dims, attrs: List[Dict[str, Any]]):
             if dims == 1:

From f3f3d57a159caf3b77f93a4d86cb233e6a1c159a Mon Sep 17 00:00:00 2001
From: yangguohao <70266361+yangguohao@users.noreply.github.com>
Date: Mon, 8 May 2023 11:55:52 +0800
Subject: [PATCH 207/405] [Dy2St]Following update of register_hook for static
 mode (#53572)

---
 python/paddle/fluid/framework.py              |  2 -
 .../paddle/jit/dy2static/ast_transformer.py   |  2 +
 python/paddle/jit/dy2static/ast_utils.py      | 84 -----------------
 .../jit/dy2static/tensorhook_transformer.py   | 89 +++++++++++++++++++
 python/paddle/jit/dy2static/utils.py          | 25 +++---
 test/dygraph_to_static/test_tensor_hook.py    | 14 ++-
 6 files changed, 108 insertions(+), 108 deletions(-)
 create mode 100644 python/paddle/jit/dy2static/tensorhook_transformer.py

diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py
index 7c183ea34767e..184d6cb326e5b 100644
--- a/python/paddle/fluid/framework.py
+++ b/python/paddle/fluid/framework.py
@@ -1645,8 +1645,6 @@ def register_hook(self, hook):
 
         def backward_hook_wrapper(dy):
             """call the backward hook in ."""
-            import numpy as np
-
             return hook(np.array(dy))
 
         def forward_hook_wrapper(x):
diff --git a/python/paddle/jit/dy2static/ast_transformer.py b/python/paddle/jit/dy2static/ast_transformer.py
index 3626709bc3714..66d334dc7fcc0 100644
--- a/python/paddle/jit/dy2static/ast_transformer.py
+++ b/python/paddle/jit/dy2static/ast_transformer.py
@@ -38,6 +38,7 @@
 from .return_transformer import ReturnTransformer
 from .static_analysis import StaticAnalysisVisitor
 from .tensor_shape_transformer import TensorShapeTransformer
+from .tensorhook_transformer import RegisterHookTransformer
 from .typehint_transformer import TypeHintTransformer
 from .utils import ast_to_source_code
 
@@ -92,6 +93,7 @@ def transfer_from_node_type(self, node_wrapper):
         self.visit(node_wrapper.node)
 
         transformers = [
+            RegisterHookTransformer,
             EarlyReturnTransformer,
             BasicApiTransformer,  # Basic Api
             TensorShapeTransformer,  # Tensor.shape -> paddle.shape(Tensor)
diff --git a/python/paddle/jit/dy2static/ast_utils.py b/python/paddle/jit/dy2static/ast_utils.py
index a9f40b7ce6548..7724cc1b4a13f 100644
--- a/python/paddle/jit/dy2static/ast_utils.py
+++ b/python/paddle/jit/dy2static/ast_utils.py
@@ -14,9 +14,6 @@
 
 
 import ast
-import collections
-import inspect
-import textwrap
 
 import astor
 
@@ -41,84 +38,3 @@ def pretty_source(source):
 
     source_code = astor.to_source(ast_node, pretty_source=pretty_source)
     return source_code
-
-
-class RegisterHookVisitor(gast.NodeVisitor):
-    def __init__(self, func_name):
-        self.register_hook_pos_map = collections.defaultdict(list)
-        self.assignment_pos_map = collections.defaultdict(list)
-        self.func_name = func_name
-
-    def visit_FunctionDef(self, func_def):
-        # The inner function that has register_hook will not be processed
-        if func_def.name != self.func_name:
-            return
-        register_hook_pos_map = self.register_hook_pos_map
-        assignment_pos_map = self.assignment_pos_map
-
-        for i in range(len(func_def.body) - 1, -1, -1):
-
-            body = func_def.body[i]
-            # Check if the code body contains the register_hook
-            if isinstance(body, ast.Expr):
-                for node in ast.walk(body):
-                    if (
-                        isinstance(node, ast.Attribute)
-                        and node.attr == 'register_hook'
-                    ):
-                        # parameter name for register_hook
-                        param_name = node.value.id
-                        register_hook_pos_map[param_name].append(i)
-            elif isinstance(body, ast.Assign):
-                for target in body.targets:
-                    assignment_pos_map[target.id].append(i)
-
-        # Confirm the order
-        order_map = {}
-        for k, idx_list in register_hook_pos_map.items():
-            for idx in idx_list:
-                if k not in assignment_pos_map:
-                    order_map[idx] = 1
-                else:
-                    for assignment_idx in assignment_pos_map[k]:
-                        if idx > assignment_idx:
-                            order_map[idx] = assignment_idx + 1
-                            break
-        code_order = [*range(len(func_def.body))]
-        for k, v in sorted(order_map.items(), key=lambda x: x[1], reverse=True):
-            if k == v:
-                continue
-            code_order.remove(k)
-            code_order.insert(v, k)
-
-        # rearrange the code according to the specified order
-        new_body = [func_def.body[i] for i in code_order]
-        func_def.body = new_body
-
-
-def modify_function_code(func):
-    """
-    Modify the function code for the register hook
-    """
-
-    func_ast = ast.parse(textwrap.dedent(inspect.getsource(func)))
-    # check if there is register_hook on code after visit the tree.
-    check_register_hook = next(
-        (
-            node
-            for node in ast.walk(func_ast)
-            if isinstance(node, ast.Attribute) and node.attr == 'register_hook'
-        ),
-        None,
-    )
-    if check_register_hook is None:
-        return
-
-    visitor = RegisterHookVisitor(func.__name__)
-    visitor.visit(func_ast)
-
-    def pretty_source(source):
-        return ''.join(source)
-
-    new_code = astor.to_source(func_ast, pretty_source=pretty_source)
-    return new_code
diff --git a/python/paddle/jit/dy2static/tensorhook_transformer.py b/python/paddle/jit/dy2static/tensorhook_transformer.py
new file mode 100644
index 0000000000000..ff0c4b67ca104
--- /dev/null
+++ b/python/paddle/jit/dy2static/tensorhook_transformer.py
@@ -0,0 +1,89 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+
+from paddle.utils import gast
+
+from .base_transformer import BaseTransformer
+
+
+class RegisterHookTransformer(BaseTransformer):
+    def __init__(self, wrapper_root):
+        self.register_hook_pos_map = collections.defaultdict(list)
+        self.assignment_pos_map = collections.defaultdict(list)
+        self.root = wrapper_root.node
+
+    def transform(self):
+        """
+        Main function to transform AST.
+        """
+        self.visit(self.root)
+
+    def visit_FunctionDef(self, func_def):
+        # The inner function that has register_hook will not be processed
+        check_register_hook = next(
+            (
+                node
+                for node in gast.walk(func_def)
+                if isinstance(node, gast.Attribute)
+                and node.attr == 'register_hook'
+            ),
+            None,
+        )
+        if check_register_hook is None:
+            return func_def
+
+        register_hook_pos_map = self.register_hook_pos_map
+        assignment_pos_map = self.assignment_pos_map
+
+        for i in range(len(func_def.body) - 1, -1, -1):
+
+            body = func_def.body[i]
+            # Check if the code body contains the register_hook
+            if isinstance(body, gast.Expr):
+                for node in gast.walk(body):
+                    if (
+                        isinstance(node, gast.Attribute)
+                        and node.attr == 'register_hook'
+                    ):
+                        # parameter name for register_hook
+                        param_name = node.value.id
+                        register_hook_pos_map[param_name].append(i)
+            elif isinstance(body, gast.Assign):
+                for target in body.targets:
+                    assignment_pos_map[target.id].append(i)
+
+        # Confirm the order
+        order_map = {}
+        for k, idx_list in register_hook_pos_map.items():
+            for idx in idx_list:
+                if k not in assignment_pos_map:
+                    order_map[idx] = 1
+                else:
+                    for assignment_idx in assignment_pos_map[k]:
+                        if idx > assignment_idx:
+                            order_map[idx] = assignment_idx + 1
+                            break
+        code_order = [*range(len(func_def.body))]
+        for k, v in sorted(order_map.items(), key=lambda x: x[1], reverse=True):
+            if k == v:
+                continue
+            code_order.remove(k)
+            code_order.insert(v, k)
+
+        # rearrange the code according to the specified order
+        new_body = [func_def.body[i] for i in code_order]
+        func_def.body = new_body
+        return func_def
diff --git a/python/paddle/jit/dy2static/utils.py b/python/paddle/jit/dy2static/utils.py
index 1dbbf3669ca4b..471d1d4cae1e6 100644
--- a/python/paddle/jit/dy2static/utils.py
+++ b/python/paddle/jit/dy2static/utils.py
@@ -38,7 +38,7 @@
 from paddle.fluid.wrapped_decorator import signature_safe_contextmanager
 from paddle.utils import gast
 
-from .ast_utils import ast_to_source_code, modify_function_code
+from .ast_utils import ast_to_source_code
 from .static_analysis import StaticAnalysisVisitor
 from .utils_helper import DYGRAPH_MODULE_PREFIX  # noqa: F401
 from .utils_helper import DYGRAPH_TO_STATIC_MODULE_PREFIX  # noqa: F401
@@ -643,20 +643,17 @@ def func_to_source_code(function, dedent=True):
                 type(function).__name__
             )
         )
-    # return modified function source code if there is 'register_hook', otherwise return None
-    source_code = modify_function_code(function)
-
-    if source_code is None:
-        source_code_list, _ = inspect.getsourcelines(function)
-        # Replace comments with blank lines so that error messages are not misplaced
-        source_code_list = [
-            line if not line.lstrip().startswith('#') else '\n'
-            for line in source_code_list
-        ]
-        source_code = ''.join(source_code_list)
 
-        if dedent:
-            source_code = textwrap.dedent(source_code)
+    source_code_list, _ = inspect.getsourcelines(function)
+    # Replace comments with blank lines so that error messages are not misplaced
+    source_code_list = [
+        line if not line.lstrip().startswith('#') else '\n'
+        for line in source_code_list
+    ]
+    source_code = ''.join(source_code_list)
+
+    if dedent:
+        source_code = textwrap.dedent(source_code)
 
     return source_code
 
diff --git a/test/dygraph_to_static/test_tensor_hook.py b/test/dygraph_to_static/test_tensor_hook.py
index a943c40495de4..fc53fefc95ae6 100644
--- a/test/dygraph_to_static/test_tensor_hook.py
+++ b/test/dygraph_to_static/test_tensor_hook.py
@@ -45,7 +45,7 @@ def h(g):
         jit_f = to_static(f)
         loss = jit_f(x_jit)
         loss.backward()
-        self.assertTrue(np.allclose(x.grad.numpy(), x_jit.grad.numpy()))
+        np.testing.assert_allclose(x.grad.numpy(), x_jit.grad.numpy())
 
     def test_hook_for_reassignment_parameter(self):
         def f(x):
@@ -68,7 +68,7 @@ def h(g):
         jit_f = to_static(f)
         loss = jit_f(x_jit)
         loss.backward()
-        self.assertTrue(np.allclose(x.grad.numpy(), x_jit.grad.numpy()))
+        np.testing.assert_allclose(x.grad.numpy(), x_jit.grad.numpy())
 
     def test_hook_for_repeat_register(self):
         def f(x):
@@ -91,7 +91,7 @@ def h(g):
         jit_f = to_static(f)
         loss = jit_f(x_jit)
         loss.backward()
-        self.assertTrue(np.allclose(x.grad.numpy(), x_jit.grad.numpy()))
+        np.testing.assert_allclose(x.grad.numpy(), x_jit.grad.numpy())
 
     def test_hook_in_init_for_layer(self):
         def hook(grad):
@@ -120,11 +120,9 @@ def forward(self, x):
         loss_jit = jit_layer(image_jit)
         loss_jit.backward()
         loss.backward()
-        self.assertTrue(
-            np.allclose(
-                layer.parameters()[0].grad.numpy(),
-                jit_layer.parameters()[0].grad.numpy(),
-            )
+        np.testing.assert_allclose(
+            layer.parameters()[0].grad.numpy(),
+            jit_layer.parameters()[0].grad.numpy(),
         )
 
     # def test_hook_in_forward_for_layer(self):

From ac3ff47321511b81d938e6db445932a49a6afbf3 Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Mon, 8 May 2023 13:38:43 +0800
Subject: [PATCH 208/405] change flash_attn commit id

---
 cmake/external/flashattn.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index 8cbcdb717223a..21267f3f023fc 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -19,8 +19,8 @@ add_definitions(-DPADDLE_WITH_FLASHATTN)
 set(FLASHATTN_PREFIX_DIR ${THIRD_PARTY_PATH}/flashattn)
 set(FLASHATTN_SOURCE_SUBDIR csrc/flash_attn)
 set(FLASHATTN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flashattn)
-set(FLASHATTN_REPOSITORY ${GIT_URL}/JamesLim-sy/flash-attention.git)
-set(FLASHATTN_TAG 93486d94943605436bb4cb636d9ac4a109d3fcaf)
+set(FLASHATTN_REPOSITORY ${GIT_URL}/PaddlePaddle/flash-attention.git)
+set(FLASHATTN_TAG 209f02b3852f1f6672b01e00729f20f0e52633a9)
 
 set(FLASHATTN_INCLUDE_DIR
     "${FLASHATTN_INSTALL_DIR}/include"

From fe91940004a6f90ac220c920ee3059f04dc320f5 Mon Sep 17 00:00:00 2001
From: JYChen <zoooo0820@qq.com>
Date: Mon, 8 May 2023 14:34:26 +0800
Subject: [PATCH 209/405] throw warning at __getitem__, not slice_utils
 (#53579)

---
 paddle/fluid/pybind/eager_method.cc    | 59 +++++++++++--------
 paddle/fluid/pybind/imperative.cc      | 81 ++++++++++++++++----------
 paddle/phi/kernels/funcs/slice_utils.h | 12 ----
 python/paddle/fluid/variable_index.py  |  4 ++
 4 files changed, 89 insertions(+), 67 deletions(-)

diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc
index 28a86b8db8a3c..be2771293d40d 100644
--- a/paddle/fluid/pybind/eager_method.cc
+++ b/paddle/fluid/pybind/eager_method.cc
@@ -925,36 +925,49 @@ static PyObject* tensor__getitem_index_not_tensor(TensorObject* self,
   }
 
   bool set_to_1d = FLAGS_set_to_1d;
-  if (!none_axes.empty()) {
-    if (set_to_1d) {
-      // NOTE(zoooo0820): When all axes are decreased, the output will be 1-D
-      // with FLAGS_set_to_1d=True. In this case, one `None` should be pop out,
-      // otherwise the output shape will be not correct.
-      if (static_cast<int>(decrease_axis.size()) == tensor->dims().size()) {
+
+  if (set_to_1d) {
+    // NOTE(zoooo0820): When all axes are decreased, the output will be 1-D
+    // with FLAGS_set_to_1d=True. In this case, one `None` should be pop out,
+    // otherwise the output shape will be not correct.
+    if (static_cast<int>(decrease_axis.size()) == tensor->dims().size()) {
+      VLOG(0)
+          << "Warning: In Tensor '__getitem__', if the number of scalar "
+             "elements "
+             "in the index is equal to the rank of the Tensor, the output "
+             "should "
+             "be 0-D. In order to be consistent with the behavior of previous "
+             "versions, it will be processed to 1-D. But it is not correct and "
+             "will be "
+             "removed in release 2.6. "
+             "If 1-D is still wanted, please modify the index element from "
+             "scalar to slice "
+             "(e.g. 'x[i]' => 'x[i:i+1]'). ";
+      if (!none_axes.empty()) {
         none_axes.pop_back();
       }
     }
-    if (!none_axes.empty()) {
-      paddle::Tensor new_out;
-      {
-        eager_gil_scoped_release guard;
-        // Deal with cases that decrease_axes is not empty
-        // For example:
-        // # x.shape: (2,3,4)
-        // out = x[0, 0:2, None] # out.shape : (2, 1, 4)
-        for (auto& axis : none_axes) {
-          int len = 0;
-          for (int da : decrease_axis) {
-            if (da < axis) {
-              len++;
-            }
+  }
+  if (!none_axes.empty()) {
+    paddle::Tensor new_out;
+    {
+      eager_gil_scoped_release guard;
+      // Deal with cases that decrease_axes is not empty
+      // For example:
+      // # x.shape: (2,3,4)
+      // out = x[0, 0:2, None] # out.shape : (2, 1, 4)
+      for (auto& axis : none_axes) {
+        int len = 0;
+        for (int da : decrease_axis) {
+          if (da < axis) {
+            len++;
           }
-          axis -= len;
         }
-        new_out = unsqueeze_ad_func(out, none_axes);
+        axis -= len;
       }
-      return ToPyObject(new_out);
+      new_out = unsqueeze_ad_func(out, none_axes);
     }
+    return ToPyObject(new_out);
   }
 
   // the index is a list
diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc
index 8d5bd524a1c14..44f0cfc9eb8b4 100644
--- a/paddle/fluid/pybind/imperative.cc
+++ b/paddle/fluid/pybind/imperative.cc
@@ -1067,44 +1067,61 @@ void BindImperative(py::module *m_ptr) {
              }
 
              bool set_to_1d = FLAGS_set_to_1d;
-             if (!none_axes.empty()) {
-               if (set_to_1d) {
-                 // NOTE(zoooo0820): When all axes are decreased, the output
-                 // will be 1-D with FLAGS_set_to_1d=True. In this case, one
-                 // `None` should be pop out, otherwise the output shape will be
-                 // not correct.
-                 if (static_cast<int>(decrease_axis.size()) ==
-                     tensor->dims().size()) {
+
+             if (set_to_1d) {
+               // NOTE(zoooo0820): When all axes are decreased, the output
+               // will be 1-D with FLAGS_set_to_1d=True. In this case, one
+               // `None` should be pop out, otherwise the output shape will be
+               // not correct.
+               if (static_cast<int>(decrease_axis.size()) ==
+                   tensor->dims().size()) {
+                 VLOG(0) << "Warning: In Tensor '__getitem__', if the number "
+                            "of scalar "
+                            "elements "
+                            "in the index is equal to the rank of the Tensor, "
+                            "the output "
+                            "should "
+                            "be 0-D. In order to be consistent with the "
+                            "behavior of previous "
+                            "versions, it will be processed to 1-D. But it is "
+                            "not correct and "
+                            "will be "
+                            "removed in release 2.6. "
+                            "If 1-D is still wanted, please modify the index "
+                            "element from "
+                            "scalar to slice "
+                            "(e.g. 'x[i]' => 'x[i:i+1]'). ";
+                 if (!none_axes.empty()) {
                    none_axes.pop_back();
                  }
                }
-               if (!none_axes.empty()) {
-                 // Deal with cases that decrease_axes is not empty
-                 // For example:
-                 // # x.shape: (2,3,4)
-                 // out = x[0, 0:2, None] # out.shape : (2, 1, 4)
-                 for (auto &axis : none_axes) {
-                   int len = 0;
-                   for (int da : decrease_axis) {
-                     if (da < axis) {
-                       len++;
-                     }
+             }
+             if (!none_axes.empty()) {
+               // Deal with cases that decrease_axes is not empty
+               // For example:
+               // # x.shape: (2,3,4)
+               // out = x[0, 0:2, None] # out.shape : (2, 1, 4)
+               for (auto &axis : none_axes) {
+                 int len = 0;
+                 for (int da : decrease_axis) {
+                   if (da < axis) {
+                     len++;
                    }
-                   axis -= len;
                  }
-
-                 imperative::NameVarBaseMap ins = {{"X", {out}}};
-                 framework::AttributeMap attrs = {{"axes", none_axes}};
-                 auto new_out = std::shared_ptr<imperative::VarBase>(
-                     new imperative::VarBase(tracer->GenerateUniqueName()));
-                 auto out_xshape = std::shared_ptr<imperative::VarBase>(
-                     new imperative::VarBase(tracer->GenerateUniqueName()));
-                 imperative::NameVarBaseMap outs = {{"Out", {new_out}},
-                                                    {"XShape", {out_xshape}}};
-                 tracer->TraceOp("unsqueeze2", ins, outs, std::move(attrs));
-
-                 return new_out;
+                 axis -= len;
                }
+
+               imperative::NameVarBaseMap ins = {{"X", {out}}};
+               framework::AttributeMap attrs = {{"axes", none_axes}};
+               auto new_out = std::shared_ptr<imperative::VarBase>(
+                   new imperative::VarBase(tracer->GenerateUniqueName()));
+               auto out_xshape = std::shared_ptr<imperative::VarBase>(
+                   new imperative::VarBase(tracer->GenerateUniqueName()));
+               imperative::NameVarBaseMap outs = {{"Out", {new_out}},
+                                                  {"XShape", {out_xshape}}};
+               tracer->TraceOp("unsqueeze2", ins, outs, std::move(attrs));
+
+               return new_out;
              }
 
              // the index is a list
diff --git a/paddle/phi/kernels/funcs/slice_utils.h b/paddle/phi/kernels/funcs/slice_utils.h
index 81a35b6774ae2..31a481883eae6 100644
--- a/paddle/phi/kernels/funcs/slice_utils.h
+++ b/paddle/phi/kernels/funcs/slice_utils.h
@@ -208,18 +208,6 @@ inline DDim GetDecreasedDims(const DDim slice_dims,
     if (FLAGS_set_to_1d && new_shape.size() == 0) {
       // NOTE(zoooo0820): Hack procssing to 1-D, when axes decrease to 0-D in
       // slice. This will remove in release 2.6.
-      VLOG(0)
-          << "Warning:: In Tensor '__getitem__', if the number of scalar "
-             "elements "
-             "in the index is equal to the rank of the Tensor, the output "
-             "should "
-             "be 0-D. In order to be consistent with the behavior of previous "
-             "versions, it will be processed to 1-D. But it is not correct and "
-             "will be "
-             "removed in release 2.6. "
-             "If 1-D is still wanted, please modify the index element from "
-             "scalar to slice "
-             "(e.g. 'x[i]' => 'x[i:i+1]'). ";
       new_shape.push_back(1);
     }
     decreased_dims = phi::make_ddim(new_shape);
diff --git a/python/paddle/fluid/variable_index.py b/python/paddle/fluid/variable_index.py
index 86ba9d5d58dcd..889da4ffaa82b 100644
--- a/python/paddle/fluid/variable_index.py
+++ b/python/paddle/fluid/variable_index.py
@@ -17,6 +17,7 @@
 from . import unique_name
 from . import core
 import paddle
+import warnings
 
 MAX_INTEGER = 2**31 - 1
 
@@ -579,6 +580,9 @@ def _getitem_impl_(var, item):
     # otherwise the output shape will be not correct.
     set_to_1d = paddle.get_flags('FLAGS_set_to_1d')['FLAGS_set_to_1d']
     if set_to_1d and len(decrease_axes) == len(var.shape):
+        warnings.warn(
+            "Warning: In Tensor '__getitem__', if the number of scalar elements in the index is equal to the rank of the Tensor, the output should be 0-D. In order to be consistent with the behavior of previous versions, it will be processed to 1-D. But it is not correct and will be removed in release 2.6. If 1-D is still wanted, please modify the index element from scalar to slice (e.g. 'x[i]' => 'x[i:i+1]')."
+        )
         none_axes = none_axes[1:]
 
     if len(none_axes) > 0:

From 10f9249b991805e571846d85b6f5218c8a381a1a Mon Sep 17 00:00:00 2001
From: Zhang Jun <ewalker@live.cn>
Date: Mon, 8 May 2023 14:49:31 +0800
Subject: [PATCH 210/405] [inference][trt]Unary operation support 0d (#53506)

* fix trt Unary operation do not support 0d when TRT < 8.6

* update unary ut

* add rsqrt to unary_list

* move rsqrt to act_list
---
 paddle/fluid/inference/tensorrt/op_teller.cc  | 19 +++++++++++++++++--
 .../inference/test_trt_convert_activation.py  | 10 +++++++++-
 test/ir/inference/test_trt_convert_unary.py   | 16 +++++++++++++++-
 3 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index beb289b420f2d..56d7fba985ce1 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -104,7 +104,16 @@ struct SimpleOpTypeSetTeller : public Teller {
         "atanh",      "ceil",        "celu",
         "erf",        "floor",       "round",
         "sign",       "silu",        "logical_not",
-        "reciprocal", "tanh_shrink", "logsigmoid"};
+        "reciprocal", "tanh_shrink", "logsigmoid",
+        "rsqrt"};
+    std::unordered_set<std::string> unary_list = {
+        "exp",        "log",  "sqrt",        "abs",        "sin",
+        "cos",        "tan",  "tanh",        "sinh",       "cosh",
+        "asin",       "acos", "atan",        "asinh",      "acosh",
+        "atanh",      "ceil", "celu",        "floor",      "round",
+        "sign",       "silu", "logical_not", "reciprocal", "tanh_shrink",
+        "logsigmoid", "erf",  "bitwise_not", "equal",      "not_equal",
+        "rsqrt"};
     if (act_op_list.find(op_type) != act_op_list.end()) {
       auto* block = desc.Block();
       if (block == nullptr) {
@@ -127,9 +136,15 @@ struct SimpleOpTypeSetTeller : public Teller {
         VLOG(3) << op_type << " op does not support tensorrt.";
         return false;
       }
+#endif
+#if !IS_TRT_VERSION_GE(8600)
+      if (x_shape.size() == 0 && unary_list.find(op_type) != unary_list.end()) {
+        VLOG(3) << op_type
+                << " op does not support 0 dim input when TensorRT < 8.6.";
+        return false;
+      }
 #endif
     }
-
     // In static shape in Paddle-TRT, we can't allow that one op has a
     // 1D intermediate tensor as input.
     if (!with_dynamic_shape) {
diff --git a/test/ir/inference/test_trt_convert_activation.py b/test/ir/inference/test_trt_convert_activation.py
index bba6beae14223..4b7052e682da7 100644
--- a/test/ir/inference/test_trt_convert_activation.py
+++ b/test/ir/inference/test_trt_convert_activation.py
@@ -49,7 +49,6 @@ def generate_input1(dims, batch, attrs: List[Dict[str, Any]]):
                 for op_type in [
                     "relu",
                     "sigmoid",
-                    "tanh",
                     "relu6",
                     "elu",
                     "selu",
@@ -146,6 +145,15 @@ def clear_dynamic_shape():
         def generate_trt_nodes_num(attrs, dynamic_shape):
             if not dynamic_shape and (self.dims == 1 or self.dims == 0):
                 return 0, 3
+            runtime_version = paddle_infer.get_trt_runtime_version()
+            if (
+                runtime_version[0] * 1000
+                + runtime_version[1] * 100
+                + runtime_version[2] * 10
+                < 8600
+                and self.dims == 0
+            ) and program_config.ops[0].type in ["celu", "logsigmoid"]:
+                return 0, 3
             return 1, 2
 
         attrs = [
diff --git a/test/ir/inference/test_trt_convert_unary.py b/test/ir/inference/test_trt_convert_unary.py
index 97e83e79714a5..322130c83f54d 100644
--- a/test/ir/inference/test_trt_convert_unary.py
+++ b/test/ir/inference/test_trt_convert_unary.py
@@ -35,6 +35,8 @@ def sample_program_configs(self):
         self.trt_param.workspace_size = 1073741824
 
         def generate_input1(dims, batch, attrs: List[Dict[str, Any]]):
+            if dims == 0:
+                return np.random.random([]).astype(np.float32)
             if dims == 2:
                 return np.random.random([3, 32]).astype(np.float32)
             elif dims == 3:
@@ -43,6 +45,8 @@ def generate_input1(dims, batch, attrs: List[Dict[str, Any]]):
                 return np.random.random([batch, 3, 32, 32]).astype(np.float32)
 
         def generate_int_input(dims, batch, attrs: List[Dict[str, Any]]):
+            if dims == 0:
+                return np.random.random([]).astype(np.int32)
             if dims == 2:
                 return np.random.random([3, 32]).astype(np.int32)
             elif dims == 3:
@@ -50,7 +54,7 @@ def generate_int_input(dims, batch, attrs: List[Dict[str, Any]]):
             else:
                 return np.random.random([batch, 3, 32, 32]).astype(np.int32)
 
-        for dims in [2, 3, 4]:
+        for dims in [0, 2, 3, 4]:
             for batch in [1, 4]:
                 for op_type in [
                     "exp",
@@ -60,6 +64,7 @@ def generate_int_input(dims, batch, attrs: List[Dict[str, Any]]):
                     "sin",
                     "cos",
                     "tan",
+                    "tanh",
                     "sinh",
                     "cosh",
                     "asin",
@@ -179,6 +184,15 @@ def generate_trt_nodes_num(attrs, dynamic_shape):
                 )
             ):
                 return 0, 3
+            runtime_version = paddle_infer.get_trt_runtime_version()
+            if (
+                runtime_version[0] * 1000
+                + runtime_version[1] * 100
+                + runtime_version[2] * 10
+                < 8600
+                and self.dims == 0
+            ):
+                return 0, 3
             return 1, 2
 
         attrs = [

From e988251404a96a8d36d628b49d6002d28efc795b Mon Sep 17 00:00:00 2001
From: Asthestarsfalll <72954905+Asthestarsfalll@users.noreply.github.com>
Date: Mon, 8 May 2023 14:53:06 +0800
Subject: [PATCH 211/405] docs: Fix code example vsplit in docstring (#53558)

---
 python/paddle/tensor/manipulation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py
index 5c4a029711b0f..d0427886be2a6 100644
--- a/python/paddle/tensor/manipulation.py
+++ b/python/paddle/tensor/manipulation.py
@@ -2098,7 +2098,7 @@ def vsplit(x, num_or_sections, name=None):
 
             # x is a Tensor of shape [8, 6, 7]
             x = paddle.rand([8, 6, 7])
-            out0, out1, out2 = paddle.vsplit(x, num_or_sections=2)
+            out0, out1 = paddle.vsplit(x, num_or_sections=2)
             print(out0.shape)  # [4, 6, 7]
             print(out1.shape)  # [4, 6, 7]
             out0, out1, out2 = paddle.vsplit(x, num_or_sections=[1, 3, 4])

From 186f5e0fed26290c85fdf874a414ed5ad2e29359 Mon Sep 17 00:00:00 2001
From: JYChen <zoooo0820@qq.com>
Date: Mon, 8 May 2023 14:58:11 +0800
Subject: [PATCH 212/405] hack 1-D tensor to Scalar (#53552)

---
 .../auto_parallel/operators/dist_slice.py        | 16 +++++++++++++++-
 python/paddle/fluid/variable_index.py            |  9 +++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/python/paddle/distributed/auto_parallel/operators/dist_slice.py b/python/paddle/distributed/auto_parallel/operators/dist_slice.py
index 69ccd8d7bc868..0110f54d481a0 100644
--- a/python/paddle/distributed/auto_parallel/operators/dist_slice.py
+++ b/python/paddle/distributed/auto_parallel/operators/dist_slice.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import paddle
+
 from ..utils import compute_compatible_dim_mapping, is_dim_shard
 from .common import (
     DistributedOperatorImpl,
@@ -70,7 +72,14 @@ def is_output_compatible(self, dist_op):
             if i not in decrease_axis:
                 ref_indices.append(i)
         if ref_indices == []:
-            assert len(out_dims_mapping) == 0
+            # NOTE(zoooo0820): When all axes are decreased, the output will be 1-D
+            # with FLAGS_set_to_1d=True.
+            if paddle.get_flags('FLAGS_set_to_1d')['FLAGS_set_to_1d']:
+                assert len(out_dims_mapping) == 1
+                if is_dim_shard(out_dims_mapping[0]):
+                    return False
+            else:
+                assert len(out_dims_mapping) == 0
         else:
             for i in range(len(out_dims_mapping)):
                 ref_index = ref_indices[i]
@@ -140,6 +149,11 @@ def update_dims_mapping(self, dist_op):
                 ref_indices.append(i)
 
         if ref_dims_mapping == []:
+            # NOTE(zoooo0820): When all axes are decreased, the output will be 1-D
+            # with FLAGS_set_to_1d=True.
+            if paddle.get_flags('FLAGS_set_to_1d')['FLAGS_set_to_1d']:
+                ref_dims_mapping = [-1]
+                assert ref_dims_mapping[0] == out_dims_mapping[0]
             assert len(ref_dims_mapping) == len(out_dims_mapping)
             changed = False
         else:
diff --git a/python/paddle/fluid/variable_index.py b/python/paddle/fluid/variable_index.py
index 889da4ffaa82b..fbd55ba83aa35 100644
--- a/python/paddle/fluid/variable_index.py
+++ b/python/paddle/fluid/variable_index.py
@@ -284,6 +284,15 @@ def is_integer_or_scalar_tensor(ele):
     if isinstance(ele, int):
         return True
     elif isinstance(ele, Variable):
+        # NOTE(zoooo0820): For compatibility, if FLAGS_set_to_1d is set to True,
+        # 1-D tensor is still treated as a scalar, which means basic indexing.
+        # This will be removed in future.
+        if paddle.get_flags('FLAGS_set_to_1d')['FLAGS_set_to_1d']:
+            if len(ele.shape) == 1 and ele.shape[0] == 1:
+                warnings.warn(
+                    "1-D Tensor will be treat as advanced indexing in future version. Currently, 1-D Tensor means a scalar, not vector, and please modify it to 0-D Tensor. If advanced indexing is needed, please use `export FLAGS_set_to_1d=False` to set the flag."
+                )
+                return True
         if len(ele.shape) == 0:
             return True
     return False

From 0a59825e6fc6284af325839115f25de1b7dde201 Mon Sep 17 00:00:00 2001
From: wz1qqx <55830058+wz1qqx@users.noreply.github.com>
Date: Mon, 8 May 2023 15:01:45 +0800
Subject: [PATCH 213/405] [XPU] Optimize fp16 xpu models  (#53523)

---
 .../framework/ir/xpu/conv2d_xpu_fuse_pass.cc  | 51 +++++++++++---
 paddle/phi/api/yaml/fused_ops.yaml            |  4 +-
 paddle/phi/backends/xpu/xpu2_op_list.cc       | 19 +++--
 paddle/phi/infermeta/fusion.cc                |  1 +
 paddle/phi/infermeta/fusion.h                 |  1 +
 paddle/phi/kernels/activation_kernel.cc       |  3 +-
 paddle/phi/kernels/batch_norm_kernel.cc       |  8 ++-
 .../kernels/fusion/xpu/conv2d_xpu_kernel.cc   |  7 +-
 paddle/phi/kernels/xpu/activation_kernel.cc   | 45 +++++++++---
 paddle/phi/kernels/xpu/batch_norm_kernel.cc   | 70 ++++++++++---------
 paddle/phi/kernels/xpu/reduce_mean_kernel.cc  |  4 +-
 11 files changed, 150 insertions(+), 63 deletions(-)

diff --git a/paddle/fluid/framework/ir/xpu/conv2d_xpu_fuse_pass.cc b/paddle/fluid/framework/ir/xpu/conv2d_xpu_fuse_pass.cc
index 263ea5a09ca76..c80fda30990ff 100644
--- a/paddle/fluid/framework/ir/xpu/conv2d_xpu_fuse_pass.cc
+++ b/paddle/fluid/framework/ir/xpu/conv2d_xpu_fuse_pass.cc
@@ -34,6 +34,25 @@ class Scope;
 }  // namespace framework
 }  // namespace paddle
 
+namespace {
+
+template <typename T1, typename T2>
+void ConvertTensorType(phi::DenseTensor* tensor) {
+  phi::DenseTensor tmp_tensor;
+  tmp_tensor.set_type(phi::CppTypeToDataType<T2>::Type());
+  tmp_tensor.Resize(tensor->dims());
+  auto* tmp_data = tmp_tensor.mutable_data<T2>(paddle::platform::CPUPlace());
+  auto* data = tensor->mutable_data<T1>(paddle::platform::CPUPlace());
+  for (int i = 0; i < tensor->numel(); i++) {
+    tmp_data[i] = static_cast<T2>(data[i]);
+  }
+  tensor->clear();
+  paddle::framework::TensorCopySync(
+      tmp_tensor, paddle::platform::CPUPlace(), tensor);
+}
+
+}  // namespace
+
 namespace paddle {
 namespace framework {
 namespace ir {
@@ -157,15 +176,23 @@ Conv2dXPUPattern::Conv2dXPUPattern(PDPattern* pattern,
   if (with_bn_) {
     ew_bias_add_out->assert_is_op_input("batch_norm", "X");
     bn_bias = pattern->NewNode(bn_bias_repr())
+                  ->AsInput()
+                  ->assert_is_persistable_var()
                   ->assert_is_op_input("batch_norm", "Bias")
                   ->assert_has_n_outputs(1);
     bn_mean = pattern->NewNode(bn_mean_repr())
+                  ->AsInput()
+                  ->assert_is_persistable_var()
                   ->assert_is_op_input("batch_norm", "Mean")
                   ->assert_has_n_outputs(1);
     bn_scale = pattern->NewNode(bn_scale_repr())
+                   ->AsInput()
+                   ->assert_is_persistable_var()
                    ->assert_is_op_input("batch_norm", "Scale")
                    ->assert_has_n_outputs(1);
     bn_var = pattern->NewNode(bn_var_repr())
+                 ->AsInput()
+                 ->assert_is_persistable_var()
                  ->assert_is_op_input("batch_norm", "Variance")
                  ->assert_has_n_outputs(1);
     bn = pattern->NewNode(bn_repr())->assert_is_op("batch_norm");
@@ -420,13 +447,17 @@ int Conv2dXPUFusePass::ApplyImpl(ir::Graph* graph,
     // recompute bias and weight for conv2d_xpu op
     auto* filter_t =
         scope->FindVar(conv_filter->Name())->GetMutable<phi::DenseTensor>();
+    // conv_filter fp16 --> fp32
+    auto tensor_type = filter_t->dtype();
+    if (tensor_type == phi::DataType::FLOAT16) {
+      ConvertTensorType<float16, float>(filter_t);
+    }
     auto filter_dims = filter_t->dims();
     bool has_bias = with_bn || with_conv_bias;
-    bool has_branch = with_branch_x || with_branch_y;
     // Create conv_fusion_bias (conv bias) variable
     Node* fusion_bias_node = nullptr;
     if (has_bias) {
-      if (ew_bias_add != nullptr) {
+      if (with_conv_bias) {
         auto* ew_bias_add_y_t = scope->FindVar(ew_bias_add_y->Name())
                                     ->GetMutable<phi::DenseTensor>();
         auto ew_bias_add_y_dims = ew_bias_add_y_t->dims();
@@ -439,7 +470,7 @@ int Conv2dXPUFusePass::ApplyImpl(ir::Graph* graph,
                               filter_dims[0]));
         PrepareBias(graph, scope, block, ew_bias_add_y, &fusion_bias_node);
       }
-      if (bn != nullptr) {
+      if (with_bn) {
         auto bn_bias_t =
             scope->Var(bn_bias->Name())->GetMutable<phi::DenseTensor>();
         PADDLE_ENFORCE_EQ(filter_dims[0],
@@ -469,7 +500,7 @@ int Conv2dXPUFusePass::ApplyImpl(ir::Graph* graph,
         auto filter_len = filter_t->numel();
         auto filter_stride = filter_len / mean_len;
         float epsilon = PADDLE_GET_CONST(float, bn->Op()->GetAttr("epsilon"));
-        if (fusion_bias_node == nullptr) {  // prev node is conv
+        if (!with_conv_bias) {  // prev node is conv
           PrepareBias(graph, scope, block, bn_bias, &fusion_bias_node);
         }
         auto fusion_bias_t = scope->Var(fusion_bias_node->Name())
@@ -477,10 +508,10 @@ int Conv2dXPUFusePass::ApplyImpl(ir::Graph* graph,
         float* fusion_bias_ptr =
             fusion_bias_t->mutable_data<float>(paddle::platform::CPUPlace());
         // recompute bias and weights
-        if (ew_bias_add == nullptr) {
+        if (!with_conv_bias) {  // prev node is conv
           for (int i = 0; i < mean_len; ++i) {
             bn_scale_ptr[i] = bn_scale_ptr[i] / sqrtf(bn_var_ptr[i] + epsilon);
-            fusion_bias_ptr[i] += (0.f - bn_mean_ptr[i]) * bn_scale_ptr[i];
+            fusion_bias_ptr[i] += (0.0f - bn_mean_ptr[i]) * bn_scale_ptr[i];
             for (int j = 0; j < filter_stride; j++) {
               filter_ptr[i * filter_stride + j] *= bn_scale_ptr[i];
             }
@@ -488,21 +519,25 @@ int Conv2dXPUFusePass::ApplyImpl(ir::Graph* graph,
         } else {
           for (int i = 0; i < mean_len; ++i) {
             bn_scale_ptr[i] = bn_scale_ptr[i] / sqrtf(bn_var_ptr[i] + epsilon);
-            bn_bias_ptr[i] +=
+            fusion_bias_ptr[i] =
+                bn_bias_ptr[i] +
                 (fusion_bias_ptr[i] - bn_mean_ptr[i]) * bn_scale_ptr[i];
             for (int j = 0; j < filter_stride; j++) {
               filter_ptr[i * filter_stride + j] *= bn_scale_ptr[i];
             }
           }
-          memcpy(fusion_bias_ptr, bn_bias_ptr, mean_len * sizeof(float));
         }
       }
     }
+    if (tensor_type == phi::DataType::FLOAT16) {
+      ConvertTensorType<float, float16>(filter_t);
+    }
     // filter max
     Node* filter_int16 = nullptr;
     Node* filter_max = nullptr;
     PrepareWeight<int16_t>(
         graph, scope, block, conv_filter, &filter_int16, &filter_max, false);
+    bool has_branch = with_branch_x || with_branch_y;
     // output && output max
     std::string conv2d_xpu_out_name;
     if (!act_type.empty()) {
diff --git a/paddle/phi/api/yaml/fused_ops.yaml b/paddle/phi/api/yaml/fused_ops.yaml
index 87400ecd61c9f..d5f9a9e0d481d 100644
--- a/paddle/phi/api/yaml/fused_ops.yaml
+++ b/paddle/phi/api/yaml/fused_ops.yaml
@@ -5,14 +5,14 @@
 # otherwise the operator only could be used in static mode.
 
 - op : conv2d_xpu
-  args : (Tensor x, Tensor x_max, Tensor filter, Tensor filter_max, Tensor bias, Tensor branch, int[] paddings, int[] dilations, int[] strides, str padding_algorithm, int groups, bool has_bias, bool has_branch, int act_type, float act_param)
+  args : (Tensor x, Tensor x_max, Tensor filter, Tensor filter_max, Tensor bias, Tensor branch, Tensor branch_max, int[] paddings, int[] dilations, int[] strides, str padding_algorithm, int groups, bool has_bias, bool has_branch, int act_type, float act_param)
   output : Tensor(out), Tensor(out_max)
   infer_meta :
     func : Conv2dXPUInferMeta
   kernel :
     func : conv2d_xpu
     data_type : x
-  optional : bias, branch, x_max
+  optional : bias, branch, branch_max ,x_max
 
 - op : embedding_with_eltwise_add_xpu
   args : (Tensor[] ids, Tensor[] tables, int64_t padding_idx)
diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc
index 07ad62e442d76..c05b51ac7bdd5 100644
--- a/paddle/phi/backends/xpu/xpu2_op_list.cc
+++ b/paddle/phi/backends/xpu/xpu2_op_list.cc
@@ -58,7 +58,8 @@ XPUOpMap& get_kl2_ops() {
       {"atan_grad",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"batch_norm_grad", XPUKernelSet({phi::DataType::FLOAT32})},
-      {"batch_norm", XPUKernelSet({phi::DataType::FLOAT32})},
+      {"batch_norm",
+       XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"bmm", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"bmm_grad",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
@@ -401,7 +402,8 @@ XPUOpMap& get_kl2_ops() {
       {"grid_sampler_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"grid_sampler", XPUKernelSet({phi::DataType::FLOAT32})},
       {"hard_sigmoid_grad", XPUKernelSet({phi::DataType::FLOAT32})},
-      {"hard_sigmoid", XPUKernelSet({phi::DataType::FLOAT32})},
+      {"hard_sigmoid",
+       XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"hard_swish_grad",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"hard_swish", XPUKernelSet({phi::DataType::FLOAT32})},
@@ -438,7 +440,8 @@ XPUOpMap& get_kl2_ops() {
       {"layer_norm",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"leaky_relu_grad", XPUKernelSet({phi::DataType::FLOAT32})},
-      {"leaky_relu", XPUKernelSet({phi::DataType::FLOAT32})},
+      {"leaky_relu",
+       XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"less_equal",
        XPUKernelSet({phi::DataType::INT64,
                      phi::DataType::INT32,
@@ -554,7 +557,8 @@ XPUOpMap& get_kl2_ops() {
       {"reduce_max_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"reduce_max", XPUKernelSet({phi::DataType::FLOAT32})},
       {"reduce_mean_grad", XPUKernelSet({phi::DataType::FLOAT32})},
-      {"reduce_mean", XPUKernelSet({phi::DataType::FLOAT32})},
+      {"reduce_mean",
+       XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"reduce_min_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"reduce_min", XPUKernelSet({phi::DataType::FLOAT32})},
       {"reduce_prod", XPUKernelSet({phi::DataType::FLOAT32})},
@@ -646,7 +650,8 @@ XPUOpMap& get_kl2_ops() {
                      phi::DataType::INT64,
                      phi::DataType::INT32,
                      phi::DataType::FLOAT16})},
-      {"sigmoid", XPUKernelSet({phi::DataType::FLOAT32})},
+      {"sigmoid",
+       XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"sigmoid_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"sign", XPUKernelSet({phi::DataType::FLOAT32})},
       {"slice_grad",
@@ -676,7 +681,7 @@ XPUOpMap& get_kl2_ops() {
        XPUKernelSet({phi::DataType::FLOAT32,
                      phi::DataType::FLOAT16,
                      phi::DataType::INT32})},
-      {"sqrt", XPUKernelSet({phi::DataType::FLOAT32})},
+      {"sqrt", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"sqrt_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"square_grad",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
@@ -733,7 +738,7 @@ XPUOpMap& get_kl2_ops() {
                      phi::DataType::INT16,
                      phi::DataType::INT32})},
       {"sum", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
-      {"swish", XPUKernelSet({phi::DataType::FLOAT32})},
+      {"swish", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"swish_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"tanh_grad",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
diff --git a/paddle/phi/infermeta/fusion.cc b/paddle/phi/infermeta/fusion.cc
index 262c5fb04e19f..f32ee528075db 100644
--- a/paddle/phi/infermeta/fusion.cc
+++ b/paddle/phi/infermeta/fusion.cc
@@ -41,6 +41,7 @@ void Conv2dXPUInferMeta(const MetaTensor& x,
                         const MetaTensor& filter_max,
                         const MetaTensor& bias,
                         const MetaTensor& branch,
+                        const MetaTensor& branch_max,
                         const std::vector<int>& paddings,
                         const std::vector<int>& dilations,
                         const std::vector<int>& strides,
diff --git a/paddle/phi/infermeta/fusion.h b/paddle/phi/infermeta/fusion.h
index 38f4bc8c6c5be..0fddf3995ffa0 100644
--- a/paddle/phi/infermeta/fusion.h
+++ b/paddle/phi/infermeta/fusion.h
@@ -28,6 +28,7 @@ void Conv2dXPUInferMeta(const MetaTensor& x,
                         const MetaTensor& filter_max,
                         const MetaTensor& bias,
                         const MetaTensor& branch,
+                        const MetaTensor& branch_max,
                         const std::vector<int>& paddings,
                         const std::vector<int>& dilations,
                         const std::vector<int>& strides,
diff --git a/paddle/phi/kernels/activation_kernel.cc b/paddle/phi/kernels/activation_kernel.cc
index ef6135d25c99d..ab367a672696f 100644
--- a/paddle/phi/kernels/activation_kernel.cc
+++ b/paddle/phi/kernels/activation_kernel.cc
@@ -63,7 +63,8 @@ PD_REGISTER_KERNEL(swish,
 
 #if defined PADDLE_WITH_XPU
 PD_REGISTER_KERNEL(relu6, XPU, ALL_LAYOUT, phi::Relu6Kernel, float) {}
-PD_REGISTER_KERNEL(swish, XPU, ALL_LAYOUT, phi::SwishKernel, float) {}
+PD_REGISTER_KERNEL(
+    swish, XPU, ALL_LAYOUT, phi::SwishKernel, float, phi::dtype::float16) {}
 #endif
 
 #ifdef PADDLE_WITH_MKLDNN
diff --git a/paddle/phi/kernels/batch_norm_kernel.cc b/paddle/phi/kernels/batch_norm_kernel.cc
index 570ba8dae06cf..bf04c99dab0a3 100644
--- a/paddle/phi/kernels/batch_norm_kernel.cc
+++ b/paddle/phi/kernels/batch_norm_kernel.cc
@@ -106,6 +106,10 @@ PD_REGISTER_KERNEL(batch_norm_infer,
                    phi::dtype::float16) {}
 #endif
 #ifdef PADDLE_WITH_XPU
-PD_REGISTER_KERNEL(
-    batch_norm_infer, XPU, ALL_LAYOUT, phi::BatchNormInferKernel, float) {}
+PD_REGISTER_KERNEL(batch_norm_infer,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::BatchNormInferKernel,
+                   float,
+                   phi::dtype::float16) {}
 #endif
diff --git a/paddle/phi/kernels/fusion/xpu/conv2d_xpu_kernel.cc b/paddle/phi/kernels/fusion/xpu/conv2d_xpu_kernel.cc
index 0f7d8902de328..f82d9fdd9fdea 100644
--- a/paddle/phi/kernels/fusion/xpu/conv2d_xpu_kernel.cc
+++ b/paddle/phi/kernels/fusion/xpu/conv2d_xpu_kernel.cc
@@ -27,6 +27,7 @@ void Conv2dXPUKernel(const Context& ctx,
                      const DenseTensor& filter_max,
                      const paddle::optional<DenseTensor>& bias,
                      const paddle::optional<DenseTensor>& branch,
+                     const paddle::optional<DenseTensor>& branch_max,
                      const std::vector<int>& paddings,
                      const std::vector<int>& dilations,
                      const std::vector<int>& strides,
@@ -69,10 +70,12 @@ void Conv2dXPUKernel(const Context& ctx,
       branch.get_ptr() == nullptr
           ? nullptr
           : reinterpret_cast<const XPUType*>(branch.get_ptr()->data<T>());
+  const float* branch_max_data = branch_max.get_ptr() == nullptr
+                                     ? nullptr
+                                     : branch_max.get_ptr()->data<float>();
   const float* bias_data =
       bias.get_ptr() == nullptr ? nullptr : bias.get_ptr()->data<float>();
   auto* out_data = reinterpret_cast<XPUType*>(ctx.template Alloc<T>(out));
-
   xpu::Activation_t act(static_cast<xpu::Activation_t::act_enum>(act_type));
   if (act_type == xpu::Activation_t::LEAKY_RELU) {
     act.leaky_alpha = act_param;
@@ -102,7 +105,7 @@ void Conv2dXPUKernel(const Context& ctx,
           /* const float* bias */ bias_data,
           /* const TY* branch */ branch_data,
           /* const baidu::xpu::api::Activation_t& act */ act,
-          /* const float* branch_maxptr */ nullptr,
+          /* const float* branch_maxptr */ branch_max_data,
           /* const float* scale */ nullptr);
   PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d_xpu");
 }
diff --git a/paddle/phi/kernels/xpu/activation_kernel.cc b/paddle/phi/kernels/xpu/activation_kernel.cc
index 490c56d13127f..41a1cceb11935 100644
--- a/paddle/phi/kernels/xpu/activation_kernel.cc
+++ b/paddle/phi/kernels/xpu/activation_kernel.cc
@@ -195,6 +195,13 @@ void PowKernel(const Context& dev_ctx,
                const DenseTensor& x,
                const Scalar& factor,
                DenseTensor* out) {
+  // using XPUType = typename XPUTypeTrait<T>::Type;
+  // // dev_ctx.template Alloc<T>(out);
+  // auto pow_factor = factor.to<T>();
+  // const auto* x_data = reinterpret_cast<const XPUType*>(x.data<T>());
+  // auto* y_data = reinterpret_cast<XPUType*>(dev_ctx.template Alloc<T>(out));
+  // // const T* x_data = x.data<T>();
+  // // T* y_data = out->data<T>();
   dev_ctx.template Alloc<T>(out);
   float pow_factor = factor.to<float>();
   const T* x_data = x.data<T>();
@@ -534,9 +541,28 @@ PD_REGISTER_KERNEL(
     relu, XPU, ALL_LAYOUT, phi::ReluKernel, float, phi::dtype::float16) {}
 PD_REGISTER_KERNEL(
     silu, XPU, ALL_LAYOUT, phi::SiluKernel, float, phi::dtype::float16) {}
-
-#define PD_REGISTER_ACTIVATION_KERNEL(name, func) \
-  PD_REGISTER_KERNEL(name, XPU, ALL_LAYOUT, phi::func, float) {}
+PD_REGISTER_KERNEL(
+    sigmoid, XPU, ALL_LAYOUT, phi::SigmoidKernel, float, phi::dtype::float16) {}
+PD_REGISTER_KERNEL(swish_raw,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::SwishRawKernel,
+                   float,
+                   phi::dtype::float16) {}
+PD_REGISTER_KERNEL(hard_sigmoid,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::HardSigmoidKernel,
+                   float,
+                   phi::dtype::float16) {}
+PD_REGISTER_KERNEL(leaky_relu,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::LeakyReluKernel,
+                   float,
+                   phi::dtype::float16) {}
+PD_REGISTER_KERNEL(
+    sqrt, XPU, ALL_LAYOUT, phi::SqrtKernel, float, phi::dtype::float16) {}
 
 PD_REGISTER_KERNEL(
     tanh, XPU, ALL_LAYOUT, phi::TanhKernel, float, phi::dtype::float16) {}
@@ -547,18 +573,21 @@ PD_REGISTER_KERNEL(
 PD_REGISTER_KERNEL(
     log, XPU, ALL_LAYOUT, phi::LogKernel, float, phi::dtype::float16) {}
 
+#define PD_REGISTER_ACTIVATION_KERNEL(name, func) \
+  PD_REGISTER_KERNEL(name, XPU, ALL_LAYOUT, phi::func, float) {}
+
 PD_REGISTER_ACTIVATION_KERNEL(exp, ExpKernel)  // no grad
 PD_REGISTER_ACTIVATION_KERNEL(floor, FloorKernel)
-PD_REGISTER_ACTIVATION_KERNEL(leaky_relu, LeakyReluKernel)
-PD_REGISTER_ACTIVATION_KERNEL(hard_sigmoid, HardSigmoidKernel)
+// PD_REGISTER_ACTIVATION_KERNEL(leaky_relu, LeakyReluKernel)
+// PD_REGISTER_ACTIVATION_KERNEL(hard_sigmoid, HardSigmoidKernel)
 PD_REGISTER_ACTIVATION_KERNEL(hardswish, HardSwishKernel)
 PD_REGISTER_ACTIVATION_KERNEL(mish, MishKernel)
 PD_REGISTER_ACTIVATION_KERNEL(pow, PowKernel)
 PD_REGISTER_ACTIVATION_KERNEL(reciprocal, ReciprocalKernel)
 PD_REGISTER_ACTIVATION_KERNEL(relu6_raw, Relu6RawKernel)
-PD_REGISTER_ACTIVATION_KERNEL(sigmoid, SigmoidKernel)
-PD_REGISTER_ACTIVATION_KERNEL(sqrt, SqrtKernel)
-PD_REGISTER_ACTIVATION_KERNEL(swish_raw, SwishRawKernel)
+// PD_REGISTER_ACTIVATION_KERNEL(sigmoid, SigmoidKernel)
+// PD_REGISTER_ACTIVATION_KERNEL(sqrt, SqrtKernel)
+// PD_REGISTER_ACTIVATION_KERNEL(swish_raw, SwishRawKernel)
 PD_REGISTER_ACTIVATION_KERNEL(softplus, SoftplusKernel)
 PD_REGISTER_ACTIVATION_KERNEL(sin, SinKernel)
 PD_REGISTER_ACTIVATION_KERNEL(cos, CosKernel)
diff --git a/paddle/phi/kernels/xpu/batch_norm_kernel.cc b/paddle/phi/kernels/xpu/batch_norm_kernel.cc
index b3f2843315c15..b95dda1fed13d 100644
--- a/paddle/phi/kernels/xpu/batch_norm_kernel.cc
+++ b/paddle/phi/kernels/xpu/batch_norm_kernel.cc
@@ -39,6 +39,7 @@ void BatchNormKernel(const Context& dev_ctx,
                      DenseTensor* saved_mean,
                      DenseTensor* saved_variance,
                      DenseTensor* reserve_space) {
+  using XPUType = typename XPUTypeTrait<T>::Type;
   bool test_mode = is_test && (!trainable_statistics);
   bool global_stats = test_mode || use_global_stats;
   const auto data_layout = phi::StringToDataLayout(data_layout_str);
@@ -68,12 +69,12 @@ void BatchNormKernel(const Context& dev_ctx,
 
   W = W * D;
 
-  const auto* x_data = x.data<T>();
+  const auto* x_data = reinterpret_cast<const XPUType*>(x.data<T>());
   const auto* scale_data = scale.data<float>();
   const auto* bias_data = bias.data<float>();
 
   // alloc memory
-  auto* y_data = dev_ctx.template Alloc<T>(y);
+  auto* y_data = reinterpret_cast<XPUType*>(dev_ctx.template Alloc<T>(y));
   dev_ctx.template Alloc<float>(mean_out);
   dev_ctx.template Alloc<float>(variance_out);
   dev_ctx.template Alloc<float>(saved_mean);
@@ -95,43 +96,48 @@ void BatchNormKernel(const Context& dev_ctx,
     auto* saved_mean_data = saved_mean->data<float>();
     auto* saved_variance_data = saved_variance->data<float>();
 
-    int r = xpu::batch_norm<T>(dev_ctx.x_context(),
-                               x_data,
-                               y_data,
-                               N,
-                               C,
-                               H,
-                               W,
-                               epsilon,
-                               momentum,
-                               scale_data,
-                               bias_data,
-                               saved_mean_data,
-                               saved_variance_data,
-                               mean_out_data,
-                               variance_out_data,
-                               is_nchw);
+    int r = xpu::batch_norm<XPUType>(dev_ctx.x_context(),
+                                     x_data,
+                                     y_data,
+                                     N,
+                                     C,
+                                     H,
+                                     W,
+                                     epsilon,
+                                     momentum,
+                                     scale_data,
+                                     bias_data,
+                                     saved_mean_data,
+                                     saved_variance_data,
+                                     mean_out_data,
+                                     variance_out_data,
+                                     is_nchw);
     PADDLE_ENFORCE_XDNN_SUCCESS(r, "batch_norm");
   } else {
     const auto* mean_data = mean.data<float>();
     const auto* variance_data = variance.data<float>();
-    int r = xpu::batch_norm_infer(dev_ctx.x_context(),
-                                  x_data,
-                                  y_data,
-                                  N,
-                                  C,
-                                  H,
-                                  W,
-                                  epsilon,
-                                  scale_data,
-                                  bias_data,
-                                  mean_data,
-                                  variance_data,
-                                  is_nchw);
+    int r = xpu::batch_norm_infer<XPUType>(dev_ctx.x_context(),
+                                           x_data,
+                                           y_data,
+                                           N,
+                                           C,
+                                           H,
+                                           W,
+                                           epsilon,
+                                           scale_data,
+                                           bias_data,
+                                           mean_data,
+                                           variance_data,
+                                           is_nchw);
     PADDLE_ENFORCE_XDNN_SUCCESS(r, "batch_norm_infer");
   }
 }
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(batch_norm, XPU, ALL_LAYOUT, phi::BatchNormKernel, float) {}
+PD_REGISTER_KERNEL(batch_norm,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::BatchNormKernel,
+                   float,
+                   phi::dtype::float16) {}
diff --git a/paddle/phi/kernels/xpu/reduce_mean_kernel.cc b/paddle/phi/kernels/xpu/reduce_mean_kernel.cc
index b9ba5178a5dd7..cb0bfb6218a88 100644
--- a/paddle/phi/kernels/xpu/reduce_mean_kernel.cc
+++ b/paddle/phi/kernels/xpu/reduce_mean_kernel.cc
@@ -50,4 +50,6 @@ void MeanRawKernel(const Context& dev_ctx,
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(mean_raw, XPU, ALL_LAYOUT, phi::MeanRawKernel, float) {}
+PD_REGISTER_KERNEL(
+    mean_raw, XPU, ALL_LAYOUT, phi::MeanRawKernel, float, phi::dtype::float16) {
+}

From 2aedd9db9684142e6338f8ae2d111da216f3f506 Mon Sep 17 00:00:00 2001
From: zhwesky2010 <1183042833@qq.com>
Date: Mon, 8 May 2023 15:12:59 +0800
Subject: [PATCH 214/405] [BUG] fix paddle.to_tensor/Tensor.item/Tensor.numpy
 BF16 bug (#53567)

---
 paddle/fluid/pybind/eager_method.cc           |  4 +-
 python/paddle/fluid/data_feeder.py            | 23 +++++++----
 .../fluid/dygraph/tensor_patch_methods.py     | 10 ++++-
 .../fluid/tests/unittests/test_var_base.py    | 26 +++++++++++-
 python/paddle/framework/framework.py          |  4 +-
 python/paddle/hapi/progressbar.py             |  2 -
 python/paddle/tensor/creation.py              | 41 +++++++++++--------
 7 files changed, 73 insertions(+), 37 deletions(-)

diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc
index be2771293d40d..2caf978db6f23 100644
--- a/paddle/fluid/pybind/eager_method.cc
+++ b/paddle/fluid/pybind/eager_method.cc
@@ -1079,13 +1079,11 @@ static PyObject* tensor__getitem_from_offset(TensorObject* self,
     T b = paddle::pybind::TensorGetElement<T>(tensor, offset);               \
     Py_intptr_t py_dims[paddle::framework::DDim::kMaxRank];                  \
     Py_intptr_t py_strides[paddle::framework::DDim::kMaxRank];               \
-    py_dims[0] = 1;                                                          \
-    py_strides[0] = 1;                                                       \
     auto& api = pybind11::detail::npy_api::get();                            \
     PyObject* array = api.PyArray_NewFromDescr_(                             \
         api.PyArray_Type_,                                                   \
         api.PyArray_DescrFromType_(numpy_dtype),                             \
-        1,                                                                   \
+        0,                                                                   \
         py_dims,                                                             \
         py_strides,                                                          \
         nullptr,                                                             \
diff --git a/python/paddle/fluid/data_feeder.py b/python/paddle/fluid/data_feeder.py
index 77abddce03d82..908fedf7c4b9c 100644
--- a/python/paddle/fluid/data_feeder.py
+++ b/python/paddle/fluid/data_feeder.py
@@ -46,10 +46,6 @@
 }
 
 
-def copy_bits_from_float_to_uint16(f):
-    return struct.unpack('<I', struct.pack('<f', f))[0] >> 16
-
-
 def convert_float_to_uint16(data, data_format="NCHW"):
     if data.size == 0:
         return data.view(np.uint16)
@@ -57,16 +53,25 @@ def convert_float_to_uint16(data, data_format="NCHW"):
     if data_format == "NHWC":
         data = np.transpose(data, [0, 3, 1, 2])
 
-    new_data = []
-    for x in np.nditer(data):
-        new_data.append(np.uint16(copy_bits_from_float_to_uint16(x)))
-    new_data = np.reshape(new_data, data.shape).view(np.uint16)
+    new_data = np.vectorize(
+        lambda x: struct.unpack('<I', struct.pack('<f', x))[0] >> 16,
+        otypes=[np.uint16],
+    )(data.flat)
+    new_data = np.reshape(new_data, data.shape)
 
     if data_format == "NHWC":
-        new_data = np.transpose(new_output, [0, 2, 3, 1])
+        new_data = np.transpose(new_data, [0, 2, 3, 1])
     return new_data
 
 
+def convert_uint16_to_float(data):
+    new_data = np.vectorize(
+        lambda x: struct.unpack('<f', struct.pack('<I', x << 16))[0],
+        otypes=[np.float32],
+    )(data.flat)
+    return np.reshape(new_data, data.shape)
+
+
 def convert_dtype(dtype):
     if isinstance(dtype, core.VarDesc.VarType):
         if dtype in _PADDLE_DTYPE_2_NUMPY_DTYPE:
diff --git a/python/paddle/fluid/dygraph/tensor_patch_methods.py b/python/paddle/fluid/dygraph/tensor_patch_methods.py
index 90d896f7c7dc1..80d2b5e973cfe 100644
--- a/python/paddle/fluid/dygraph/tensor_patch_methods.py
+++ b/python/paddle/fluid/dygraph/tensor_patch_methods.py
@@ -33,7 +33,10 @@
 )
 from .base import switch_to_static_graph
 from .math_op_patch import monkey_patch_math_tensor
-from paddle.fluid.data_feeder import convert_dtype, _PADDLE_DTYPE_2_NUMPY_DTYPE
+from paddle.fluid.data_feeder import (
+    convert_uint16_to_float,
+    _PADDLE_DTYPE_2_NUMPY_DTYPE,
+)
 import paddle.utils.deprecated as deprecated
 import paddle.profiler as profiler
 from paddle.profiler.utils import in_profiler_mode
@@ -614,7 +617,10 @@ def item(self, *args):
                 print(x.item(0, 2))         #3.3
 
         """
-        return self._getitem_from_offset(*args).item()
+        scalar = self._getitem_from_offset(*args)
+        if scalar.dtype == np.uint16:
+            return convert_uint16_to_float(scalar).item()
+        return scalar.item()
 
     @property
     def inplace_version(self):
diff --git a/python/paddle/fluid/tests/unittests/test_var_base.py b/python/paddle/fluid/tests/unittests/test_var_base.py
index 9d489c77374c2..5fc9914fa1e88 100644
--- a/python/paddle/fluid/tests/unittests/test_var_base.py
+++ b/python/paddle/fluid/tests/unittests/test_var_base.py
@@ -246,10 +246,12 @@ def check_with_place(place):
                 np.testing.assert_array_equal(x.numpy(), numpy_array)
                 self.assertEqual(x.type, core.VarDesc.VarType.LOD_TENSOR)
 
-                # test dtype bfloat16
+                # test dtype=bfloat16
                 x = paddle.to_tensor(-1e6, dtype=paddle.bfloat16)
                 self.assertEqual(x.dtype, core.VarDesc.VarType.BF16)
                 self.assertTrue(x == -999424.0)
+                self.assertTrue(x.item() == -999424.0)
+                self.assertTrue(isinstance(x.item(), float))
 
                 x = paddle.to_tensor([-1e6, -1e6, -1e6], dtype='bfloat16')
                 self.assertEqual(x.dtype, core.VarDesc.VarType.BF16)
@@ -266,6 +268,28 @@ def check_with_place(place):
                 y.backward()
                 self.assertTrue(x.grad == -999424.0 * 2)
 
+                # test default_type=bfloat16
+                paddle.set_default_dtype('bfloat16')
+                x = paddle.to_tensor(-1e6)
+                self.assertEqual(x.dtype, core.VarDesc.VarType.BF16)
+                self.assertTrue(x == -999424.0)
+                self.assertTrue(x.item() == -999424.0)
+                self.assertTrue(isinstance(x.item(), float))
+
+                x = paddle.to_tensor([-1e6, -1e6, -1e6])
+                self.assertEqual(x.dtype, core.VarDesc.VarType.BF16)
+                self.assertTrue(x[0] == -999424.0)
+                self.assertTrue(x[1] == -999424.0)
+                self.assertTrue(x[2] == -999424.0)
+
+                x = paddle.to_tensor(-1e6, stop_gradient=False)
+                self.assertEqual(x.dtype, core.VarDesc.VarType.BF16)
+                self.assertTrue(x == -999424.0)
+                y = x * x
+                y.backward()
+                self.assertTrue(x.grad == -999424.0 * 2)
+                paddle.set_default_dtype('float32')
+
                 with self.assertRaises(ValueError):
                     paddle.randn([3, 2, 2]).item()
                 with self.assertRaises(ValueError):
diff --git a/python/paddle/framework/framework.py b/python/paddle/framework/framework.py
index 2e9df1a8ee353..563c4b4cd3b20 100644
--- a/python/paddle/framework/framework.py
+++ b/python/paddle/framework/framework.py
@@ -40,7 +40,7 @@ def set_default_dtype(d):
 
     """
     if isinstance(d, type):
-        # This branch is for NumPy scalar types
+        # This branch is for np.dtype
         if d in [np.float16, np.float32, np.float64]:
             d = d.__name__
         else:
@@ -49,7 +49,7 @@ def set_default_dtype(d):
                 ", but received %s" % d.__name__
             )
     else:
-        # This branch is for np.dtype and str
+        # This branch is for str
         if d in ['float16', 'float32', 'float64', 'bfloat16']:
             # NOTE(SigureMo): Since the np.dtype object is not an instance of
             # type, so it will not be handled by the previous branch. We need
diff --git a/python/paddle/hapi/progressbar.py b/python/paddle/hapi/progressbar.py
index 08d6cf2b12181..c5b25a58c12e1 100644
--- a/python/paddle/hapi/progressbar.py
+++ b/python/paddle/hapi/progressbar.py
@@ -83,8 +83,6 @@ def convert_uint16_to_float(in_list):
             if k == "loss":
                 if isinstance(val, list):
                     scalar_val = val[0]
-                elif isinstance(val, np.ndarray):
-                    scalar_val = val.item()
                 else:
                     scalar_val = val
                 if isinstance(scalar_val, np.uint16):
diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py
index db1006b0b1af0..0dca909bd5833 100644
--- a/python/paddle/tensor/creation.py
+++ b/python/paddle/tensor/creation.py
@@ -542,18 +542,28 @@ def logspace(start, stop, num, base=10.0, dtype=None, name=None):
 
 
 def _to_tensor_non_static(data, dtype=None, place=None, stop_gradient=True):
+    def _handle_tensor_dtype(tensor, dtype):
+        if dtype:
+            if convert_dtype(dtype) != convert_dtype(tensor.dtype):
+                return tensor.astype(convert_dtype(dtype))
+        return tensor
+
+    def _handle_np_dtype(ndarray, dtype):
+        if dtype:
+            if convert_dtype(dtype) != convert_dtype(ndarray.dtype):
+                # should not ndarray.astype('uint16') directly, data bits is wrong
+                if convert_dtype(dtype) in ['uint16']:
+                    return convert_float_to_uint16(ndarray.astype('float32'))
+                else:
+                    return ndarray.astype(convert_dtype(dtype))
+
+        return ndarray
 
     if isinstance(data, np.number):  # Special case for numpy scalars
         data = np.array(data)
 
     if not isinstance(data, np.ndarray):
 
-        def _handle_dtype(data, dtype):
-            if dtype:
-                if convert_dtype(dtype) != convert_dtype(data.dtype):
-                    return data.astype(convert_dtype(dtype))
-            return data
-
         if np.isscalar(data) and not isinstance(data, str):
             data = np.array(data)
         elif isinstance(data, (list, tuple)):
@@ -565,12 +575,12 @@ def _handle_dtype(data, dtype):
                 )
         elif isinstance(data, paddle.Tensor) and not in_dygraph_mode():
             data = data._copy_to(place, False)
-            data = _handle_dtype(data, dtype)
+            data = _handle_tensor_dtype(data, dtype)
             data.stop_gradient = stop_gradient
             return data
         elif isinstance(data, core.eager.Tensor) and in_dygraph_mode():
             data = data._copy_to(place, False)
-            data = _handle_dtype(data, dtype)
+            data = _handle_tensor_dtype(data, dtype)
             data.stop_gradient = stop_gradient
             return data
         elif isinstance(data, (core.LoDTensor, core.Tensor)):
@@ -583,7 +593,7 @@ def _handle_dtype(data, dtype):
                 data = paddle.Tensor(data)
             if not data.place._equals(place):
                 data = data._copy_to(place, False)
-            data = _handle_dtype(data, dtype)
+            data = _handle_tensor_dtype(data, dtype)
             data.stop_gradient = stop_gradient
             return data
         else:
@@ -607,18 +617,13 @@ def _handle_dtype(data, dtype):
                         if default_type in ['float16', 'float32']
                         else 'complex128'
                     )
-                data = data.astype(default_type)
+                data = _handle_np_dtype(data, default_type)
             # Windows default type is 'int32', while Linux/Mac is 'int64'. Unify they.
             if data.dtype in ['int32']:
-                default_type = "int64"
-                data = data.astype(default_type)
+                data = data.astype("int64")
 
-    if dtype and convert_dtype(dtype) != data.dtype:
-        if convert_dtype(dtype) in ['uint16']:
-            # should not ndarray.astype('uint16') directly, data bits is wrong
-            data = convert_float_to_uint16(data.astype('float32'))
-        else:
-            data = data.astype(convert_dtype(dtype))
+    if dtype:
+        data = _handle_np_dtype(data, dtype)
 
     if _in_eager_without_dygraph_check() and isinstance(data, np.ndarray):
         return core.eager.Tensor(

From 26c3077a3e0905e4b1cf3c9b06b4667dcc9cfdd4 Mon Sep 17 00:00:00 2001
From: Roc <30228238+sljlp@users.noreply.github.com>
Date: Mon, 8 May 2023 15:14:30 +0800
Subject: [PATCH 215/405] Fix timeout v2 (#53514)

---
 .../tests/unittests/collective/CMakeLists.txt | 16 ++++
 .../unittests/collective/fleet/CMakeLists.txt | 76 +++++++++++++++++++
 tools/gen_ut_cmakelists.py                    | 41 +++++++---
 3 files changed, 124 insertions(+), 9 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt b/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt
index 1aec76f705236..ae17fbb193829 100644
--- a/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt
@@ -44,6 +44,8 @@ if((WITH_ROCM OR WITH_GPU) AND (LINUX))
     test_collective_split_embedding
     START_BASH
     ../dist_test.sh
+    TIMEOUT
+    "300"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -85,6 +87,8 @@ if((WITH_GPU OR WITH_ROCM) AND (LINUX))
     test_collective_alltoall_single
     START_BASH
     ../dist_test.sh
+    TIMEOUT
+    "350"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -112,6 +116,8 @@ if((WITH_GPU OR WITH_ROCM) AND (LINUX))
     test_collective_batch_isend_irecv
     START_BASH
     ../dist_test.sh
+    TIMEOUT
+    "350"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -176,6 +182,8 @@ if((WITH_GPU OR WITH_ROCM) AND (LINUX))
     test_collective_process_group
     START_BASH
     ../dist_test.sh
+    TIMEOUT
+    "350"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -202,6 +210,8 @@ if((WITH_GPU OR WITH_ROCM) AND (LINUX))
     test_collective_reduce_scatter
     START_BASH
     ../dist_test.sh
+    TIMEOUT
+    "350"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -391,6 +401,8 @@ if((WITH_ROCM OR WITH_GPU) AND (LINUX))
     test_world_size_and_rank
     START_BASH
     test_world_size_and_rank.sh
+    TIMEOUT
+    "120"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -414,6 +426,8 @@ if((WITH_ROCM OR WITH_GPU) AND (LINUX))
     test_strategy_group
     START_BASH
     test_strategy_group.sh
+    TIMEOUT
+    "120"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -425,6 +439,8 @@ if((WITH_ROCM OR WITH_GPU) AND (LINUX))
     test_orthogonal_strategy
     START_BASH
     test_orthogonal_strategy.sh
+    TIMEOUT
+    "120"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt b/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt
index dad5f1d4b5b6b..17408305e5992 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt
@@ -37,6 +37,8 @@ if(WITH_NCCL)
       test_parallel_margin_cross_entropy
       START_BASH
       ../../dist_test.sh
+      TIMEOUT
+      "120"
       LABELS
       "RUN_TYPE=DIST"
       ENVS
@@ -95,6 +97,8 @@ if(WITH_NCCL)
       test_parallel_dygraph_mp_layers
       START_BASH
       ../../dist_test.sh
+      TIMEOUT
+      "120"
       LABELS
       "RUN_TYPE=DIST"
       ENVS
@@ -120,6 +124,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_dygraph_sharding_stage3_for_eager
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "350"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -146,6 +152,8 @@ if((WITH_GPU) AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_pipeline_parallel
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "500"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -159,6 +167,8 @@ if((WITH_GPU) AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_pipeline_parallel_with_virtual_stage
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "500"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -173,6 +183,8 @@ if((WITH_GPU) AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_pp_adaptor
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "500"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -193,6 +205,8 @@ if(WITH_NCCL)
       test_parallel_class_center_sample
       START_BASH
       ../../dist_test.sh
+      TIMEOUT
+      "120"
       LABELS
       "RUN_TYPE=DIST"
       ENVS
@@ -207,6 +221,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_pipeline
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "120"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -226,6 +242,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_static_model_parallel
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "240"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -239,6 +257,8 @@ if(WITH_NCCL)
       test_parallel_dygraph_no_sync
       START_BASH
       ../../dist_test.sh
+      TIMEOUT
+      "300"
       LABELS
       "RUN_TYPE=DIST"
       ENVS
@@ -252,6 +272,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_dygraph_sharding_stage2
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -264,6 +286,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_control_flow
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "350"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -288,6 +312,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_hybrid_parallel_inference_helper
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "120"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -365,6 +391,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_sharding_parallel
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "120"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -378,6 +406,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_tensor_parallel
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -391,6 +421,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_dygraph_group_sharded_api_for_eager
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "120"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -417,6 +449,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_unused_variables
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "350"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -442,6 +476,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_no_sync_gradient_check
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "60"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -472,6 +508,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_qat
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "120"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -486,6 +524,8 @@ if(WITH_NCCL)
         test_parallel_dygraph_sparse_embedding
         START_BASH
         ../../dist_test.sh
+        TIMEOUT
+        "200"
         LABELS
         "RUN_TYPE=DIST"
         ENVS
@@ -501,6 +541,8 @@ if((WITH_ROCM) AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_sparse_embedding
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -521,6 +563,8 @@ if(WITH_NCCL)
         test_parallel_dygraph_sparse_embedding_over_height
         START_BASH
         ../../dist_test.sh
+        TIMEOUT
+        "150"
         LABELS
         "RUN_TYPE=DIST"
         ENVS
@@ -536,6 +580,8 @@ if((WITH_ROCM) AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_sparse_embedding_over_height
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "350"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -554,6 +600,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_auto_parallel_parallelizer
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "120"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -587,6 +635,8 @@ if((WITH_GPU OR WITH_XPU) AND (LINUX))
     test_c_comm_init_op
     START_BASH
     test_c_comm_init_op.sh
+    TIMEOUT
+    "120"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -598,6 +648,8 @@ if((WITH_GPU) AND (LINUX))
     test_fused_attention_pass_with_mp
     START_BASH
     test_fused_attention_pass_with_mp.sh
+    TIMEOUT
+    "120"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -610,6 +662,8 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
     test_ir_pass_pipeline
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "120"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -622,6 +676,8 @@ if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_mnist
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -634,6 +690,8 @@ if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT)
     test_parallel_dygraph_se_resnext
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=DIST"
     ENVS
@@ -704,6 +762,8 @@ if(LOCAL_ALL_ARCH AND (LINUX))
     test_auto_checkpoint
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=EXCLUSIVE:NIGHTLY"
     ENVS
@@ -716,6 +776,8 @@ if(LOCAL_ALL_ARCH AND (LINUX))
     test_auto_checkpoint1
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=EXCLUSIVE:NIGHTLY"
     ENVS
@@ -728,6 +790,8 @@ if(LOCAL_ALL_ARCH AND (LINUX))
     test_auto_checkpoint2
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=EXCLUSIVE:NIGHTLY"
     ENVS
@@ -740,6 +804,8 @@ if(LOCAL_ALL_ARCH AND (LINUX))
     test_auto_checkpoint3
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=EXCLUSIVE:NIGHTLY"
     ENVS
@@ -752,6 +818,8 @@ if(LOCAL_ALL_ARCH AND (LINUX))
     test_auto_checkpoint_multiple
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=EXCLUSIVE:NIGHTLY"
     ENVS
@@ -764,6 +832,8 @@ if(LOCAL_ALL_ARCH AND (LINUX))
     test_auto_checkpoint_dist_basic
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=EXCLUSIVE:NIGHTLY"
     ENVS
@@ -776,6 +846,8 @@ if(LOCAL_ALL_ARCH AND (LINUX))
     test_hdfs1
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=EXCLUSIVE:NIGHTLY"
     ENVS
@@ -788,6 +860,8 @@ if(LOCAL_ALL_ARCH AND (LINUX))
     test_hdfs2
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=EXCLUSIVE:NIGHTLY"
     ENVS
@@ -800,6 +874,8 @@ if(LOCAL_ALL_ARCH AND (LINUX))
     test_hdfs3
     START_BASH
     ../../dist_test.sh
+    TIMEOUT
+    "200"
     LABELS
     "RUN_TYPE=EXCLUSIVE:NIGHTLY"
     ENVS
diff --git a/tools/gen_ut_cmakelists.py b/tools/gen_ut_cmakelists.py
index 26cabcbaced84..7ceb99db637b5 100644
--- a/tools/gen_ut_cmakelists.py
+++ b/tools/gen_ut_cmakelists.py
@@ -97,6 +97,8 @@ def _proccess_archs(arch):
     arch = arch.upper().strip()
     if len(arch) > 0:
         for a in arch.split(";"):
+            if '' == a:
+                continue
             assert a in [
                 "GPU",
                 "ROCM",
@@ -372,11 +374,12 @@ def parse_assigned_dist_ut_ports(self, current_work_dir, depth=0):
 
 
 class CMakeGenerator:
-    def __init__(self, current_dirs, ignore_dirs):
+    def __init__(self, current_dirs, only_check, ignore_dirs):
         self.processed_dirs = set()
         self.port_manager = DistUTPortManager(ignore_dirs)
         self.current_dirs = _norm_dirs(current_dirs)
         self.modified_or_created_files = []
+        self._only_check = only_check
 
     def prepare_dist_ut_port(self):
         for c in self._find_root_dirs():
@@ -457,6 +460,10 @@ def _parse_line(self, line, curdir):
         for c in conditions:
             cmd += f"if ({c})\n"
 
+        time_out_str = (
+            f' TIMEOUT "{timeout}"' if len(timeout.strip()) > 0 else ''
+        )
+
         if launcher[-3:] == ".sh":
             run_type = _process_run_type(run_type)
             dist_ut_port = self.port_manager.process_dist_port_num(num_port)
@@ -466,6 +473,7 @@ def _parse_line(self, line, curdir):
         {name}
         START_BASH
         {launcher}
+        {time_out_str}
         LABELS
         "RUN_TYPE={run_type}"
         ENVS
@@ -492,9 +500,6 @@ def _parse_line(self, line, curdir):
             run_type_str = (
                 "" if len(run_type) == 0 else f' LABELS "RUN_TYPE={run_type}"'
             )
-        time_out_str = (
-            f' TIMEOUT "{timeout}"' if len(timeout.strip()) > 0 else ''
-        )
         run_serial_str = (
             f' RUN_SERIAL {run_serial}' if len(run_serial) > 0 else ''
         )
@@ -562,8 +567,13 @@ def _gen_cmakelists(self, current_work_dir, depth=0):
 
         # check whether the generated file are thge same with the existing file, ignoring the blank chars
         # if the are same, skip the weiting process
-        with open(f"{current_work_dir}/CMakeLists.txt", "r") as old_cmake_file:
-            char_seq = old_cmake_file.read().split()
+        if os.path.isfile(f"{current_work_dir}/CMakeLists.txt"):
+            with open(
+                f"{current_work_dir}/CMakeLists.txt", "r"
+            ) as old_cmake_file:
+                char_seq = old_cmake_file.read().split()
+        else:
+            char_seq = []
         char_seq = "".join(char_seq)
 
         if char_seq != "".join(cmds.split()):
@@ -574,8 +584,11 @@ def _gen_cmakelists(self, current_work_dir, depth=0):
             self.modified_or_created_files.append(
                 f"{current_work_dir}/CMakeLists.txt"
             )
-            with open(f"{current_work_dir}/CMakeLists.txt", "w") as cmake_file:
-                print(cmds, end="", file=cmake_file)
+            if not self._only_check:
+                with open(
+                    f"{current_work_dir}/CMakeLists.txt", "w"
+                ) as cmake_file:
+                    print(cmds, end="", file=cmake_file)
 
 
 if __name__ == "__main__":
@@ -607,6 +620,14 @@ def _gen_cmakelists(self, current_work_dir, depth=0):
         nargs='*',
         help="To keep dist ports the same with old version cmake, old cmakelists.txt files are needed to parse dist_ports. If a directories are newly created and there is no cmakelists.txt file, the directory path must be specified by this option. The dirs are not recursive.",
     )
+    parser.add_argument(
+        "--only-check-changed",
+        '-o',
+        type=lambda x: x.lower() not in ["false", "0", "off"],
+        required=False,
+        default=False,
+        help="Only check wheather the CMake files should be rewriten, do not write it enven if it should be write",
+    )
     args = parser.parse_args()
 
     assert not (
@@ -624,7 +645,9 @@ def _gen_cmakelists(self, current_work_dir, depth=0):
     if len(args.dirpaths) >= 1:
         current_work_dirs = current_work_dirs + list(args.dirpaths)
 
-    cmake_generator = CMakeGenerator(current_work_dirs, args.ignore_cmake_dirs)
+    cmake_generator = CMakeGenerator(
+        current_work_dirs, args.only_check_changed, args.ignore_cmake_dirs
+    )
     cmake_generator.prepare_dist_ut_port()
     created = cmake_generator.parse_csvs()
 

From 7dcf5e53dfb82879c4c74b29ff3923a0b562a8c2 Mon Sep 17 00:00:00 2001
From: niuliling123 <51102941+niuliling123@users.noreply.github.com>
Date: Mon, 8 May 2023 15:38:54 +0800
Subject: [PATCH 216/405] Fix core dumped in training when check_nan_inf=1

---
 paddle/fluid/eager/nan_inf_utils.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/paddle/fluid/eager/nan_inf_utils.cc b/paddle/fluid/eager/nan_inf_utils.cc
index c183c64fa7a1d..a66bc211d513c 100644
--- a/paddle/fluid/eager/nan_inf_utils.cc
+++ b/paddle/fluid/eager/nan_inf_utils.cc
@@ -115,7 +115,9 @@ void CheckTensorHasNanOrInf(const std::string& api_name, const Tensor& tensor) {
 
 void CheckTensorHasNanOrInf(const std::string& api_name,
                             const paddle::optional<Tensor>& tensor) {
-  CheckTensorHasNanOrInf(api_name, tensor.get());
+  if (tensor) {
+    CheckTensorHasNanOrInf(api_name, *tensor);
+  }
 }
 
 void CheckTensorHasNanOrInf(const std::string& api_name,
@@ -169,7 +171,7 @@ void CheckTensorHasNanOrInf(
     const std::string& api_name,
     const paddle::optional<std::vector<Tensor>>& tensors) {
   if (tensors) {
-    CheckTensorHasNanOrInf(api_name, tensors.get());
+    CheckTensorHasNanOrInf(api_name, *tensors);
   }
 }
 

From 6d396ace08f03a11911ac19b0d167b80b7a3053d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?=
 <83450930+Liyulingyue@users.noreply.github.com>
Date: Mon, 8 May 2023 15:58:11 +0800
Subject: [PATCH 217/405] rm npu (#53566)

---
 paddle/fluid/framework/dlpack_tensor.cc       |  5 --
 .../framework/ir/auto_mixed_precision_pass.cc |  2 -
 paddle/fluid/framework/op_registry.h          |  9 ---
 paddle/fluid/framework/parallel_executor.cc   |  2 -
 .../fluid/inference/api/analysis_predictor.cc |  2 -
 paddle/fluid/inference/api/api_impl.cc        |  2 -
 .../inference/api/details/zero_copy_tensor.cc |  3 -
 paddle/fluid/inference/api/paddle_tensor.h    |  2 +-
 .../collective/c_sync_comm_stream_op.cc       |  3 -
 paddle/fluid/operators/fill_constant_op.cc    |  6 +-
 paddle/fluid/operators/memcpy_d2h_op.cc       | 15 +++--
 paddle/fluid/operators/memcpy_h2d_op.cc       |  7 +--
 paddle/fluid/operators/memcpy_op.cc           | 11 ++--
 paddle/fluid/platform/device_context.h        |  1 -
 paddle/fluid/platform/device_event.h          |  1 -
 paddle/fluid/platform/place.cc                |  2 -
 paddle/fluid/platform/place.h                 |  6 --
 paddle/fluid/pybind/eager.cc                  |  2 +-
 paddle/fluid/pybind/eager_math_op_patch.cc    |  2 +-
 paddle/fluid/pybind/eager_utils.cc            |  6 +-
 paddle/fluid/pybind/imperative.cc             | 61 +------------------
 paddle/fluid/pybind/inference_api.cc          |  1 -
 paddle/fluid/pybind/place.cc                  | 35 -----------
 paddle/fluid/pybind/tensor.cc                 | 23 +------
 paddle/phi/api/include/tensor.h               |  2 +-
 paddle/phi/api/profiler/device_tracer.cc      |  2 -
 paddle/phi/common/backend.h                   |  7 ---
 paddle/phi/common/place.cc                    |  4 --
 paddle/phi/common/place.h                     | 11 ----
 paddle/phi/core/compat/convert_utils.cc       |  2 -
 paddle/phi/kernels/funcs/math_function.cc     |  7 ---
 python/paddle/amp/auto_cast.py                |  2 +-
 python/paddle/framework/__init__.py           |  1 -
 python/paddle/tensor/creation.py              |  2 +-
 tools/timeline.py                             |  9 ---
 35 files changed, 26 insertions(+), 232 deletions(-)

diff --git a/paddle/fluid/framework/dlpack_tensor.cc b/paddle/fluid/framework/dlpack_tensor.cc
index 37a129297f07e..f80f2359cf701 100644
--- a/paddle/fluid/framework/dlpack_tensor.cc
+++ b/paddle/fluid/framework/dlpack_tensor.cc
@@ -89,11 +89,6 @@ struct DLDeviceVisitor
         platform::errors::Unimplemented("platform::XPUPlace is not supported"));
   }
 
-  inline ::DLDevice operator()(const platform::NPUPlace &place) const {
-    PADDLE_THROW(
-        platform::errors::Unimplemented("platform::NPUPlace is not supported"));
-  }
-
   inline ::DLDevice operator()(const platform::NPUPinnedPlace &place) const {
     PADDLE_THROW(platform::errors::Unimplemented(
         "platform::NPUPinnedPlace is not supported"));
diff --git a/paddle/fluid/framework/ir/auto_mixed_precision_pass.cc b/paddle/fluid/framework/ir/auto_mixed_precision_pass.cc
index fa570394d80f3..01eb5dfbd9f02 100644
--- a/paddle/fluid/framework/ir/auto_mixed_precision_pass.cc
+++ b/paddle/fluid/framework/ir/auto_mixed_precision_pass.cc
@@ -55,8 +55,6 @@ static phi::Backend ConvertPlaceToBackend(const phi::Place& place) {
       return phi::Backend::GPU;
     case phi::AllocationType::XPU:
       return phi::Backend::XPU;
-    case phi::AllocationType::NPU:
-      return phi::Backend::NPU;
     default:
       PADDLE_THROW(platform::errors::InvalidArgument(
           "Cannot convert place(%d).", static_cast<int>(place.GetType())));
diff --git a/paddle/fluid/framework/op_registry.h b/paddle/fluid/framework/op_registry.h
index 941e8ea050e08..1d2a372684339 100644
--- a/paddle/fluid/framework/op_registry.h
+++ b/paddle/fluid/framework/op_registry.h
@@ -374,9 +374,6 @@ struct OpKernelRegistrarFunctorEx<PlaceType,
 #define REGISTER_OP_XPU_KERNEL(op_type, ...) \
   REGISTER_OP_KERNEL(op_type, XPU, ::paddle::platform::XPUPlace, __VA_ARGS__)
 
-#define REGISTER_OP_NPU_KERNEL(op_type, ...) \
-  REGISTER_OP_KERNEL(op_type, NPU, ::paddle::platform::NPUPlace, __VA_ARGS__)
-
 #define REGISTER_OP_KERNEL_EX(op_type, library_type, place_class,  \
                               customized_name,                     \
                               customized_type_value,               \
@@ -413,12 +410,6 @@ struct OpKernelRegistrarFunctorEx<PlaceType,
       ::paddle::framework::OpKernelType::kDefaultCustomizedTypeValue, \
       __VA_ARGS__)
 
-#define REGISTER_OP_NPU_KERNEL_FUNCTOR(op_type, ...)                  \
-  REGISTER_OP_KERNEL_EX(                                              \
-      op_type, NPU, ::paddle::platform::NPUPlace, DEFAULT_TYPE,       \
-      ::paddle::framework::OpKernelType::kDefaultCustomizedTypeValue, \
-      __VA_ARGS__)
-
 #define REGISTER_OP_IPU_KERNEL_FUNCTOR(op_type, ...)                  \
   REGISTER_OP_KERNEL_EX(                                              \
       op_type, IPU, ::paddle::platform::IPUPlace, DEFAULT_TYPE,       \
diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index 3820667116bc6..8937d58f8465e 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -1327,8 +1327,6 @@ void ParallelExecutor::InitExecutorPrivateMemberInfo(
     device_name = "CPU";
   } else if (member_->use_device_ == p::kCUDA) {
     device_name = "CUDA";
-  } else if (member_->use_device_ == p::kNPU) {
-    device_name = "NPU";
   } else if (member_->use_device_ == p::kXPU) {
     device_name = "XPU";
   } else {
diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc
index 928153dab5e62..466eee7d9414f 100644
--- a/paddle/fluid/inference/api/analysis_predictor.cc
+++ b/paddle/fluid/inference/api/analysis_predictor.cc
@@ -138,8 +138,6 @@ phi::Backend ConvertBackend(paddle_infer::PlaceType backend) {
     case paddle_infer::PlaceType::kGPU:
       // NOTE: phi also support phi::Backend::GPUDNN.
       return phi::Backend::GPU;
-    case paddle_infer::PlaceType::kNPU:
-      return phi::Backend::NPU;
     case paddle_infer::PlaceType::kXPU:
       return phi::Backend::XPU;
     case paddle_infer::PlaceType::kCPU:
diff --git a/paddle/fluid/inference/api/api_impl.cc b/paddle/fluid/inference/api/api_impl.cc
index 0d5c8f98020a8..28353150c265c 100644
--- a/paddle/fluid/inference/api/api_impl.cc
+++ b/paddle/fluid/inference/api/api_impl.cc
@@ -82,8 +82,6 @@ bool NativePaddlePredictor::Init(
     place_ = paddle::platform::CUDAPlace(config_.device);
   } else if (config_.use_xpu) {
     place_ = paddle::platform::XPUPlace(config_.device);
-  } else if (config_.use_npu) {
-    place_ = paddle::platform::NPUPlace(config_.device);
   } else {
     place_ = paddle::platform::CPUPlace();
   }
diff --git a/paddle/fluid/inference/api/details/zero_copy_tensor.cc b/paddle/fluid/inference/api/details/zero_copy_tensor.cc
index 9fd94fa5d57e2..f5a9f4f02f8bc 100644
--- a/paddle/fluid/inference/api/details/zero_copy_tensor.cc
+++ b/paddle/fluid/inference/api/details/zero_copy_tensor.cc
@@ -124,9 +124,6 @@ T *Tensor::mutable_data(PlaceType place) {
     case static_cast<int>(PlaceType::kXPU): {
       return tensor->mutable_data<T>(paddle::platform::XPUPlace(device_));
     }
-    case static_cast<int>(PlaceType::kNPU): {
-      return tensor->mutable_data<T>(paddle::platform::NPUPlace(device_));
-    }
     case static_cast<int>(PlaceType::kCUSTOM): {
       return tensor->mutable_data<T>(
           paddle::platform::CustomPlace(device_type_, device_));
diff --git a/paddle/fluid/inference/api/paddle_tensor.h b/paddle/fluid/inference/api/paddle_tensor.h
index 3838bc1c829c4..25b5ba8be7a21 100644
--- a/paddle/fluid/inference/api/paddle_tensor.h
+++ b/paddle/fluid/inference/api/paddle_tensor.h
@@ -67,7 +67,7 @@ enum DataType {
   // TODO(Inference): support more data types if needed.
 };
 
-enum class PlaceType { kUNK = -1, kCPU, kGPU, kXPU, kNPU, kIPU, kCUSTOM };
+enum class PlaceType { kUNK = -1, kCPU, kGPU, kXPU, kIPU, kCUSTOM };
 
 enum class DataLayout { kUNK = -1, kAny, kNHWC, kNCHW };
 
diff --git a/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc b/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc
index 4a0160affefcb..2b34747fd124d 100644
--- a/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc
+++ b/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc
@@ -53,6 +53,3 @@ namespace plat = paddle::platform;
 REGISTER_OP_WITHOUT_GRADIENT(c_sync_comm_stream,
                              ops::CSyncCommStreamOp,
                              ops::CSyncCommStreamOpMaker);
-
-REGISTER_OP_NPU_KERNEL(c_sync_comm_stream,
-                       ops::CSyncCommStreamKernel<float, plat::NPUPlace>);
diff --git a/paddle/fluid/operators/fill_constant_op.cc b/paddle/fluid/operators/fill_constant_op.cc
index 4b2ee8763cd65..8af460259a37b 100644
--- a/paddle/fluid/operators/fill_constant_op.cc
+++ b/paddle/fluid/operators/fill_constant_op.cc
@@ -95,9 +95,6 @@ class FillConstantOp : public framework::OperatorWithKernel {
         case 3:
           kt.set_backend(phi::Backend::XPU);
           break;
-        case 4:
-          kt.set_backend(phi::Backend::NPU);
-          break;
         default:
           PADDLE_THROW(platform::errors::Unimplemented(
               "Could NOT determine the place of variable, place_type = %d .",
@@ -161,8 +158,7 @@ class FillConstantOpMaker : public framework::OpProtoAndCheckerMaker {
                  "0: CPUPlace. "
                  "1: CUDAPlace. "
                  "2: CUDAPinnedPlace. "
-                 "3: XPUPlace. "
-                 "4: NPUPlace. ")
+                 "3: XPUPlace. ")
         .SetDefault(-1);
     AddOutput("Out",
               "(Tensor) Tensor of specified shape will be filled "
diff --git a/paddle/fluid/operators/memcpy_d2h_op.cc b/paddle/fluid/operators/memcpy_d2h_op.cc
index 60d7a6ee14ba7..4b73dfd0353e1 100644
--- a/paddle/fluid/operators/memcpy_d2h_op.cc
+++ b/paddle/fluid/operators/memcpy_d2h_op.cc
@@ -86,16 +86,15 @@ class MemcpyD2HOpProtoMaker : public framework::OpProtoAndCheckerMaker {
     AddOutput("Out",
               "(phi::DenseTensor) The type of output "
               "is the same as input X.");
-    AddAttr<int>(
-        "dst_place_type",
-        "Determine the dst place of tensor copy. "
-        "By Now it ONLY support XPU/NPUPlace/CUDAPlace <-> CUDAPinnedPlace/CPU"
-        "Other place type is Unimplemented and will cause ERROR."
-        "0: dst is on CPUPlace. "
-        "1: dst is on CUDAPinnedPlace. ");
+    AddAttr<int>("dst_place_type",
+                 "Determine the dst place of tensor copy. "
+                 "By Now it ONLY support XPU/CUDAPlace <-> CUDAPinnedPlace/CPU"
+                 "Other place type is Unimplemented and will cause ERROR."
+                 "0: dst is on CPUPlace. "
+                 "1: dst is on CUDAPinnedPlace. ");
     AddComment(R"DOC(
     MemcpyD2H Operator.
-    By now, it ONLY supports the memcopy between NPUPlace/CUDAPlace <-> CUDAPinnedPlace/CPU.
+    By now, it ONLY supports the memcopy between CUDAPlace <-> CUDAPinnedPlace/CPU.
     You would have to update it if you want other more capacities.
 Out = X,  when type in [phi::DenseTensor]
 raise error if the type is not listed above.
diff --git a/paddle/fluid/operators/memcpy_h2d_op.cc b/paddle/fluid/operators/memcpy_h2d_op.cc
index 8997205af193a..d90243f8f9ec3 100644
--- a/paddle/fluid/operators/memcpy_h2d_op.cc
+++ b/paddle/fluid/operators/memcpy_h2d_op.cc
@@ -91,13 +91,12 @@ class MemcpyH2DOpProtoMaker : public framework::OpProtoAndCheckerMaker {
                  "Determine the dst place of tensor copy. "
                  "By Now it support:"
                  "0. CUDAPinnedPlace/CPU <->CUDAPlace"
-                 "1. NPUPinnedPlace/CPU <-> NPUPlace"
-                 "2. CPU <->XPUPlace"
-                 "3. CPU <->IPUPlace"
+                 "1. CPU <->XPUPlace"
+                 "2. CPU <->IPUPlace"
                  "Other place type is Unimplemented and will cause ERROR.");
     AddComment(R"DOC(
     MemcpyD2H Operator.
-    By now, it ONLY supports the memcopy between CUDAPinnedPlace/CPU <-> NPUPlace/CUDAPlace.
+    By now, it ONLY supports the memcopy between CUDAPinnedPlace/CPU <-> CUDAPlace.
     You would have to update it if you want other more capacities.
 Out = X,  when type in [phi::DenseTensor]
 raise error if the type is not listed above.
diff --git a/paddle/fluid/operators/memcpy_op.cc b/paddle/fluid/operators/memcpy_op.cc
index 6d6baf1e5295c..c1ee17789ac46 100644
--- a/paddle/fluid/operators/memcpy_op.cc
+++ b/paddle/fluid/operators/memcpy_op.cc
@@ -105,20 +105,17 @@ class MemcpyOpProtoMaker : public framework::OpProtoAndCheckerMaker {
               "is the same as input X.");
     AddAttr<int>("dst_place_type",
                  "Determine the dst place of tensor copy. "
-                 "By Now it ONLY support CUDAPlace <-> CUDAPinnedPlace or "
-                 "NPUPlace <-> CPUPlace. "
+                 "By Now it ONLY support CUDAPlace <-> CUDAPinnedPlace."
                  "Other place type is Unimplemented and will cause ERROR."
                  "0: dst is on CPUPlace. "
                  "1: dst is on CUDAPlace. "
                  "2: dst is on CUDAPinnedPlace. "
                  "3: dst is on XPUPlace. "
-                 "4: dst is on NPUPlace. "
-                 "5: dst is on NPUPinnerPlace. "
-                 "6: dst is on CustomDevicePlace");
+                 "4: dst is on NPUPinnerPlace. "
+                 "5: dst is on CustomDevicePlace");
     AddComment(R"DOC(
     Memcpy Operator.
-    By now, it ONLY supports the memcopy between CUDAPinnedPlace <-> CUDAPlace or
-    NPUPlace <-> CPUPlace, and used as an internal op by Recompute-Offload.
+    By now, it ONLY supports the memcopy between CUDAPinnedPlace <-> CUDAPlace, and used as an internal op by Recompute-Offload.
     You would have to update it if you want other more capacities.
 
 Out = X,  when type in [phi::DenseTensor]
diff --git a/paddle/fluid/platform/device_context.h b/paddle/fluid/platform/device_context.h
index 96ddef2c60efe..f6cdb19f8d1f8 100644
--- a/paddle/fluid/platform/device_context.h
+++ b/paddle/fluid/platform/device_context.h
@@ -106,7 +106,6 @@ DeviceType Place2DeviceType(const platform::Place& place);
 constexpr DeviceType kCPU = DeviceType::CPU;
 constexpr DeviceType kCUDA = DeviceType::CUDA;
 constexpr DeviceType kXPU = DeviceType::XPU;
-constexpr DeviceType kNPU = DeviceType::NPU;
 constexpr DeviceType kIPU = DeviceType::IPU;
 constexpr DeviceType kCUSTOM_DEVICE = DeviceType::CUSTOM_DEVICE;
 
diff --git a/paddle/fluid/platform/device_event.h b/paddle/fluid/platform/device_event.h
index dc40fbe186e88..402974b89e5c9 100644
--- a/paddle/fluid/platform/device_event.h
+++ b/paddle/fluid/platform/device_event.h
@@ -26,7 +26,6 @@
 using ::paddle::platform::kCPU;
 using ::paddle::platform::kCUDA;
 using ::paddle::platform::kCUSTOM_DEVICE;
-using ::paddle::platform::kNPU;
 using ::paddle::platform::kXPU;
 
 USE_EVENT(kCPU)
diff --git a/paddle/fluid/platform/place.cc b/paddle/fluid/platform/place.cc
index ba5a25aa21a73..b8452a594e358 100644
--- a/paddle/fluid/platform/place.cc
+++ b/paddle/fluid/platform/place.cc
@@ -102,8 +102,6 @@ Place PlaceHelper::CreatePlace(const std::string &dev_type, size_t dev_id) {
     return platform::CPUPlace();
   } else if (dev_type == "gpu") {
     return platform::CUDAPlace(dev_id);
-  } else if (dev_type == "npu") {
-    return platform::NPUPlace(dev_id);
   } else if (dev_type == "xpu") {
     return platform::XPUPlace(dev_id);
   } else {
diff --git a/paddle/fluid/platform/place.h b/paddle/fluid/platform/place.h
index 3ea1dcded2c62..fe20429ae829e 100644
--- a/paddle/fluid/platform/place.h
+++ b/paddle/fluid/platform/place.h
@@ -28,7 +28,6 @@ using Place = phi::Place;
 using CPUPlace = phi::CPUPlace;
 using CUDAPlace = phi::GPUPlace;
 using CUDAPinnedPlace = phi::GPUPinnedPlace;
-using NPUPlace = phi::NPUPlace;
 using NPUPinnedPlace = phi::NPUPinnedPlace;
 using XPUPlace = phi::XPUPlace;
 using IPUPlace = phi::IPUPlace;
@@ -88,11 +87,6 @@ typename Visitor::result_type VisitPlace(const Place &place,
       return typename Visitor::result_type();
 #endif
     }
-    case phi::AllocationType::NPU: {
-      PADDLE_THROW(platform::errors::Unavailable(
-          "Paddle is not compiled with NPU. Cannot visit npu_pinned"));
-      return typename Visitor::result_type();
-    }
     case phi::AllocationType::NPUPINNED: {
       PADDLE_THROW(platform::errors::Unavailable(
           "Paddle is not compiled with NPU. Cannot visit npu_pinned"));
diff --git a/paddle/fluid/pybind/eager.cc b/paddle/fluid/pybind/eager.cc
index d69417a6c0a73..3029cee9e0d31 100644
--- a/paddle/fluid/pybind/eager.cc
+++ b/paddle/fluid/pybind/eager.cc
@@ -159,7 +159,7 @@ void InitTensorWithNumpyValue(TensorObject* self,
   } else {
     PADDLE_THROW(platform::errors::InvalidArgument(
         "Place should be one of "
-        "CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/NPUPlace/CustomPlace"));
+        "CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/CustomPlace"));
   }
 }
 
diff --git a/paddle/fluid/pybind/eager_math_op_patch.cc b/paddle/fluid/pybind/eager_math_op_patch.cc
index d764aedd8a1d6..69d0465bf7cdd 100644
--- a/paddle/fluid/pybind/eager_math_op_patch.cc
+++ b/paddle/fluid/pybind/eager_math_op_patch.cc
@@ -108,7 +108,7 @@ void InitTensorWithNumpyValue(const py::object& array,
   } else {
     PADDLE_THROW(platform::errors::InvalidArgument(
         "Place should be one of "
-        "CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/NPUPlace/CustomPlace"));
+        "CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/CustomPlace"));
   }
 }
 
diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc
index 0312ad8d96041..59bc745aedc03 100644
--- a/paddle/fluid/pybind/eager_utils.cc
+++ b/paddle/fluid/pybind/eager_utils.cc
@@ -52,7 +52,6 @@ extern PyTypeObject* g_place_pytype;
 extern PyTypeObject* g_cudaplace_pytype;
 extern PyTypeObject* g_cpuplace_pytype;
 extern PyTypeObject* g_xpuplace_pytype;
-extern PyTypeObject* g_npuplace_pytype;
 extern PyTypeObject* g_cudapinnedplace_pytype;
 extern PyTypeObject* g_customplace_pytype;
 extern PyTypeObject* g_framework_tensor_pytype;
@@ -529,9 +528,6 @@ platform::Place CastPyArg2Place(PyObject* obj, ssize_t arg_pos) {
   } else if (PyObject_IsInstance(
                  obj, reinterpret_cast<PyObject*>(g_xpuplace_pytype))) {
     place = ::pybind11::handle(obj).cast<platform::XPUPlace>();
-  } else if (PyObject_IsInstance(
-                 obj, reinterpret_cast<PyObject*>(g_npuplace_pytype))) {
-    place = ::pybind11::handle(obj).cast<platform::NPUPlace>();
   } else if (PyObject_IsInstance(
                  obj, reinterpret_cast<PyObject*>(g_cudapinnedplace_pytype))) {
     place = ::pybind11::handle(obj).cast<platform::CUDAPinnedPlace>();
@@ -542,7 +538,7 @@ platform::Place CastPyArg2Place(PyObject* obj, ssize_t arg_pos) {
     PADDLE_THROW(platform::errors::InvalidArgument(
         "argument (position %d) must be "
         "one "
-        "of(Place,CUDAPlace,CPUPlace,XPUPlace,NPUPlace,CUDAPinnedPlace,"
+        "of(Place,CUDAPlace,CPUPlace,XPUPlace,CUDAPinnedPlace,"
         "CustomPlace), "
         "but got %s",
         arg_pos + 1,
diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc
index 44f0cfc9eb8b4..aa1ee4724925e 100644
--- a/paddle/fluid/pybind/imperative.cc
+++ b/paddle/fluid/pybind/imperative.cc
@@ -144,8 +144,6 @@ static const platform::Place PyObjectToPlace(const py::object &place_obj) {
     return place_obj.cast<platform::XPUPlace>();
   } else if (py::isinstance<platform::CUDAPinnedPlace>(place_obj)) {
     return place_obj.cast<platform::CUDAPinnedPlace>();
-  } else if (py::isinstance<platform::NPUPlace>(place_obj)) {
-    return place_obj.cast<platform::NPUPlace>();
   } else if (py::isinstance<platform::IPUPlace>(place_obj)) {
     return place_obj.cast<platform::IPUPlace>();
   } else if (py::isinstance<platform::Place>(place_obj)) {
@@ -155,7 +153,7 @@ static const platform::Place PyObjectToPlace(const py::object &place_obj) {
   } else {
     PADDLE_THROW(platform::errors::InvalidArgument(
         "Place should be one of "
-        "Place/CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/NPUPlace/IPUPlace/"
+        "Place/CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/IPUPlace/"
         "CustomPlace"));
   }
 }
@@ -208,7 +206,7 @@ static void InitVarBaseAndTensor(imperative::VarBase *self,
   } else {
     PADDLE_THROW(platform::errors::InvalidArgument(
         "Place should be one of "
-        "CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/NPUPlace/IPUPlace/"));
+        "CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/IPUPlace/"));
   }
   self->SetDataType(framework::TransToProtoVarType(tensor->dtype()));
 }
@@ -711,14 +709,6 @@ void BindImperative(py::module *m_ptr) {
            py::arg("zero_copy") = false,
            py::arg("name") = "",
            py::arg("stop_gradient") = -1)
-      .def("__init__",
-           &InitVarBaseFromNumpyWithArg<platform::NPUPlace>,
-           py::arg("value"),
-           py::arg("place"),
-           py::arg("persistable") = false,
-           py::arg("zero_copy") = false,
-           py::arg("name") = "",
-           py::arg("stop_gradient") = -1)
       .def("__init__",
            &InitVarBaseFromNumpyWithArg<platform::CustomPlace>,
            py::arg("value"),
@@ -752,11 +742,6 @@ void BindImperative(py::module *m_ptr) {
            py::arg("tensor"),
            py::arg("place"),
            py::arg("name") = "")
-      .def("__init__",
-           &InitVarBaseFromTensorWithArg<platform::NPUPlace>,
-           py::arg("tensor"),
-           py::arg("place"),
-           py::arg("name") = "")
       .def("__init__",
            &InitVarBaseFromTensorWithArg<platform::CustomPlace>,
            py::arg("tensor"),
@@ -1877,18 +1862,6 @@ void BindImperative(py::module *m_ptr) {
             return new_var;
           },
           py::return_value_policy::copy)
-      .def(
-          "_copy_to",
-          [](const std::shared_ptr<imperative::VarBase> &self,
-             const platform::NPUPlace &place,
-             bool blocking) {
-            auto new_var = self->NewVarBase(place, blocking);
-            if (!blocking) {
-              IncreaseVarbaseReferenceCountUntilCopyComplete(self, place);
-            }
-            return new_var;
-          },
-          py::return_value_policy::copy)
       .def(
           "_copy_to",
           [](const std::shared_ptr<imperative::VarBase> &self,
@@ -2219,11 +2192,6 @@ void BindImperative(py::module *m_ptr) {
               self.SetExpectedPlace(*p);
               VLOG(4) << "Tracer(" << &self << ")"
                       << " set expected place " << *p;
-            } else if (py::isinstance<platform::NPUPlace>(obj)) {
-              auto p = obj.cast<platform::NPUPlace *>();
-              self.SetExpectedPlace(*p);
-              VLOG(4) << "Tracer(" << &self << ")"
-                      << " set expected place " << *p;
             } else if (py::isinstance<platform::IPUPlace>(obj)) {
               auto p = obj.cast<platform::IPUPlace *>();
               self.SetExpectedPlace(*p);
@@ -2242,7 +2210,7 @@ void BindImperative(py::module *m_ptr) {
             } else {
               PADDLE_THROW(platform::errors::InvalidArgument(
                   "Incompatible Place Type: supports XPUPlace, CUDAPlace, "
-                  "CPUPlace, NPUPlace, IPUPlace"
+                  "CPUPlace, IPUPlace"
                   "and CUDAPinnedPlace, "
                   "but got Unknown Type!"));
             }
@@ -2375,28 +2343,6 @@ void BindImperative(py::module *m_ptr) {
                                                  inplace_map);
              }
            })
-      .def("trace",
-           [](imperative::Tracer &self,
-              const std::string &type,
-              const PyNameVarBaseMap &ins,
-              const PyNameVarBaseMap &outs,
-              framework::AttributeMap attrs,
-              const platform::NPUPlace &place,
-              bool trace_backward,
-              const std::map<std::string, std::string> &inplace_map = {}) {
-             auto ins_map = ConvertToNameVarBaseMap(ins);
-             auto outs_map = ConvertToNameVarBaseMap(outs);
-             {
-               py::gil_scoped_release release;
-               self.TraceOp<imperative::VarBase>(type,
-                                                 std::move(ins_map),
-                                                 std::move(outs_map),
-                                                 std::move(attrs),
-                                                 place,
-                                                 trace_backward,
-                                                 inplace_map);
-             }
-           })
       .def("trace",
            [](imperative::Tracer &self,
               const std::string &type,
@@ -2488,7 +2434,6 @@ void BindImperative(py::module *m_ptr) {
   m.def("varbase_copy", &VarBaseCopy<platform::CUDAPlace>);
   m.def("varbase_copy", &VarBaseCopy<platform::XPUPlace>);
   m.def("varbase_copy", &VarBaseCopy<platform::CUDAPinnedPlace>);
-  m.def("varbase_copy", &VarBaseCopy<platform::NPUPlace>);
   m.def("varbase_copy", &VarBaseCopy<platform::CustomPlace>);
 
   m.def(
diff --git a/paddle/fluid/pybind/inference_api.cc b/paddle/fluid/pybind/inference_api.cc
index e00c22423eb28..a8bfd5a9917e7 100644
--- a/paddle/fluid/pybind/inference_api.cc
+++ b/paddle/fluid/pybind/inference_api.cc
@@ -637,7 +637,6 @@ void BindPaddlePlace(py::module *m) {
       .value("CPU", PaddlePlace::kCPU)
       .value("GPU", PaddlePlace::kGPU)
       .value("XPU", PaddlePlace::kXPU)
-      .value("NPU", PaddlePlace::kNPU)
       .value("CUSTOM", PaddlePlace::kCUSTOM);
 }
 
diff --git a/paddle/fluid/pybind/place.cc b/paddle/fluid/pybind/place.cc
index 59ed791c4522c..ae25b00c93465 100644
--- a/paddle/fluid/pybind/place.cc
+++ b/paddle/fluid/pybind/place.cc
@@ -189,7 +189,6 @@ PyTypeObject *g_customplace_pytype = nullptr;
 PyTypeObject *g_cudaplace_pytype = nullptr;
 PyTypeObject *g_cpuplace_pytype = nullptr;
 PyTypeObject *g_xpuplace_pytype = nullptr;
-PyTypeObject *g_npuplace_pytype = nullptr;
 PyTypeObject *g_cudapinnedplace_pytype = nullptr;
 PyTypeObject *g_ipuplace_pytype = nullptr;
 
@@ -366,7 +365,6 @@ void BindPlace(pybind11::module &m) {  // NOLINT
       .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CUDAPlace>)
       .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CPUPlace>)
       .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::XPUPlace>)
-      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::NPUPlace>)
       .def("_equals",
            &IsSamePlace<platform::CUDAPlace, platform::CUDAPinnedPlace>)
       .def("_get_device_id",
@@ -495,7 +493,6 @@ void BindPlace(pybind11::module &m) {  // NOLINT
       .def("_type", &PlaceIndex<platform::CPUPlace>)
       .def("_equals", &IsSamePlace<platform::CPUPlace, platform::Place>)
       .def("_equals", &IsSamePlace<platform::CPUPlace, platform::XPUPlace>)
-      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::NPUPlace>)
       .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CUDAPlace>)
       .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CPUPlace>)
       .def("_equals",
@@ -548,8 +545,6 @@ void BindPlace(pybind11::module &m) {  // NOLINT
            &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPlace>)
       .def("_equals",
            &IsSamePlace<platform::CUDAPinnedPlace, platform::XPUPlace>)
-      .def("_equals",
-           &IsSamePlace<platform::CUDAPinnedPlace, platform::NPUPlace>)
       .def("_equals",
            &IsSamePlace<platform::CUDAPinnedPlace, platform::CPUPlace>)
       .def("_equals",
@@ -557,30 +552,6 @@ void BindPlace(pybind11::module &m) {  // NOLINT
       .def("__repr__", string::to_string<const platform::CUDAPinnedPlace &>)
       .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);
 
-  // NPUPlace
-  py::class_<platform::NPUPlace> npuplace(m, "NPUPlace", R"DOC(
-    NPUPlace is a descriptor of a device.
-    It represents a NPU device on which a tensor will be allocated and a model will run.
-    Examples:
-        .. code-block:: python
-          # required: npu
-          import paddle
-          place = paddle.NPUPlace(0)
-        )DOC");
-  g_npuplace_pytype = reinterpret_cast<PyTypeObject *>(npuplace.ptr());
-  npuplace.def("__init__", [](platform::NPUPlace &self, int dev_id) {})
-      .def("_type", &PlaceIndex<platform::NPUPlace>)
-      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::Place>)
-      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::CUDAPlace>)
-      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::CPUPlace>)
-      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::XPUPlace>)
-      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::NPUPlace>)
-      .def("_equals",
-           &IsSamePlace<platform::NPUPlace, platform::CUDAPinnedPlace>)
-      .def("get_device_id",
-           [](const platform::NPUPlace &self) { return self.GetDeviceId(); })
-      .def("__str__", string::to_string<const platform::NPUPlace &>);
-
   // IPUPlace
   py::class_<platform::IPUPlace> ipuplace(m, "IPUPlace", R"DOC(
     IPUPlace is a descriptor of a device.
@@ -625,7 +596,6 @@ void BindPlace(pybind11::module &m) {  // NOLINT
       .def("_equals", &IsSamePlace<platform::IPUPlace, platform::CUDAPlace>)
       .def("_equals", &IsSamePlace<platform::IPUPlace, platform::CPUPlace>)
       .def("_equals", &IsSamePlace<platform::IPUPlace, platform::XPUPlace>)
-      .def("_equals", &IsSamePlace<platform::IPUPlace, platform::NPUPlace>)
       .def("_equals", &IsSamePlace<platform::IPUPlace, platform::IPUPlace>)
       .def("_equals",
            &IsSamePlace<platform::IPUPlace, platform::CUDAPinnedPlace>)
@@ -639,7 +609,6 @@ void BindPlace(pybind11::module &m) {  // NOLINT
       .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPlace>)
       .def("_equals", &IsSamePlace<platform::Place, platform::CPUPlace>)
       .def("_equals", &IsSamePlace<platform::Place, platform::XPUPlace>)
-      .def("_equals", &IsSamePlace<platform::Place, platform::NPUPlace>)
       .def("_equals", &IsSamePlace<platform::Place, platform::IPUPlace>)
       .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPinnedPlace>)
       .def("_equals", &IsSamePlace<platform::Place, platform::CustomPlace>)
@@ -685,10 +654,6 @@ void BindPlace(pybind11::module &m) {  // NOLINT
               const platform::CUDAPinnedPlace &cuda_pinned_place) {
              self = cuda_pinned_place;
            })
-      .def("set_place",
-           [](platform::Place &self, const platform::NPUPlace &npu_place) {
-             self = npu_place;
-           })
       .def("set_place",
            [](platform::Place &self, const platform::IPUPlace &ipu_place) {
              self = ipu_place;
diff --git a/paddle/fluid/pybind/tensor.cc b/paddle/fluid/pybind/tensor.cc
index 799e48b1594de..bdd2a50d96e01 100644
--- a/paddle/fluid/pybind/tensor.cc
+++ b/paddle/fluid/pybind/tensor.cc
@@ -245,10 +245,6 @@ void BindTensor(pybind11::module &m) {  // NOLINT
            [](phi::DenseTensor &self, paddle::platform::CPUPlace &place) {
              self.mutable_data<float>(place);
            })
-      .def("_alloc_float",
-           [](phi::DenseTensor &self, paddle::platform::NPUPlace &place) {
-             self.mutable_data<float>(place);
-           })
       .def("_alloc_double",
            [](phi::DenseTensor &self, paddle::platform::CPUPlace &place) {
              self.mutable_data<double>(place);
@@ -315,13 +311,6 @@ void BindTensor(pybind11::module &m) {  // NOLINT
                  self.mutable_data(place, framework::TransToPhiDataType(type)));
            })
       .def("_clear", &phi::DenseTensor::clear)
-      .def("_mutable_data",
-           [](phi::DenseTensor &self,
-              paddle::platform::NPUPlace &place,
-              paddle::framework::proto::VarType::Type type) {
-             return reinterpret_cast<uintptr_t>(
-                 self.mutable_data(place, framework::TransToPhiDataType(type)));
-           })
       .def("_copy_from",
            &TensorCopyFrom<paddle::platform::CPUPlace>,
            py::arg("tensor"),
@@ -342,11 +331,6 @@ void BindTensor(pybind11::module &m) {  // NOLINT
            py::arg("tensor"),
            py::arg("place"),
            py::arg("batch_size") = -1)
-      .def("_copy_from",
-           &TensorCopyFrom<paddle::platform::NPUPlace>,
-           py::arg("tensor"),
-           py::arg("place"),
-           py::arg("batch_size") = -1)
       .def("_copy_from",
            &TensorCopyFrom<paddle::platform::CUDAPinnedPlace>,
            py::arg("tensor"),
@@ -382,11 +366,6 @@ void BindTensor(pybind11::module &m) {  // NOLINT
            py::arg("array"),
            py::arg("place"),
            py::arg("zero_copy") = false)
-      .def("set",
-           SetTensorFromPyArray<paddle::platform::NPUPlace>,
-           py::arg("array"),
-           py::arg("place"),
-           py::arg("zero_copy") = false)
       .def("set",
            SetTensorFromPyArray<paddle::platform::IPUPlace>,
            py::arg("array"),
@@ -402,7 +381,7 @@ void BindTensor(pybind11::module &m) {  // NOLINT
 
         Args:
           lod (numpy.ndarray): The data to set.
-          place (CPUPlace|CUDAPlace|XPUPlace|IPUPlace|CUDAPinnedPlace|NPUPlace): The place where the
+          place (CPUPlace|CUDAPlace|XPUPlace|IPUPlace|CUDAPinnedPlace): The place where the
           Tensor is to be set.
           zero_copy (bool, optional): Whether to share memory with the input numpy array.
           This parameter only works with CPUPlace. Default: False.
diff --git a/paddle/phi/api/include/tensor.h b/paddle/phi/api/include/tensor.h
index 24bcc63dbd278..603d12c4f7750 100644
--- a/paddle/phi/api/include/tensor.h
+++ b/paddle/phi/api/include/tensor.h
@@ -629,7 +629,7 @@ class PADDLE_API Tensor final {
    * unified to Tensor, but Tensor itself is heterogeneous.
    *
    * Tensor can generally be represented by void* and size_t, place.
-   * This is suitable for most scenarios including CPU, GPU, HIP, NPU, etc.,
+   * This is suitable for most scenarios including CPU, GPU, HIP, etc.,
    * but there are a few cases where this definition cannot be described,
    * such as the Tensor representation in third-party lib such as Metal,
    * OpenCL, etc., as well as some special Tensor implementations, including
diff --git a/paddle/phi/api/profiler/device_tracer.cc b/paddle/phi/api/profiler/device_tracer.cc
index 7bd09ff8413b0..ae026683f1509 100644
--- a/paddle/phi/api/profiler/device_tracer.cc
+++ b/paddle/phi/api/profiler/device_tracer.cc
@@ -696,8 +696,6 @@ class DeviceTracerImpl : public DeviceTracer {
           event->set_device_id(r.place.GetDeviceId());
         } else if (r.place.GetType() == phi::AllocationType::GPUPINNED) {
           event->set_place(proto::MemEvent::CUDAPinnedPlace);
-        } else if (r.place.GetType() == phi::AllocationType::NPU) {
-          event->set_place(proto::MemEvent::NPUPlace);
         } else {
           PADDLE_THROW(
               errors::Unimplemented("The current place is not supported."));
diff --git a/paddle/phi/common/backend.h b/paddle/phi/common/backend.h
index b7f30797ca78e..dd76eccca7af2 100644
--- a/paddle/phi/common/backend.h
+++ b/paddle/phi/common/backend.h
@@ -91,9 +91,6 @@ inline std::ostream& operator<<(std::ostream& os, Backend backend) {
     case Backend::XPU:
       os << "XPU";
       break;
-    case Backend::NPU:
-      os << "NPU";
-      break;
     case Backend::ONEDNN:
       os << "ONEDNN";
       break;
@@ -137,8 +134,6 @@ inline Backend StringToBackend(const char* backend_cstr) {
     return Backend::GPU;
   } else if (s == std::string("XPU")) {
     return Backend::XPU;
-  } else if (s == std::string("NPU")) {
-    return Backend::NPU;
   } else if (s == std::string("OneDNN")) {
     return Backend::ONEDNN;
   } else if (s == std::string("GPUDNN")) {
@@ -173,8 +168,6 @@ inline std::string BackendToString(const Backend& backend) {
       return "GPU";
     case Backend::XPU:
       return "XPU";
-    case Backend::NPU:
-      return "NPU";
     case Backend::ONEDNN:
       return "ONEDNN";
     case Backend::GPUDNN:
diff --git a/paddle/phi/common/place.cc b/paddle/phi/common/place.cc
index bbe30cacfcf3f..d97f3f4fd7598 100644
--- a/paddle/phi/common/place.cc
+++ b/paddle/phi/common/place.cc
@@ -35,8 +35,6 @@ const char *AllocationTypeStr(AllocationType type) {
       return "gpu_pinned";
     case AllocationType::XPU:
       return "xpu";
-    case AllocationType::NPU:
-      return "npu";
     case AllocationType::NPUPINNED:
       return "npu_pinned";
     case AllocationType::IPU:
@@ -76,8 +74,6 @@ Place GetPinnedPlace(const Place &place) {
     case AllocationType::GPU:
       return phi::GPUPinnedPlace();
       break;
-    case AllocationType::NPU:
-      return phi::NPUPinnedPlace();
     default:
       return place;
   }
diff --git a/paddle/phi/common/place.h b/paddle/phi/common/place.h
index 543a79977eb7e..fcd9634a1892d 100644
--- a/paddle/phi/common/place.h
+++ b/paddle/phi/common/place.h
@@ -163,16 +163,6 @@ class XPUPlace : public Place {
       : Place(AllocationType::XPU, place.GetDeviceId()) {}
 };
 
-class NPUPlace : public Place {
- public:
-  NPUPlace() : Place(AllocationType::NPU, 0) {}
-  explicit NPUPlace(int device_id) : Place(AllocationType::NPU, device_id) {}
-
-  NPUPlace(const NPUPlace&) = default;
-  NPUPlace(const Place& place)  // NOLINT
-      : Place(AllocationType::NPU, place.GetDeviceId()) {}
-};
-
 class NPUPinnedPlace : public Place {
  public:
   NPUPinnedPlace() : Place(AllocationType::NPUPINNED) {}
@@ -220,7 +210,6 @@ namespace experimental {
 using AllocationType = phi::AllocationType;
 using GPUPinnedPlace = phi::GPUPinnedPlace;
 using XPUPlace = phi::XPUPlace;
-using NPUPlace = phi::NPUPlace;
 }  // namespace experimental
 
 using AllocationType = phi::AllocationType;
diff --git a/paddle/phi/core/compat/convert_utils.cc b/paddle/phi/core/compat/convert_utils.cc
index d41047a32166a..947c7fb45c5fc 100644
--- a/paddle/phi/core/compat/convert_utils.cc
+++ b/paddle/phi/core/compat/convert_utils.cc
@@ -37,8 +37,6 @@ Backend TransToPhiBackend(const phi::Place& place) {
       return Backend::GPU;
     case AllocationType::XPU:
       return Backend::XPU;
-    case AllocationType::NPU:
-      return Backend::NPU;
     case AllocationType::IPU:
       return Backend::IPU;
     case AllocationType::CUSTOM:
diff --git a/paddle/phi/kernels/funcs/math_function.cc b/paddle/phi/kernels/funcs/math_function.cc
index 6e54718afb542..d13ce8a2a90b5 100644
--- a/paddle/phi/kernels/funcs/math_function.cc
+++ b/paddle/phi/kernels/funcs/math_function.cc
@@ -161,13 +161,6 @@ void set_constant_with_place<phi::XPUPlace>(const phi::DeviceContext& context,
 #endif
 }
 
-template <>
-void set_constant_with_place<phi::NPUPlace>(const phi::DeviceContext& context,
-                                            phi::DenseTensor* tensor,
-                                            float value) {
-  PADDLE_THROW(phi::errors::Unimplemented("NPUPlace is not supported"));
-}
-
 template <>
 void set_constant_with_place<phi::NPUPinnedPlace>(
     const phi::DeviceContext& context, phi::DenseTensor* tensor, float value) {
diff --git a/python/paddle/amp/auto_cast.py b/python/paddle/amp/auto_cast.py
index d1eaf0dbd13ae..2cb0af3530eab 100644
--- a/python/paddle/amp/auto_cast.py
+++ b/python/paddle/amp/auto_cast.py
@@ -348,7 +348,7 @@ def amp_guard(
         or tracer._expected_place.is_custom_place()
     ):
         warnings.warn(
-            'amp_guard can only be enabled on CUDAPlace, XPUPlace, NPUPlace, and CustomPlace, current place is %s, so it makes no effect.'
+            'amp_guard can only be enabled on CUDAPlace, XPUPlace, and CustomPlace, current place is %s, so it makes no effect.'
             % tracer._expected_place
         )
         enable = False
diff --git a/python/paddle/framework/__init__.py b/python/paddle/framework/__init__.py
index 3c6338e345ddb..66456d48253f8 100755
--- a/python/paddle/framework/__init__.py
+++ b/python/paddle/framework/__init__.py
@@ -24,7 +24,6 @@
 from ..fluid.core import IPUPlace  # noqa: F401
 from ..fluid.core import CUDAPlace  # noqa: F401
 from ..fluid.core import CUDAPinnedPlace  # noqa: F401
-from ..fluid.core import NPUPlace  # noqa: F401
 from ..fluid.core import CustomPlace  # noqa: F401
 
 from ..fluid import core  # noqa: F401
diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py
index 0dca909bd5833..ee546ffe17ea0 100644
--- a/python/paddle/tensor/creation.py
+++ b/python/paddle/tensor/creation.py
@@ -2214,7 +2214,7 @@ def _memcpy(input, place=None, output=None):
     """
 
     The OP copies the :attr:`input` to the :attr:`output`.
-    NOTE: currently, only support CUDAPlace <-> CUDAPinnedPlace or NPUPlace <-> CPUPlace.
+    NOTE: currently, only support CUDAPlace <-> CUDAPinnedPlace.
 
     Parameters:
         input (Tensor): A tensor. Its data type supports float16, float32, float64, int32, int64, and bool.
diff --git a/tools/timeline.py b/tools/timeline.py
index 814d51d5f3943..0ef4a886f4623 100644
--- a/tools/timeline.py
+++ b/tools/timeline.py
@@ -194,14 +194,6 @@ def _allocate_pids(self):
                             % (k, mevent.device_id),
                             pid,
                         )
-                elif mevent.place == profiler_pb2.MemEvent.NPUPlace:
-                    if (k, mevent.device_id, "NPU") not in self._mem_devices:
-                        pid = self._allocate_pid()
-                        self._mem_devices[(k, mevent.device_id, "NPU")] = pid
-                        self._chrome_trace.emit_pid(
-                            "memory usage on %s:npu:%d" % (k, mevent.device_id),
-                            pid,
-                        )
                 if (k, 0, "CPU") not in self._mem_devices:
                     pid = self._allocate_pid()
                     self._mem_devices[(k, 0, "CPU")] = pid
@@ -259,7 +251,6 @@ def _allocate_memory_event(self):
             profiler_pb2.MemEvent.CPUPlace: "CPU",
             profiler_pb2.MemEvent.CUDAPlace: "GPU",
             profiler_pb2.MemEvent.CUDAPinnedPlace: "CUDAPinnedPlace",
-            profiler_pb2.MemEvent.NPUPlace: "NPU",
         }
         for k, profile_pb in self._profile_dict.items():
             mem_list = []

From b6c0407ee5dd29f5e1be67be989eff605d2a4b2b Mon Sep 17 00:00:00 2001
From: Zhang Zheng <32410583+ZzSean@users.noreply.github.com>
Date: Mon, 8 May 2023 16:15:16 +0800
Subject: [PATCH 218/405] Fix the calculation of y_grad in divide_backward
 (#53582)

---
 paddle/phi/kernels/funcs/elementwise_functor.h           | 8 ++++----
 paddle/phi/kernels/gpu/elementwise_divide_grad_kernel.cu | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/paddle/phi/kernels/funcs/elementwise_functor.h b/paddle/phi/kernels/funcs/elementwise_functor.h
index a291c1542d3e2..c51417730850c 100644
--- a/paddle/phi/kernels/funcs/elementwise_functor.h
+++ b/paddle/phi/kernels/funcs/elementwise_functor.h
@@ -118,7 +118,7 @@ struct DivGradXYFunctor {
     // dy = - dout * out / y
     phi::Array<OutT, 2> outs;
     outs[0] = a / c;
-    outs[1] = -a * b / c;
+    outs[1] = -a * ((b / c) / c);
     return outs;
   }
 };
@@ -131,7 +131,7 @@ struct DivGradXYFunctor<ComplexType<InT>, ComplexType<OutT>> {
       const ComplexType<InT> c) {
     phi::Array<ComplexType<OutT>, 2> outs;
     ComplexType<InT> c_conj(c.real, -c.imag);
-    ComplexType<InT> out_div_c_conj((b / c).real, -(b / c).imag);
+    ComplexType<InT> out_div_c_conj(((b / c) / c).real, -((b / c) / c).imag);
     outs[0] = a / c_conj;
     outs[1] = -a * out_div_c_conj;
     return outs;
@@ -158,7 +158,7 @@ struct DivGradXFunctor<ComplexType<T>> {
 template <typename T>
 struct DivGradYFunctor {
   inline HOSTDEVICE T operator()(const T a, const T b, const T c) const {
-    return -a * b / c;
+    return -a * ((b / c) / c);
   }
 };
 
@@ -168,7 +168,7 @@ struct DivGradYFunctor<ComplexType<T>> {
   inline HOSTDEVICE ComplexType<T> operator()(const ComplexType<T> a,
                                               const ComplexType<T> b,
                                               const ComplexType<T> c) const {
-    ComplexType<T> out_div_c_conj((b / c).real, -(b / c).imag);
+    ComplexType<T> out_div_c_conj(((b / c) / c).real, -((b / c) / c).imag);
     return -a * out_div_c_conj;
   }
 };
diff --git a/paddle/phi/kernels/gpu/elementwise_divide_grad_kernel.cu b/paddle/phi/kernels/gpu/elementwise_divide_grad_kernel.cu
index 58cdc63b4a3dc..4897060cf44ac 100644
--- a/paddle/phi/kernels/gpu/elementwise_divide_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/elementwise_divide_grad_kernel.cu
@@ -36,7 +36,7 @@ void DivideGradKernel(const Context& dev_ctx,
                       DenseTensor* dy) {
   const auto place = dev_ctx.GetPlace();
   if (dx != nullptr && dy != nullptr) {
-    std::vector<const DenseTensor*> ins = {&dout, &out, &y};
+    std::vector<const DenseTensor*> ins = {&dout, &x, &y};
     GetGradXAndYOut<T>(dev_ctx,
                        place,
                        axis,
@@ -50,7 +50,7 @@ void DivideGradKernel(const Context& dev_ctx,
     GetGradXOrYOut<T>(
         dev_ctx, place, axis, ins, dout, dx, funcs::DivGradXFunctor<T>());
   } else if (dy != nullptr && dx == nullptr) {
-    std::vector<const DenseTensor*> ins = {&dout, &out, &y};
+    std::vector<const DenseTensor*> ins = {&dout, &x, &y};
     GetGradXOrYOut<T>(
         dev_ctx, place, axis, ins, dout, dy, funcs::DivGradYFunctor<T>());
   }

From f74237cd73c35b8a63d7981a190a302d0ebcd03f Mon Sep 17 00:00:00 2001
From: kangguangli <kangguangli@hotmail.com>
Date: Mon, 8 May 2023 17:59:07 +0800
Subject: [PATCH 219/405] fix: newExe didnot gc transferred variable (#53545)

---
 .../new_executor/interpreter/interpreter_util.cc | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
index 3e630c32278ed..99c1ba3d98395 100644
--- a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
+++ b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
@@ -844,6 +844,22 @@ void BuildOpFuncList(const platform::Place& place,
       interpreter::LogDeviceMemoryStats(place);
     }
   }
+
+  // in the unused_var_map, we did not record the variable who are created in
+  // ApplyDataTransform. So we erase these variables here.
+  std::deque<std::shared_ptr<memory::Allocation>>* garbages =
+      new std::deque<std::shared_ptr<memory::Allocation>>();
+  for (const auto& transferred_var : var_scope->DataTransferAddedVars()) {
+    const auto& var_name = transferred_var.first;
+    auto* var = local_scope->FindVar(var_name);
+    if (var == nullptr) continue;
+    VLOG(6) << "Erase variable " << var_name;
+    if (var->IsType<phi::DenseTensor>()) {
+      garbages->emplace_back(
+          var->GetMutable<phi::DenseTensor>()->MoveMemoryHolder());
+    }
+  }
+  delete garbages;
 }
 
 void BuildVariableScope(const framework::BlockDesc& block,

From 116fcadaf19b7cbbf6443a6f0788067381d2dbf4 Mon Sep 17 00:00:00 2001
From: feifei-111 <2364819892@qq.com>
Date: Mon, 8 May 2023 19:09:45 +0800
Subject: [PATCH 220/405] =?UTF-8?q?=E3=80=90BugFix=E3=80=91fix=20err=20of?=
 =?UTF-8?q?=20api=20`to=5Ftensor`,=20which=20caused=20by=20numpy=20version?=
 =?UTF-8?q?=20update=20(#53534)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix

* update code

* pre-commit

* remove scale check (0-D tensor is usable)

* fix data dtype err

* fix numpy default dtype diff

* fix data dtype

* fix data dtype

* update

* fix coverage
---
 python/paddle/tensor/creation.py         | 62 +++++++++++++-----------
 test/dygraph_to_static/test_to_tensor.py | 16 ++++++
 2 files changed, 51 insertions(+), 27 deletions(-)

diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py
index ee546ffe17ea0..6fe3f97cd656a 100644
--- a/python/paddle/tensor/creation.py
+++ b/python/paddle/tensor/creation.py
@@ -646,8 +646,10 @@ def _handle_np_dtype(ndarray, dtype):
 
 def _to_tensor_static(data, dtype=None, stop_gradient=None):
 
-    if isinstance(data, Variable) and (dtype is None or dtype == data.dtype):
+    if isinstance(data, Variable):
         output = data
+        if dtype is not None and dtype != data.dtype:
+            output = paddle.cast(output, dtype)
     else:
         if isinstance(data, np.number):  # Special case for numpy scalars
             data = np.array(data)
@@ -656,17 +658,37 @@ def _to_tensor_static(data, dtype=None, stop_gradient=None):
             if np.isscalar(data) and not isinstance(data, str):
                 data = np.array(data)
             elif isinstance(data, (list, tuple)):
-                data = np.array(data)
+                try:
+                    '''
+                    In numpy version >= 1.24.0, case like:
+                        np.array([Variable, 1, 2])
+                    is not supported, it will raise error (numpy returns an numpy array with dtype='object' in version <= 1.23.5)
+
+                    Thus, process nested structure in except block
+                    '''
+                    data = np.array(data)
+
+                    # for numpy version <= 1.23.5
+                    if data.dtype == 'object':
+                        raise RuntimeError("Numpy get dtype `object`.")
+
+                except:
+                    to_stack_list = [None] * len(data)
+                    for idx, d in enumerate(data):
+                        to_stack_list[idx] = _to_tensor_static(
+                            d, dtype, stop_gradient
+                        )
+                    data = paddle.stack(to_stack_list)
+                    data = paddle.squeeze(data, -1)
 
-            if (
-                isinstance(data, np.ndarray)
-                and not dtype
-                and data.dtype != 'object'
-            ):
-                if data.dtype in ['float16', 'float32', 'float64']:
-                    data = data.astype(paddle.get_default_dtype())
-                elif data.dtype in ['int32']:
-                    data = data.astype('int64')
+            else:
+                raise RuntimeError(
+                    f"Do not support transform type `{type(data)}` to tensor"
+                )
+
+            # fix numpy default dtype
+            if data.dtype in ['float16', 'float32', 'float64']:
+                data = data.astype(paddle.get_default_dtype())
 
         if dtype:
             target_dtype = dtype
@@ -674,24 +696,10 @@ def _to_tensor_static(data, dtype=None, stop_gradient=None):
             target_dtype = data.dtype
         else:
             target_dtype = paddle.get_default_dtype()
-
         target_dtype = convert_dtype(target_dtype)
 
-        if (
-            isinstance(data, np.ndarray)
-            and len(data.shape) > 0
-            and any(isinstance(x, Variable) for x in data)
-        ):
-            to_stack_list = [None] * data.shape[0]
-            for idx, d in enumerate(data):
-                to_stack_list[idx] = _to_tensor_static(d, dtype, stop_gradient)
-            data = paddle.stack(to_stack_list)
-            data = paddle.squeeze(data, -1)
-
-        if not isinstance(data, Variable):
-            output = assign(data)
-        else:
-            output = data
+        output = assign(data)
+
         if convert_dtype(output.dtype) != target_dtype:
             output = paddle.cast(output, target_dtype)
 
diff --git a/test/dygraph_to_static/test_to_tensor.py b/test/dygraph_to_static/test_to_tensor.py
index 05cd5ec78f2c8..e96c5247a0d52 100644
--- a/test/dygraph_to_static/test_to_tensor.py
+++ b/test/dygraph_to_static/test_to_tensor.py
@@ -90,6 +90,11 @@ def case7(x):
     return a
 
 
+def case8(x):
+    a = paddle.to_tensor({1: 1})
+    return a
+
+
 class TestToTensorReturnVal(unittest.TestCase):
     def test_to_tensor_badreturn(self):
         paddle.disable_static()
@@ -143,6 +148,17 @@ def test_to_tensor_badreturn(self):
         self.assertTrue(a.stop_gradient == b.stop_gradient)
         self.assertTrue(a.place._equals(b.place))
 
+    def test_to_tensor_err_log(self):
+        paddle.disable_static()
+        x = paddle.to_tensor([3])
+        try:
+            a = paddle.jit.to_static(case8)(x)
+        except Exception as e:
+            self.assertTrue(
+                "Do not support transform type `<class 'dict'>` to tensor"
+                in str(e)
+            )
+
 
 class TestStatic(unittest.TestCase):
     def test_static(self):

From 70180df5db12565c440387b0034a71bed6b00acc Mon Sep 17 00:00:00 2001
From: Wilber <jiweibo@baidu.com>
Date: Mon, 8 May 2023 20:34:32 +0800
Subject: [PATCH 221/405] A copy of
 https://github.com/PaddlePaddle/Paddle/pull/51438 (#53587)

---
 .../convert/generic_and_custom_plugin_creater.cc   | 12 ++++++++++--
 .../tensorrt/convert/test_custom_plugin_creater.cc | 14 +++++++-------
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc b/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc
index a889fc9fa3147..da64a8fbb6811 100644
--- a/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc
+++ b/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc
@@ -70,9 +70,17 @@ class CustomPluginCreater : public OpConverter {
     std::list<std::vector<int>> ints_attrs;
     std::list<std::vector<float>> floats_attrs;
 
-    for (auto &attr_name : op_attrs_names) {
+    for (auto &attr_name_and_type : op_attrs_names) {
+      auto attr_name =
+          attr_name_and_type.substr(0, attr_name_and_type.find_first_of(":"));
       nvinfer1::PluginField plugindata;
-      plugindata.name = attr_name.c_str();
+
+      // NOTE: to avoid string rewrite by iterator, deep copy here
+      std::vector<char> plugin_attr_name(attr_name.length() + 1, 0);
+      snprintf(
+          plugin_attr_name.data(), attr_name.length() + 1, attr_name.c_str());
+      plugindata.name = plugin_attr_name.data();
+
       if (op_desc.GetAttrType(attr_name) == framework::proto::AttrType::INT) {
         int_attrs.push_back(PADDLE_GET_CONST(int, attrs.at(attr_name)));
         plugindata.data = &int_attrs.back();
diff --git a/paddle/fluid/inference/tensorrt/convert/test_custom_plugin_creater.cc b/paddle/fluid/inference/tensorrt/convert/test_custom_plugin_creater.cc
index eee4d0c12edbe..ba85108aee539 100644
--- a/paddle/fluid/inference/tensorrt/convert/test_custom_plugin_creater.cc
+++ b/paddle/fluid/inference/tensorrt/convert/test_custom_plugin_creater.cc
@@ -23,13 +23,13 @@ PD_BUILD_OP(custom_op)
     .Inputs({"Input"})
     .Outputs({"Output"})
     .Attrs({
-        "float_attr",
-        "int_attr",
-        "bool_attr",
-        "string_attr",
-        "ints_attr",
-        "floats_attr",
-        "bools_attr",
+        "float_attr: float",
+        "int_attr: int",
+        "bool_attr: bool",
+        "string_attr: std::string",
+        "ints_attr: std::vector<int>",
+        "floats_attr: std::vector<float>",
+        "bools_attr: std::vector<bool>",
     });
 
 namespace paddle {

From ce937f619864507dadc319492957b9375e33d792 Mon Sep 17 00:00:00 2001
From: JamesLim-sy <santamonic@sina.com>
Date: Mon, 8 May 2023 20:56:52 +0800
Subject: [PATCH 222/405] fix op unitest for fused_gate_attn

---
 .../paddle/fluid/tests/unittests/test_fused_gate_attention_op.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py b/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py
index d9bc88d4b2c6d..e88b923f287b0 100644
--- a/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py
+++ b/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py
@@ -274,6 +274,7 @@ def get_fused_gate_attention_out(self):
             _,
             _,
             softmax_out,
+            _,
             fmha_out,
             gate_out,
             out,

From e522ceb7848ac1a7f73bc542b4ee28a1cb3036d6 Mon Sep 17 00:00:00 2001
From: GGBond8488 <33050871+GGBond8488@users.noreply.github.com>
Date: Mon, 8 May 2023 21:28:00 +0800
Subject: [PATCH 223/405] add complex support for  optest (#53356)

* add complex support for  optest

* add complex grad test

* append one

* move some debug info

* move some debug info

* move some debug info

* move some debug info

* add more complex test

* Fix naming ambiguity

* Revert "add more complex test"

This reverts commit dbcb0516b8e53ba42e2d6089878a39b395345969.

* change backward gradient, add TODO
---
 paddle/fluid/pybind/tensor_py.h               |  1 +
 .../kernels/cpu/reduce_mean_grad_kernel.cc    |  4 +-
 paddle/phi/kernels/cpu/reduce_mean_kernel.cc  | 11 ++-
 .../kernels/gpu/reduce_mean_grad_kernel.cu    |  4 +-
 paddle/phi/kernels/kps/reduce_mean_kernel.cu  |  4 +-
 paddle/phi/kernels/reduce_mean_kernel.cc      | 15 ++++-
 .../fluid/tests/unittests/eager_op_test.py    | 67 +++++++++++++++++--
 .../unittests/test_elementwise_mul_op.py      | 49 ++++++++++++++
 .../fluid/tests/unittests/test_reduce_op.py   | 14 ++++
 9 files changed, 157 insertions(+), 12 deletions(-)

diff --git a/paddle/fluid/pybind/tensor_py.h b/paddle/fluid/pybind/tensor_py.h
index 05bacbbf54144..65132bc68fa0d 100644
--- a/paddle/fluid/pybind/tensor_py.h
+++ b/paddle/fluid/pybind/tensor_py.h
@@ -35,6 +35,7 @@ limitations under the License. */
 #include "paddle/fluid/operators/math/concat_and_split.h"
 #include "paddle/fluid/platform/bfloat16.h"
 #include "paddle/fluid/platform/device/device_wrapper.h"
+#include "paddle/fluid/pybind/complex.h"
 #include "paddle/phi/kernels/funcs/strided_memcpy.h"
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 #include "paddle/fluid/platform/cuda_device_guard.h"
diff --git a/paddle/phi/kernels/cpu/reduce_mean_grad_kernel.cc b/paddle/phi/kernels/cpu/reduce_mean_grad_kernel.cc
index b19f6ebdad806..52d13a402301f 100644
--- a/paddle/phi/kernels/cpu/reduce_mean_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/reduce_mean_grad_kernel.cc
@@ -47,4 +47,6 @@ PD_REGISTER_KERNEL(mean_grad,
                    phi::ReduceMeanGradKernel,
                    bool,
                    float,
-                   double) {}
+                   double,
+                   phi::dtype::complex<float>,
+                   phi::dtype::complex<double>) {}
diff --git a/paddle/phi/kernels/cpu/reduce_mean_kernel.cc b/paddle/phi/kernels/cpu/reduce_mean_kernel.cc
index 2ab1b3e5a4739..a8d6723cce6d1 100644
--- a/paddle/phi/kernels/cpu/reduce_mean_kernel.cc
+++ b/paddle/phi/kernels/cpu/reduce_mean_kernel.cc
@@ -36,5 +36,12 @@ void MeanRawKernel(const Context& dev_ctx,
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(
-    mean_raw, CPU, ALL_LAYOUT, phi::MeanRawKernel, float, double, bool) {}
+PD_REGISTER_KERNEL(mean_raw,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::MeanRawKernel,
+                   float,
+                   double,
+                   bool,
+                   phi::dtype::complex<float>,
+                   phi::dtype::complex<double>) {}
diff --git a/paddle/phi/kernels/gpu/reduce_mean_grad_kernel.cu b/paddle/phi/kernels/gpu/reduce_mean_grad_kernel.cu
index 0eac15902af04..13683af9cb9c8 100644
--- a/paddle/phi/kernels/gpu/reduce_mean_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/reduce_mean_grad_kernel.cu
@@ -67,4 +67,6 @@ PD_REGISTER_KERNEL(mean_grad,
                    float,
                    double,
                    phi::dtype::float16,
-                   phi::dtype::bfloat16) {}
+                   phi::dtype::bfloat16,
+                   phi::dtype::complex<float>,
+                   phi::dtype::complex<double>) {}
diff --git a/paddle/phi/kernels/kps/reduce_mean_kernel.cu b/paddle/phi/kernels/kps/reduce_mean_kernel.cu
index dded5972e3088..e62f9d1ff0a36 100644
--- a/paddle/phi/kernels/kps/reduce_mean_kernel.cu
+++ b/paddle/phi/kernels/kps/reduce_mean_kernel.cu
@@ -48,5 +48,7 @@ PD_REGISTER_KERNEL(mean_raw,
                    phi::dtype::bfloat16,
                    float16,
                    int,
-                   int64_t) {}
+                   int64_t,
+                   phi::dtype::complex<float>,
+                   phi::dtype::complex<double>) {}
 #endif
diff --git a/paddle/phi/kernels/reduce_mean_kernel.cc b/paddle/phi/kernels/reduce_mean_kernel.cc
index 10a7c0a2d5c61..9ff9bfc933a48 100644
--- a/paddle/phi/kernels/reduce_mean_kernel.cc
+++ b/paddle/phi/kernels/reduce_mean_kernel.cc
@@ -31,8 +31,15 @@ void MeanKernel(const Context& dev_ctx,
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(
-    mean, CPU, ALL_LAYOUT, phi::MeanKernel, float, double, bool) {}
+PD_REGISTER_KERNEL(mean,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::MeanKernel,
+                   float,
+                   double,
+                   bool,
+                   phi::dtype::complex<float>,
+                   phi::dtype::complex<double>) {}
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 PD_REGISTER_KERNEL(mean,
@@ -45,7 +52,9 @@ PD_REGISTER_KERNEL(mean,
                    int,
                    int64_t,
                    phi::dtype::float16,
-                   phi::dtype::bfloat16) {}
+                   phi::dtype::bfloat16,
+                   phi::dtype::complex<float>,
+                   phi::dtype::complex<double>) {}
 #endif
 
 #if defined(PADDLE_WITH_XPU_KP) && !defined(PADDLE_WITH_XPU)
diff --git a/python/paddle/fluid/tests/unittests/eager_op_test.py b/python/paddle/fluid/tests/unittests/eager_op_test.py
index 1be451d65e83b..469caf4feae87 100644
--- a/python/paddle/fluid/tests/unittests/eager_op_test.py
+++ b/python/paddle/fluid/tests/unittests/eager_op_test.py
@@ -200,6 +200,10 @@ def __get_elem__(tensor, i):
             return tensor._get_float_element(i)
         elif tensor_to_check_dtype == np.float64:
             return tensor._get_double_element(i)
+        elif tensor_to_check_dtype == np.complex64:
+            return tensor._get_complex64_element(i)
+        elif tensor_to_check_dtype == np.complex128:
+            return tensor._get_complex128_element(i)
         else:
             raise TypeError(
                 "Unsupported test data type %s." % tensor_to_check_dtype
@@ -224,6 +228,10 @@ def __set_elem__(tensor, i, e):
             tensor._set_float_element(i, e)
         elif tensor_to_check_dtype == np.float64:
             tensor._set_double_element(i, e)
+        elif tensor_to_check_dtype == np.complex64:
+            return tensor._set_complex64_element(i, e)
+        elif tensor_to_check_dtype == np.complex128:
+            return tensor._set_complex128_element(i, e)
         else:
             raise TypeError(
                 "Unsupported test data type %s." % tensor_to_check_dtype
@@ -242,6 +250,13 @@ def __set_elem__(tensor, i, e):
         __set_elem__(tensor_to_check, i, x_pos)
         y_pos = get_output()
 
+        if tensor_to_check_dtype in [np.complex64, np.complex128]:
+            if in_place:
+                set_input(scope, op, inputs, place)
+            x_pos_j = origin + 1j * delta
+            __set_elem__(tensor_to_check, i, x_pos_j)
+            y_pos_j = get_output()
+
         if in_place:
             set_input(scope, op, inputs, place)
 
@@ -249,8 +264,44 @@ def __set_elem__(tensor, i, e):
         __set_elem__(tensor_to_check, i, x_neg)
         y_neg = get_output()
 
+        if tensor_to_check_dtype in [np.complex64, np.complex128]:
+            if in_place:
+                set_input(scope, op, inputs, place)
+
+            x_neg_j = origin - 1j * delta
+            __set_elem__(tensor_to_check, i, x_neg_j)
+            y_neg_j = get_output()
+
+        __set_elem__(tensor_to_check, i, origin)
+
+        if tensor_to_check_dtype in [np.complex64, np.complex128]:
+            # always assume real output, because this function has
+            # no input for dl/di, though it should do. so there di will be zero
+
+            # TODO: Here is a trick to be consistent with the existing OpTest, it
+            # need to support variable gradients input
+            f_ajoint = np.array(1 + 0j)
+            df_over_dr = (y_pos - y_neg) / delta / 2
+            df_over_di = (y_pos_j - y_neg_j) / delta / 2
+
+            dl_over_du, dl_over_dv = f_ajoint.real, f_ajoint.imag
+
+            du_over_dr, dv_over_dr = df_over_dr.real, df_over_dr.imag
+
+            du_over_di, dv_over_di = df_over_di.real, df_over_di.imag
+
+            dl_over_dr = np.sum(
+                dl_over_du * du_over_dr + dl_over_dv * dv_over_dr
+            )
+            dl_over_di = np.sum(
+                dl_over_du * du_over_di + dl_over_dv * dv_over_di
+            )
+            gradient_flat[i] = dl_over_dr + 1j * dl_over_di
+        else:
+            df_over_dr = y_pos - y_neg
+            gradient_flat[i] = df_over_dr / delta / 2
+
         __set_elem__(tensor_to_check, i, origin)
-        gradient_flat[i] = (y_pos - y_neg) / delta / 2
 
     return gradient_flat.reshape(tensor_to_check.shape())
 
@@ -375,6 +426,13 @@ def is_rocm_op_test():
         def is_custom_device_op_test():
             return hasattr(cls, "use_custom_device") and cls.use_custom_device
 
+        def is_complex_test():
+            return (
+                hasattr(cls, "test_complex")
+                and cls.test_complex
+                or (cls.dtype in [np.complex64, np.complex128])
+            )
+
         if not hasattr(cls, "op_type"):
             raise AssertionError(
                 "This test do not have op_type in class attrs, "
@@ -382,8 +440,10 @@ def is_custom_device_op_test():
             )
 
         # case in NO_FP64_CHECK_GRAD_CASES and op in NO_FP64_CHECK_GRAD_OP_LIST should be fixed
-        if not hasattr(cls, "no_need_check_grad") and not is_empty_grad_op(
-            cls.op_type
+        if (
+            not hasattr(cls, "no_need_check_grad")
+            and not is_empty_grad_op(cls.op_type)
+            and not is_complex_test()
         ):
             if cls.dtype is None or (
                 cls.dtype == np.float16
@@ -2496,7 +2556,6 @@ def check_grad_with_place(
             max_relative_error = (
                 0.001 if max_relative_error < 0.001 else max_relative_error
             )
-
         self._assert_is_close(
             numeric_grads,
             analytic_grads,
diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py
index 4d7ec8f2ed79e..79b9bcba7660f 100644
--- a/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py
+++ b/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py
@@ -93,6 +93,55 @@ def if_enable_cinn(self):
         pass
 
 
+class TestComplexElementwiseMulOpWithCheckGrad(ElementwiseMulOp):
+    def setUp(self):
+        self.op_type = "elementwise_mul"
+        self.python_api = paddle.multiply
+        self.public_python_api = paddle.multiply
+        self.dtype = np.complex128
+        self.axis = -1
+        self.init_dtype()
+        self.init_input_output()
+        self.init_kernel_type()
+        self.init_axis()
+        self.if_enable_cinn()
+
+        self.inputs = {
+            'X': OpTest.np_dtype_to_fluid_dtype(self.x),
+            'Y': OpTest.np_dtype_to_fluid_dtype(self.y),
+        }
+        self.outputs = {'Out': self.out}
+        self.attrs = {'axis': self.axis}
+
+    def init_input_output(self):
+        self.x = np.array([3 + 4j, 1 + 2j]).astype(self.dtype)
+        self.y = np.array([3 + 4j, 5 + 6j]).astype(self.dtype)
+        self.out = np.multiply(self.x, self.y)
+
+    def if_enable_cinn(self):
+        self.enable_cinn = False
+
+    def test_check_grad_normal(self):
+        self.check_grad(
+            ['X', 'Y'],
+            'Out',
+        )
+
+    def test_check_grad_ingore_x(self):
+        self.check_grad(
+            ['Y'],
+            'Out',
+            no_grad_set=set("X"),
+        )
+
+    def test_check_grad_ingore_y(self):
+        self.check_grad(
+            ['X'],
+            'Out',
+            no_grad_set=set('Y'),
+        )
+
+
 class TestElementwiseMulOp_ZeroDim1(ElementwiseMulOp):
     def init_input_output(self):
         self.x = np.random.uniform(0.1, 1, []).astype(self.dtype)
diff --git a/python/paddle/fluid/tests/unittests/test_reduce_op.py b/python/paddle/fluid/tests/unittests/test_reduce_op.py
index 631b760a7b8da..9c22e9c043a88 100644
--- a/python/paddle/fluid/tests/unittests/test_reduce_op.py
+++ b/python/paddle/fluid/tests/unittests/test_reduce_op.py
@@ -57,6 +57,20 @@ def test_check_grad(self):
         self.check_grad(['X'], 'Out', check_prim=True)
 
 
+class TestComplexSumOP(TestSumOp):
+    def init_dtype(self):
+        self.dtype = np.complex128
+
+    def init_input(self):
+        self.x = np.random.random((3, 4)).astype(self.dtype)
+
+    def init_attrs(self):
+        self.attrs = {'dim': [0]}
+
+    def test_check_grad(self):
+        self.check_grad(['X'], 'Out', check_prim=False)
+
+
 class TestSumOp_ZeroDim(TestSumOp):
     def init_attrs(self):
         self.attrs = {'dim': [], 'reduce_all': True}

From e4bf1a8ed97ff1e2808f4f19f18d4f19981fd687 Mon Sep 17 00:00:00 2001
From: zhwesky2010 <1183042833@qq.com>
Date: Tue, 9 May 2023 00:23:02 +0800
Subject: [PATCH 224/405] [Zero-Dim] Support p_norm/reduce_sum_p output 0D
 (#53421)

---
 .../operators/prim_ops/reduce_sum_p_op.cc     |  3 --
 paddle/phi/infermeta/unary.cc                 | 36 ++++++++++---------
 .../operators/dist_reduce_sum_p.py            |  2 +-
 .../tests/unittests/test_pairwise_distance.py |  4 ---
 test/auto_parallel/test_dist_pnorm.py         | 12 ++++++-
 test/auto_parallel/test_prim_dist_op.py       |  3 +-
 test/autograd/test_primops.py                 |  2 +-
 7 files changed, 33 insertions(+), 29 deletions(-)

diff --git a/paddle/fluid/operators/prim_ops/reduce_sum_p_op.cc b/paddle/fluid/operators/prim_ops/reduce_sum_p_op.cc
index b31b4934706a9..89754a7dbfc21 100644
--- a/paddle/fluid/operators/prim_ops/reduce_sum_p_op.cc
+++ b/paddle/fluid/operators/prim_ops/reduce_sum_p_op.cc
@@ -79,9 +79,6 @@ class ReduceSumPrimOpShapeInference : public framework::InferShapeBase {
       x_shape.erase(remove(x_shape.begin(), x_shape.end(), kDelFlag),
                     x_shape.end());
     }
-    if (!keepdim && x_shape.size() == 0) {
-      x_shape.push_back(1);
-    }
 
     PADDLE_GET(framework::VarDesc *, y_var_ptr)->SetShape(x_shape);
   }
diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc
index 1d1d83339829b..42b99dee7cb50 100644
--- a/paddle/phi/infermeta/unary.cc
+++ b/paddle/phi/infermeta/unary.cc
@@ -2750,31 +2750,33 @@ void PNormInferMeta(const MetaTensor& x,
                         x_rank,
                         x_dim));
 
-  std::vector<int> reduce_dims;
+  std::vector<int> out_dim_vector;
   if (asvector) {
-    reduce_dims.emplace_back(1);
     if (keepdim) {
-      for (int i = 1; i < x_dim.size(); ++i) {
-        reduce_dims.emplace_back(1);
+      for (int i = 0; i < x_rank; ++i) {
+        out_dim_vector.emplace_back(1);
       }
-      x_dim = phi::make_ddim(reduce_dims);
+    } else {
+      out_dim_vector = {};
     }
   } else {
-    if (axis < 0) axis = x_dim.size() + axis;
-    for (int i = 0; i < x_dim.size(); ++i) {
-      if (i != axis) reduce_dims.emplace_back(x_dim[i]);
-    }
-    if (reduce_dims.size() == 0) {
-      reduce_dims.emplace_back(1);
+    if (axis < 0) axis = axis + x_rank;
+    if (keepdim) {
+      for (int i = 0; i < x_dim.size(); ++i) {
+        if (i != axis) {
+          out_dim_vector.emplace_back(x_dim[i]);
+        } else {
+          out_dim_vector.emplace_back(1);
+        }
+      }
+    } else {
+      for (int i = 0; i < x_dim.size(); ++i) {
+        if (i != axis) out_dim_vector.emplace_back(x_dim[i]);
+      }
     }
-    x_dim[axis] = 1;
   }
 
-  if (keepdim) {
-    out->set_dims(x_dim);
-  } else {
-    out->set_dims(phi::make_ddim(reduce_dims));
-  }
+  out->set_dims(phi::make_ddim(out_dim_vector));
   out->set_dtype(x.dtype());
 }
 
diff --git a/python/paddle/distributed/auto_parallel/operators/dist_reduce_sum_p.py b/python/paddle/distributed/auto_parallel/operators/dist_reduce_sum_p.py
index 50a4d3466b0fc..919cad3d83e8d 100644
--- a/python/paddle/distributed/auto_parallel/operators/dist_reduce_sum_p.py
+++ b/python/paddle/distributed/auto_parallel/operators/dist_reduce_sum_p.py
@@ -58,7 +58,7 @@ def is_output_compatible(self, dist_op):
 
         output_name = outputs[0]
         output_var = dist_op.serial_op.block._var_recursive(output_name)
-        if output_var.shape != (1,):
+        if output_var.shape != ():
             return False
 
         return True
diff --git a/python/paddle/fluid/tests/unittests/test_pairwise_distance.py b/python/paddle/fluid/tests/unittests/test_pairwise_distance.py
index b8e2c51a1834b..e89a713282db2 100644
--- a/python/paddle/fluid/tests/unittests/test_pairwise_distance.py
+++ b/python/paddle/fluid/tests/unittests/test_pairwise_distance.py
@@ -22,10 +22,6 @@
 
 def np_pairwise_distance(x, y, p=2.0, epsilon=1e-6, keepdim=False):
     distance = np.linalg.norm(x - y + epsilon, ord=p, axis=-1, keepdims=keepdim)
-    # Paddle currently has not supported for 0-d Tensors, so even if keep_dim is False,
-    # and neither x nor y is batched, a Tensor of shape (1, ) is returned
-    if distance.ndim == 0:
-        distance = np.expand_dims(distance, axis=0)
     return distance
 
 
diff --git a/test/auto_parallel/test_dist_pnorm.py b/test/auto_parallel/test_dist_pnorm.py
index deb7c8f715652..5ff30d27b6d6e 100644
--- a/test/auto_parallel/test_dist_pnorm.py
+++ b/test/auto_parallel/test_dist_pnorm.py
@@ -121,9 +121,19 @@ def check_program(self):
             op_dist_attr = self.dist_context.get_op_dist_attr_for_program(op)
             if op.type == "p_norm":
                 assert op_dist_attr.impl_type == "p_norm"
-            if op.type in ["p_norm", "p_norm_grad"]:
                 for input_attr in op_dist_attr.inputs_dist_attrs.values():
                     assert set(input_attr.dims_mapping) == {-1}
+                for output_attr in op_dist_attr.outputs_dist_attrs.values():
+                    if len(output_attr.dims_mapping) == 0:
+                        assert output_attr.dims_mapping == []
+                    else:
+                        assert set(output_attr.dims_mapping) == {-1}
+            if op.type == "p_norm_grad":
+                for input_attr in op_dist_attr.inputs_dist_attrs.values():
+                    if len(input_attr.dims_mapping) == 0:
+                        assert input_attr.dims_mapping == []
+                    else:
+                        assert set(input_attr.dims_mapping) == {-1}
                 for output_attr in op_dist_attr.outputs_dist_attrs.values():
                     assert set(output_attr.dims_mapping) == {-1}
             if op.type == 'c_allgather':
diff --git a/test/auto_parallel/test_prim_dist_op.py b/test/auto_parallel/test_prim_dist_op.py
index 0fc153a3a3346..5a4a1b5a512a3 100644
--- a/test/auto_parallel/test_prim_dist_op.py
+++ b/test/auto_parallel/test_prim_dist_op.py
@@ -55,7 +55,7 @@ def init_prog(self):
         self.tmp1 = paddle.static.data(name='tmp1', shape=[20], dtype='float')
         self.tmp2 = paddle.static.data(name='tmp2', shape=[20], dtype='float')
         self.batch_reduced = paddle.static.data(
-            name='batch_reduced', shape=[1], dtype='float'
+            name='batch_reduced', shape=[], dtype='float'
         )
         self.attrs = {}
 
@@ -108,7 +108,6 @@ def test_loss_and_grad_allreduce(self):
             self.main_program, self.startup_program, [(self.w, self.w_grad)]
         )
         ops = dist_main_prog.global_block().ops
-
         self.assertTrue(ops[1].type == "c_allreduce_sum")
         self.assertTrue(ops[3].type == "c_allreduce_sum")
 
diff --git a/test/autograd/test_primops.py b/test/autograd/test_primops.py
index 65f818fa054c9..f8fa82aea37e1 100644
--- a/test/autograd/test_primops.py
+++ b/test/autograd/test_primops.py
@@ -110,7 +110,7 @@
             primops.reduce_sum,
             randn(2, 3),
             {'axis': (0, 1)},
-            (1,),
+            (),
             'float64',
         ),
         (

From 727fa27d42d7b2f228ce30593b540472bbff4da0 Mon Sep 17 00:00:00 2001
From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com>
Date: Tue, 9 May 2023 10:20:52 +0800
Subject: [PATCH 225/405] remove some [-Wunused-parameter]warning and
 WITH_DISTRIBUTE flag (#53532)

* test,test=develop

* test,test=develop
---
 .../fluid/distributed/ps/table/common_graph_table.h   | 11 ++++++-----
 paddle/fluid/framework/CMakeLists.txt                 |  4 +---
 paddle/phi/backends/onednn/onednn_reuse.h             |  2 +-
 paddle/phi/core/utils/unroll_array_ops.h              |  5 +++--
 paddle/phi/kernels/complex_kernel.h                   |  2 +-
 paddle/phi/kernels/cpu/cross_entropy_grad_kernel.cc   |  2 +-
 paddle/phi/kernels/cpu/full_kernel.cc                 |  6 +++---
 paddle/phi/kernels/cpu/graph_send_recv_funcs.h        |  2 +-
 paddle/phi/kernels/full_kernel.cc                     |  2 +-
 paddle/phi/kernels/funcs/gather_scatter_functor.cc    |  4 ++--
 paddle/phi/kernels/funcs/math_function.cc             |  2 +-
 paddle/phi/kernels/funcs/reduce_functor.h             |  2 +-
 paddle/phi/kernels/funcs/segment_pooling.cc           |  4 ++--
 paddle/phi/kernels/funcs/unique_functor.h             |  2 +-
 paddle/phi/kernels/impl/gumbel_softmax_kernel_impl.h  |  2 +-
 paddle/phi/kernels/impl/lerp_grad_kernel_impl.h       |  4 ++--
 paddle/phi/kernels/reverse_kernel.cc                  |  2 +-
 paddle/phi/kernels/sparse/cpu/full_kernel.cc          |  2 +-
 paddle/phi/kernels/sparse/sparse_utils_kernel.h       |  4 ++--
 19 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/paddle/fluid/distributed/ps/table/common_graph_table.h b/paddle/fluid/distributed/ps/table/common_graph_table.h
index f5288a3f90b12..142327368281a 100644
--- a/paddle/fluid/distributed/ps/table/common_graph_table.h
+++ b/paddle/fluid/distributed/ps/table/common_graph_table.h
@@ -511,7 +511,7 @@ class GraphTable : public Table {
   }
   virtual ~GraphTable();
 
-  virtual void *GetShard(size_t shard_idx) { return 0; }
+  virtual void *GetShard(size_t shard_idx UNUSED) { return 0; }
 
   static int32_t sparse_local_shard_num(uint32_t shard_num,
                                         uint32_t server_num) {
@@ -624,15 +624,16 @@ class GraphTable : public Table {
   Node *find_node(GraphTableType table_type, int idx, uint64_t id);
   Node *find_node(GraphTableType table_type, uint64_t id);
 
-  virtual int32_t Pull(TableContext &context) { return 0; }  // NOLINT
-  virtual int32_t Push(TableContext &context) { return 0; }  // NOLINT
+  virtual int32_t Pull(TableContext &context UNUSED) { return 0; }  // NOLINT
+  virtual int32_t Push(TableContext &context UNUSED) { return 0; }  // NOLINT
 
   virtual int32_t clear_nodes(GraphTableType table_type, int idx);
   virtual void Clear() {}
   virtual int32_t Flush() { return 0; }
-  virtual int32_t Shrink(const std::string &param) { return 0; }
+  virtual int32_t Shrink(const std::string &param UNUSED) { return 0; }
   // 指定保存路径
-  virtual int32_t Save(const std::string &path, const std::string &converter) {
+  virtual int32_t Save(const std::string &path UNUSED,
+                       const std::string &converter UNUSED) {
     return 0;
   }
   virtual int32_t InitializeShard() { return 0; }
diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt
index a96cb146f6833..b95d996b88a94 100755
--- a/paddle/fluid/framework/CMakeLists.txt
+++ b/paddle/fluid/framework/CMakeLists.txt
@@ -928,9 +928,7 @@ if(WITH_DISTRIBUTE)
            fleet_executor)
   endif()
 elseif(WITH_PSLIB)
-  set(DISTRIBUTE_COMPILE_FLAGS
-      "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor"
-  )
+  set(DISTRIBUTE_COMPILE_FLAGS "")
   if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
     set(DISTRIBUTE_COMPILE_FLAGS "${DISTRIBUTE_COMPILE_FLAGS} -faligned-new")
   endif()
diff --git a/paddle/phi/backends/onednn/onednn_reuse.h b/paddle/phi/backends/onednn/onednn_reuse.h
index c7dcdea0e487c..463c55a7c5317 100644
--- a/paddle/phi/backends/onednn/onednn_reuse.h
+++ b/paddle/phi/backends/onednn/onednn_reuse.h
@@ -1178,7 +1178,7 @@ class ReductionOneDNNHandler
                          const dnnl::engine engine,
                          Place cpu_place,
                          const DenseTensor* x,
-                         const DenseTensor* out,
+                         const DenseTensor* out UNUSED,
                          std::vector<int64_t> out_tz,
                          const dnnl::primitive_attr& attrs = NULL)
       : OneDNNHandlerNoCachingT<T, dnnl::reduction>(engine, cpu_place) {
diff --git a/paddle/phi/core/utils/unroll_array_ops.h b/paddle/phi/core/utils/unroll_array_ops.h
index 2e1d84080af48..665c2ae512829 100644
--- a/paddle/phi/core/utils/unroll_array_ops.h
+++ b/paddle/phi/core/utils/unroll_array_ops.h
@@ -87,7 +87,8 @@ struct UnrollCompare {
 template <size_t kStart, size_t kEnd>
 struct UnrollCompare<kStart, kEnd, true> {
   template <typename T>
-  HOSTDEVICE inline constexpr static bool Run(const T *d1, const T *d2) {
+  HOSTDEVICE inline constexpr static bool Run(const T *d1 UNUSED,
+                                              const T *d2 UNUSED) {
     return true;
   }
 };
@@ -104,7 +105,7 @@ struct UnrollProduct {
 template <size_t kStart, size_t kEnd>
 struct UnrollProduct<kStart, kEnd, true> {
   template <typename T>
-  HOSTDEVICE inline constexpr static T Run(const T *d) {
+  HOSTDEVICE inline constexpr static T Run(const T *d UNUSED) {
     return 1;
   }
 };
diff --git a/paddle/phi/kernels/complex_kernel.h b/paddle/phi/kernels/complex_kernel.h
index ad66b890b3d5a..4f1a1c40402de 100644
--- a/paddle/phi/kernels/complex_kernel.h
+++ b/paddle/phi/kernels/complex_kernel.h
@@ -58,7 +58,7 @@ template <
     std::enable_if_t<!std::is_same<T, phi::dtype::complex<float>>::value &&
                          !std::is_same<T, phi::dtype::complex<double>>::value,
                      bool> = true>
-DenseTensor Conj(const Context& dev_ctx, const DenseTensor& x) {
+DenseTensor Conj(const Context& dev_ctx UNUSED, const DenseTensor& x) {
   return x;
 }
 
diff --git a/paddle/phi/kernels/cpu/cross_entropy_grad_kernel.cc b/paddle/phi/kernels/cpu/cross_entropy_grad_kernel.cc
index 1499f01002b6b..eacf1ad0cf3ad 100644
--- a/paddle/phi/kernels/cpu/cross_entropy_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/cross_entropy_grad_kernel.cc
@@ -30,7 +30,7 @@ void CrossEntropyWithSoftmaxGradCPUKernel(const CPUContext& dev_ctx,
                                           const DenseTensor& loss_grad,
                                           bool soft_label,
                                           bool use_softmax,
-                                          bool numeric_stable_mode,
+                                          bool numeric_stable_mode UNUSED,
                                           int ignore_index,
                                           int axis,
                                           DenseTensor* logits_grad) {
diff --git a/paddle/phi/kernels/cpu/full_kernel.cc b/paddle/phi/kernels/cpu/full_kernel.cc
index d9ab771664a8f..a295bfcc20c3a 100644
--- a/paddle/phi/kernels/cpu/full_kernel.cc
+++ b/paddle/phi/kernels/cpu/full_kernel.cc
@@ -32,7 +32,7 @@ template <typename T, typename Context>
 void FullKernel(const Context& dev_ctx,
                 const IntArray& shape,
                 const Scalar& val,
-                DataType dtype,
+                DataType dtype UNUSED,
                 DenseTensor* out) {
   out->Resize(phi::make_ddim(shape.GetData()));
   FullValue<T>(dev_ctx, out, val.to<T>());
@@ -40,9 +40,9 @@ void FullKernel(const Context& dev_ctx,
 
 template <typename T, typename Context>
 void FullLikeKernel(const Context& dev_ctx,
-                    const DenseTensor& x,
+                    const DenseTensor& x UNUSED,
                     const Scalar& val,
-                    DataType dtype,
+                    DataType dtype UNUSED,
                     DenseTensor* out) {
   auto value = val.to<double>();
   using CommonType = typename std::common_type<
diff --git a/paddle/phi/kernels/cpu/graph_send_recv_funcs.h b/paddle/phi/kernels/cpu/graph_send_recv_funcs.h
index df6d9c87be0ed..c67480cc9e33e 100644
--- a/paddle/phi/kernels/cpu/graph_send_recv_funcs.h
+++ b/paddle/phi/kernels/cpu/graph_send_recv_funcs.h
@@ -25,7 +25,7 @@ namespace phi {
 
 template <typename T>
 struct GraphSendRecvSumFunctor {
-  void operator()(const bool& first_flag,
+  void operator()(const bool& first_flag UNUSED,
                   const DenseTensor& src_slice,
                   DenseTensor* dst_slice) {
     auto eigen_src = phi::EigenVector<T>::Flatten(src_slice);
diff --git a/paddle/phi/kernels/full_kernel.cc b/paddle/phi/kernels/full_kernel.cc
index ce898210633b7..38beafbfa51b9 100644
--- a/paddle/phi/kernels/full_kernel.cc
+++ b/paddle/phi/kernels/full_kernel.cc
@@ -21,7 +21,7 @@ namespace phi {
 template <typename T, typename Context>
 void FullBatchSizeLikeKernel(const Context& dev_ctx,
                              const DenseTensor& x,
-                             const std::vector<int>& shape,
+                             const std::vector<int>& shape UNUSED,
                              const Scalar& val,
                              DataType dtype,
                              int x_batch_size_dim,
diff --git a/paddle/phi/kernels/funcs/gather_scatter_functor.cc b/paddle/phi/kernels/funcs/gather_scatter_functor.cc
index 57a8d679f346f..e88dbf0f7ccdb 100644
--- a/paddle/phi/kernels/funcs/gather_scatter_functor.cc
+++ b/paddle/phi/kernels/funcs/gather_scatter_functor.cc
@@ -181,11 +181,11 @@ void cpu_scatter_mul_kernel(phi::DenseTensor self,
 }
 
 template <typename tensor_t, typename index_t>
-void cpu_scatter_input_grad_kernel(phi::DenseTensor self,
+void cpu_scatter_input_grad_kernel(phi::DenseTensor self UNUSED,
                                    int dim,
                                    const phi::DenseTensor& index,
                                    phi::DenseTensor output,
-                                   const phi::DeviceContext& ctx) {
+                                   const phi::DeviceContext& ctx UNUSED) {
   auto* index_data = index.data<index_t>();
   auto* output_data = output.data<tensor_t>();
 
diff --git a/paddle/phi/kernels/funcs/math_function.cc b/paddle/phi/kernels/funcs/math_function.cc
index d13ce8a2a90b5..1a61f48579188 100644
--- a/paddle/phi/kernels/funcs/math_function.cc
+++ b/paddle/phi/kernels/funcs/math_function.cc
@@ -92,7 +92,7 @@ DEFINE_CPU_TRANS(6);
 
 template <typename DeviceContext, typename T>
 void TransposeNormal<DeviceContext, T>::operator()(
-    const DeviceContext& context,
+    const DeviceContext& context UNUSED,
     const phi::DenseTensor& in,
     phi::DenseTensor* out,
     const std::vector<int>& axis) {
diff --git a/paddle/phi/kernels/funcs/reduce_functor.h b/paddle/phi/kernels/funcs/reduce_functor.h
index 80217520b126f..e5e440d64d541 100644
--- a/paddle/phi/kernels/funcs/reduce_functor.h
+++ b/paddle/phi/kernels/funcs/reduce_functor.h
@@ -41,7 +41,7 @@ struct FrobeniusNormGradFunctor {
                   DX* dx,
                   DY* dy,
                   const Dim& dim,
-                  int size) {
+                  int size UNUSED) {
     dx->device(place) = y->broadcast(dim);
     dx->device(place) = *dx + dx->constant(1e-12f);
     dx->device(place) = (*x / *dx) * (dy->broadcast(dim));
diff --git a/paddle/phi/kernels/funcs/segment_pooling.cc b/paddle/phi/kernels/funcs/segment_pooling.cc
index c5d0944d8c190..9dd800aba1a7b 100644
--- a/paddle/phi/kernels/funcs/segment_pooling.cc
+++ b/paddle/phi/kernels/funcs/segment_pooling.cc
@@ -31,7 +31,7 @@ class SegmentPoolFunctor<phi::CPUContext, T, IndexT> {
                   const DenseTensor& input,
                   const DenseTensor& segments,
                   DenseTensor* output,
-                  DenseTensor* index,
+                  DenseTensor* index UNUSED,
                   const std::string pooltype = "SUM") {
     const IndexT* segment_ids = segments.data<IndexT>();
     auto curent_id = segment_ids[0];
@@ -90,7 +90,7 @@ class SegmentPoolGradFunctor<phi::CPUContext, T, IndexT> {
                   const DenseTensor& out_grad,
                   const DenseTensor& segments,
                   DenseTensor* in_grad,
-                  const paddle::optional<DenseTensor>& index,
+                  const paddle::optional<DenseTensor>& index UNUSED,
                   const std::string pooltype = "SUM") {
     const IndexT* segment_ids = segments.data<IndexT>();
     auto& place = *dev_ctx.eigen_device();
diff --git a/paddle/phi/kernels/funcs/unique_functor.h b/paddle/phi/kernels/funcs/unique_functor.h
index d704d2d60fa8d..806d7cca84851 100644
--- a/paddle/phi/kernels/funcs/unique_functor.h
+++ b/paddle/phi/kernels/funcs/unique_functor.h
@@ -190,7 +190,7 @@ static void UniqueFlattendTensor(const Context& context,
 }
 
 template <typename Context, typename ForwardIt, typename InT, typename IndexT>
-static ForwardIt UniqueDimImpl(const Context& context,
+static ForwardIt UniqueDimImpl(const Context& context UNUSED,
                                ForwardIt first,
                                ForwardIt last,
                                const std::vector<IndexT>& sorted_indices_vec,
diff --git a/paddle/phi/kernels/impl/gumbel_softmax_kernel_impl.h b/paddle/phi/kernels/impl/gumbel_softmax_kernel_impl.h
index c2229b50deee1..184d149110e9b 100644
--- a/paddle/phi/kernels/impl/gumbel_softmax_kernel_impl.h
+++ b/paddle/phi/kernels/impl/gumbel_softmax_kernel_impl.h
@@ -27,7 +27,7 @@ namespace phi {
 
 template <typename Context, typename T, int64_t Rank>
 struct ArgMaxFunctor {
-  void operator()(const Context& ctx,
+  void operator()(const Context& ctx UNUSED,
                   const DenseTensor& in,
                   DenseTensor* index_tensor,
                   const int64_t& axis) {
diff --git a/paddle/phi/kernels/impl/lerp_grad_kernel_impl.h b/paddle/phi/kernels/impl/lerp_grad_kernel_impl.h
index 541de0cc162cc..e9c54c7aca81a 100644
--- a/paddle/phi/kernels/impl/lerp_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/lerp_grad_kernel_impl.h
@@ -21,8 +21,8 @@ namespace phi {
 
 template <typename Context, typename T, size_t D>
 static void LerpGradFunction(const Context& ctx,
-                             const DenseTensor& x,
-                             const DenseTensor& y,
+                             const DenseTensor& x UNUSED,
+                             const DenseTensor& y UNUSED,
                              const DenseTensor& weight,
                              const DenseTensor& out,
                              const DenseTensor& out_grad,
diff --git a/paddle/phi/kernels/reverse_kernel.cc b/paddle/phi/kernels/reverse_kernel.cc
index b2fe61ad41fc6..771acacedf024 100644
--- a/paddle/phi/kernels/reverse_kernel.cc
+++ b/paddle/phi/kernels/reverse_kernel.cc
@@ -23,7 +23,7 @@ namespace phi {
 template <typename T, typename Context>
 void ReverseArrayKernel(const Context& dev_ctx,
                         const TensorArray& x,
-                        const IntArray& axis,
+                        const IntArray& axis UNUSED,
                         TensorArray* out) {
   PADDLE_ENFORCE_EQ(
       x.size(),
diff --git a/paddle/phi/kernels/sparse/cpu/full_kernel.cc b/paddle/phi/kernels/sparse/cpu/full_kernel.cc
index ac13327caeeaa..5659bcf2159da 100644
--- a/paddle/phi/kernels/sparse/cpu/full_kernel.cc
+++ b/paddle/phi/kernels/sparse/cpu/full_kernel.cc
@@ -34,7 +34,7 @@ template <typename T, typename Context>
 void FullLikeCooKernel(const Context& dev_ctx,
                        const SparseCooTensor& x,
                        const Scalar& val,
-                       DataType dtype,
+                       DataType dtype UNUSED,
                        SparseCooTensor* out) {
   phi::Copy<Context>(dev_ctx,
                      x.non_zero_indices(),
diff --git a/paddle/phi/kernels/sparse/sparse_utils_kernel.h b/paddle/phi/kernels/sparse/sparse_utils_kernel.h
index 8639f91469454..241f3d8b0670a 100644
--- a/paddle/phi/kernels/sparse/sparse_utils_kernel.h
+++ b/paddle/phi/kernels/sparse/sparse_utils_kernel.h
@@ -144,14 +144,14 @@ DenseTensor CsrToDense(const Context& dev_ctx, const SparseCsrTensor& x) {
 }
 
 template <typename T, typename Context>
-void ValuesCooKernel(const Context& dev_ctx,
+void ValuesCooKernel(const Context& dev_ctx UNUSED,
                      const SparseCooTensor& x,
                      DenseTensor* out) {
   *out = x.non_zero_elements();
 }
 
 template <typename T, typename Context>
-void ValuesCsrKernel(const Context& dev_ctx,
+void ValuesCsrKernel(const Context& dev_ctx UNUSED,
                      const SparseCsrTensor& x,
                      DenseTensor* out) {
   *out = x.non_zero_elements();

From 8d340ee1a542229dd6926b63b37094411c01e8aa Mon Sep 17 00:00:00 2001
From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com>
Date: Tue, 9 May 2023 11:14:27 +0800
Subject: [PATCH 226/405] Fix xpu2 kp compile error (#53548)

---
 paddle/phi/kernels/funcs/broadcast_function.h | 38 ++++++++++--
 .../primitive/datamover_primitives_xpu2.h     | 59 +++++++++++++++++++
 2 files changed, 91 insertions(+), 6 deletions(-)

diff --git a/paddle/phi/kernels/funcs/broadcast_function.h b/paddle/phi/kernels/funcs/broadcast_function.h
index d0f3fba392a86..e754ce3bf49e4 100644
--- a/paddle/phi/kernels/funcs/broadcast_function.h
+++ b/paddle/phi/kernels/funcs/broadcast_function.h
@@ -110,10 +110,29 @@ struct BroadcastDataLoader {
                                                const Array3 &use_broadcast,
                                                const int block_offset,
                                                const int num,
-                                               const uint32_t numel) {
+                                               const uint32_t numel,
+                                               int read_lens) {
     using Type = std::tuple_element_t<Index, ArgsT>;
+#ifdef PADDLE_WITH_XPU_KP
+    kps::Init<Type, ArgsT, Index, VecSize>(
+        args, static_cast<Type>(1.0f), read_lens);
+    if (use_broadcast[Index]) {
+      kps::ReadDataBc<Type, VecSize, 1, ArgsT, Index, IsBoundary>(
+          args,
+          reinterpret_cast<const _ptr_ Type *>(ins[Index]),
+          block_offset,
+          configs[Index],
+          numel,
+          read_lens);
+    } else {
+      kps::ReadData<Type, VecSize, 1, ArgsT, Index, IsBoundary>(
+          args,
+          reinterpret_cast<const _ptr_ Type *>(ins[Index]) + block_offset,
+          num,
+          read_lens);
+    }
+#else
     kps::Init<Type, ArgsT, Index, VecSize>(args, static_cast<Type>(1.0f));
-
     if (use_broadcast[Index]) {
       kps::ReadDataBc<Type, VecSize, 1, ArgsT, Index, IsBoundary>(
           args,
@@ -133,6 +152,7 @@ struct BroadcastDataLoader {
           num,
           VecSize);
     }
+#endif
   }
 };
 
@@ -148,7 +168,8 @@ struct BroadcastDataLoader<Index, VecSize, true, kElementwise> {
                                                const Array3 &use_broadcast,
                                                const int block_offset,
                                                const int num,
-                                               const uint32_t numel) {
+                                               const uint32_t numel,
+                                               int read_lens) {
     using Type = std::tuple_element_t<Index, ArgsT>;
     int thread_offset = threadIdx.x * VecSize + block_offset;
 #pragma unroll
@@ -173,7 +194,8 @@ struct BroadcastDataLoader<Index, VecSize, false, kElementwise> {
                                                const Array3 &use_broadcast,
                                                const int block_offset,
                                                const int num,
-                                               const uint32_t numel) {
+                                               const uint32_t numel,
+                                               int read_lens) {
     using Type = std::tuple_element_t<Index, ArgsT>;
     using VecType = phi::kps::details::VectorType<Type, VecSize>;
     VecType vec_temp;
@@ -269,6 +291,10 @@ __device__ void VectorizedBroadcastKernelImpl(
   __simd__ ArgsT args[VecSize];
   __simd__ ConditionalT<OutT, NumOuts> result[VecSize];
 
+#ifdef PADDLE_WITH_XPU_KP
+  BcUnroller<BroadcastDataLoader, IsBoundary, LoadType, VecSize, Arity>::step(
+      ins, args, configs, use_broadcast, block_offset, num, numel, read_lens);
+#else
   if (LoadType == kBroadcast) {
     uint32_t index_bc[Arity][VecSize] = {0};
     Unroller<BroadcastDataInit, VecSize, Arity>::step(args);
@@ -291,9 +317,9 @@ __device__ void VectorizedBroadcastKernelImpl(
     Unroller<BroadcastDataSetter, VecSize, Arity>::step(ins, args, index_bc);
   } else {
     BcUnroller<BroadcastDataLoader, IsBoundary, LoadType, VecSize, Arity>::step(
-        ins, args, configs, use_broadcast, block_offset, num, numel);
+        ins, args, configs, use_broadcast, block_offset, num, numel, read_lens);
   }
-
+#endif
   SameDimsElementwisePrimitiveCaller<ConditionalT<OutT, NumOuts>,
                                      VecSize,
                                      Functor,
diff --git a/paddle/phi/kernels/primitive/datamover_primitives_xpu2.h b/paddle/phi/kernels/primitive/datamover_primitives_xpu2.h
index 14a66516f5b63..7d60b573e2775 100644
--- a/paddle/phi/kernels/primitive/datamover_primitives_xpu2.h
+++ b/paddle/phi/kernels/primitive/datamover_primitives_xpu2.h
@@ -1211,6 +1211,65 @@ __device__ __inline__ void ReadDataBc(T* dst,
   }
 }
 
+/**
+ * @brief Read 1D data from global memory to register with broadcast form.
+ * The difference from the above function is that it supports different data
+ * types of inputs.
+ * @template paraments
+ * T: The type of data stored in the global memory.
+ * NX: The number of data continuously loaded by each thread.
+ * NY: The number of data rows loaded by each thread, only NY = 1 was supported.
+ * core_id() is used as the index.
+ * IsBoundary: Indicates whether to perform block access storage out-of-bounds
+ * judgment. When the number of data processed by the block is less than
+ * NX x NY x core_num(), boundary judgment is required to avoid memory access
+ * crossing the boundary.
+ *
+ * @param：
+ * dst: The register pointer of the thread, the size is NX * NY.
+ * src: The original input data pointer of kernel.
+ * block_offset: The data offset of this block, core_num() * blockIdx.x * NX;
+ * config: Calculation configuration of broadcast. It is used to calculate the
+ * coordinate mapping relationship between output data and input data.
+ * read_lens: The number of data continuously loaded by each thread.
+ * total_num_output: Total number of original output.
+ */
+template <typename T,
+          int NX,
+          int NY,
+          typename ArgsT,
+          int Index,
+          bool IsBoundary = false>
+__device__ __forceinline__ void ReadDataBc(
+    ArgsT* dst,
+    const T _global_ptr_* src,
+    int block_offset,
+    const details::BroadcastConfig& config,
+    int total_num_output,
+    int read_lens = NX) {
+  int thread_offset = block_offset + core_id() * read_lens;
+  __local__ T in_temp[NX];
+
+  if (config.cmp_type == details::OptType::MNK_M1K) {
+    ReadDataBcM1kMnk<T>(in_temp, src, thread_offset, config, read_lens);
+  } else if (config.cmp_type == details::OptType::N_1) {
+    ReadDataBc1N<T>(in_temp, src, thread_offset, config, read_lens);
+  } else if (config.cmp_type == details::OptType::MN_M) {
+    ReadDataBcM1Mn<T>(in_temp, src, thread_offset, config, read_lens);
+  } else if (config.cmp_type == details::OptType::MN_N) {
+    ReadDataBc1NMn<T>(in_temp, src, thread_offset, config, read_lens);
+  } else if (config.cmp_type == details::OptType::MNK_1N1) {
+    ReadDataBc1N1Mnk<T>(in_temp, src, thread_offset, config, read_lens);
+  } else {
+    ReadDataBcCanNotCmp<T, IsBoundary>(
+        in_temp, src, thread_offset, config, total_num_output, read_lens);
+  }
+#pragma unroll
+  for (int idx = 0; idx < read_lens; ++idx) {
+    std::get<Index>(dst[idx]) = in_temp[idx];
+  }
+}
+
 /**
  * @brief Initialize register with data index.
  *

From af2ad8d830b0d422b69d1876b6b0cd0361059777 Mon Sep 17 00:00:00 2001
From: RedContritio <RedContritio@qq.com>
Date: Tue, 9 May 2023 11:21:55 +0800
Subject: [PATCH 227/405] fix error sample code in static.nn.loss.nce (#53588)

---
 python/paddle/static/nn/loss.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/python/paddle/static/nn/loss.py b/python/paddle/static/nn/loss.py
index e0a41c6ec657a..7d7912a06fbdd 100644
--- a/python/paddle/static/nn/loss.py
+++ b/python/paddle/static/nn/loss.py
@@ -110,7 +110,8 @@ def nce(
                                     param_attr='embed', is_sparse=True)
                 embs.append(emb)
 
-            embs = paddle.concat(x=embs, axis=1)
+            embs = paddle.concat(x=embs, axis=1)                # concat from 4 * [(-1, 1, 32)] to (-1, 4, 32)
+            embs = paddle.reshape(x=embs, shape=(-1, 4 * 32))   # reshape to (batch_size = -1, dim = 4*32)
             loss = paddle.static.nn.nce(input=embs, label=words[label_word],
                         num_total_classes=dict_size, param_attr='nce.w_0',
                         bias_attr='nce.b_0')

From aec4e38fc321a50b6a273f3642a7cb61db9b35e6 Mon Sep 17 00:00:00 2001
From: zhoutianzi666 <39978853+zhoutianzi666@users.noreply.github.com>
Date: Tue, 9 May 2023 11:23:13 +0800
Subject: [PATCH 228/405] [Paddle-TRT] Del 2 useless pass (#53414)

* delete delete_fill_constant_op_pass and unsqueeze2_eltwise_fuse_pass
---
 paddle/fluid/framework/ir/CMakeLists.txt      |   6 -
 .../ir/delete_fill_constant_op_pass.cc        | 115 -----------
 .../ir/delete_fill_constant_op_pass.h         |  39 ----
 .../ir/unsqueeze2_eltwise_fuse_pass.cc        | 190 ------------------
 .../ir/unsqueeze2_eltwise_fuse_pass.h         |  46 -----
 .../ir/unsqueeze2_eltwise_fuse_pass_tester.cc |  67 ------
 .../inference/api/paddle_pass_builder.cc      |   2 -
 .../test_unsqueeze2_eltwise_fuse_pass.py      | 105 ----------
 tools/parallel_UT_rule.py                     |   2 -
 tools/windows/run_unittests.sh                |   1 -
 10 files changed, 573 deletions(-)
 delete mode 100644 paddle/fluid/framework/ir/delete_fill_constant_op_pass.cc
 delete mode 100644 paddle/fluid/framework/ir/delete_fill_constant_op_pass.h
 delete mode 100644 paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass.cc
 delete mode 100644 paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass.h
 delete mode 100644 paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass_tester.cc
 delete mode 100644 test/ir/inference/test_unsqueeze2_eltwise_fuse_pass.py

diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt
index 46bac61bda13b..5eb94dbbce435 100755
--- a/paddle/fluid/framework/ir/CMakeLists.txt
+++ b/paddle/fluid/framework/ir/CMakeLists.txt
@@ -104,7 +104,6 @@ pass_library(delete_dropout_op_pass inference)
 pass_library(delete_concat_op_pass inference)
 pass_library(delete_c_identity_op_pass inference)
 pass_library(preln_residual_bias_fuse_pass inference)
-pass_library(delete_fill_constant_op_pass inference)
 pass_library(constant_folding_pass inference)
 pass_library(auto_mixed_precision_pass inference)
 pass_library(conv2d_fusion_layout_transfer_pass inference)
@@ -118,7 +117,6 @@ pass_library(fused_multi_transformer_encoder_pass inference)
 pass_library(fused_multi_transformer_decoder_pass inference)
 pass_library(fuse_multi_transformer_layer_pass inference)
 pass_library(adaptive_pool2d_convert_global_pass inference)
-pass_library(unsqueeze2_eltwise_fuse_pass inference)
 pass_library(yolo_box_fuse_pass inference)
 pass_library(layer_norm_fuse_pass inference)
 pass_library(add_support_int8_pass inference)
@@ -391,10 +389,6 @@ cc_test(
   test_adaptive_pool2d_convert_global_pass
   SRCS adaptive_pool2d_convert_global_pass_tester.cc
   DEPS adaptive_pool2d_convert_global_pass)
-cc_test(
-  test_unsqueeze2_eltwise_fuse_pass_cc
-  SRCS unsqueeze2_eltwise_fuse_pass_tester.cc
-  DEPS unsqueeze2_eltwise_fuse_pass)
 cc_test(
   test_generate_pass_cc
   SRCS generate_pass_tester.cc
diff --git a/paddle/fluid/framework/ir/delete_fill_constant_op_pass.cc b/paddle/fluid/framework/ir/delete_fill_constant_op_pass.cc
deleted file mode 100644
index baae2f74e09aa..0000000000000
--- a/paddle/fluid/framework/ir/delete_fill_constant_op_pass.cc
+++ /dev/null
@@ -1,115 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/fluid/framework/ir/delete_fill_constant_op_pass.h"
-
-#include "paddle/fluid/framework/ir/graph_pattern_detector.h"
-
-namespace paddle {
-namespace framework {
-namespace ir {
-
-template <typename T>
-void FillConstData(phi::DenseTensor* out_t, T value) {
-  auto output_data = out_t->mutable_data<T>(platform::CPUPlace());
-  for (int i = 0; i < out_t->numel(); i++) {
-    output_data[i] = value;
-  }
-}
-
-void DeleteFillConstantOpPass::ApplyImpl(ir::Graph* graph) const {
-  bool with_dynamic_shape = Get<bool>("with_dynamic_shape");
-  // Not support
-  if (with_dynamic_shape) {
-    return;
-  }
-  FusePassBase::Init("delete_fill_constant_op_pass", graph);
-  GraphPatternDetector detector;
-  auto fill_constant_op =
-      detector.mutable_pattern()
-          ->NewNode("fill_constant")
-          ->assert_is_op("fill_constant")
-          ->assert_is_not_op_input("ValueTensor")
-          ->assert_is_not_op_input("str_value")
-          ->assert_is_not_op_input("ShapeTensor")
-          ->assert_is_not_op_input("ShapeTensorList")
-          ->assert_more([&](Node* node) {
-            return node->Op()
-                       ->GetAttrIfExists<std::vector<int64_t>>("shape")
-                       .size() == 1;
-          });
-  auto fill_constant_out =
-      detector.mutable_pattern()
-          ->NewNode("fill_constant_out")
-          ->assert_is_op_output("fill_constant")
-          ->assert_more([](Node* x) { return x->outputs.size() == 1UL; });
-  auto next_op = detector.mutable_pattern()
-                     ->NewNode("next_op")
-                     ->assert_is_not_op_type("conditional_block")
-                     ->assert_is_not_op_type("while");
-  // Create the topological connections for the above pattern nodes.
-  fill_constant_op->LinksTo({fill_constant_out});
-  next_op->LinksFrom({fill_constant_out});
-
-  auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph,
-                     Graph* graph) {
-    Node* fill_constant_op_node = subgraph.at(fill_constant_op);
-    Node* fill_constant_out_node = subgraph.at(fill_constant_out);
-    // Get fill_constant's attr
-    auto fill_constant = fill_constant_op_node->Op();
-    auto value = PADDLE_GET_CONST(float, fill_constant->GetAttr("value"));
-    auto shape =
-        PADDLE_GET_CONST(std::vector<int64_t>, fill_constant->GetAttr("shape"));
-    auto* scope = param_scope();
-    auto fill_constant_out_desc = fill_constant_out_node->Var();
-    fill_constant_out_desc->SetShape(shape);
-    fill_constant_out_desc->SetPersistable(true);
-    auto* fill_constant_out_tensor = scope->Var(fill_constant_out_desc->Name())
-                                         ->GetMutable<phi::DenseTensor>();
-    auto dtype =
-        framework::TransToPhiDataType(fill_constant_out_desc->GetDataType());
-    fill_constant_out_tensor->Resize(phi::make_ddim(shape));
-    switch (dtype) {
-      case phi::DataType::BOOL:
-        FillConstData<bool>(fill_constant_out_tensor, static_cast<bool>(value));
-        break;
-      case phi::DataType::INT32:
-        FillConstData<int32_t>(fill_constant_out_tensor,
-                               static_cast<int32_t>(value));
-        break;
-      case phi::DataType::INT64:
-        FillConstData<int64_t>(fill_constant_out_tensor,
-                               static_cast<int64_t>(value));
-        break;
-      case phi::DataType::FLOAT32:
-        FillConstData<float>(fill_constant_out_tensor,
-                             static_cast<float>(value));
-        break;
-      default:
-        LOG(WARNING) << "Unsupported dtype for fill_constant op: " << dtype;
-        return;
-    }
-    // Remove links in graph
-    GraphSafeRemoveNodes(graph, {fill_constant_op_node});
-  };
-
-  detector(graph, handler);
-}
-
-}  // namespace ir
-}  // namespace framework
-}  // namespace paddle
-
-REGISTER_PASS(delete_fill_constant_op_pass,
-              paddle::framework::ir::DeleteFillConstantOpPass);
diff --git a/paddle/fluid/framework/ir/delete_fill_constant_op_pass.h b/paddle/fluid/framework/ir/delete_fill_constant_op_pass.h
deleted file mode 100644
index 33d10f4502f2a..0000000000000
--- a/paddle/fluid/framework/ir/delete_fill_constant_op_pass.h
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#pragma once
-#include <vector>
-
-#include "paddle/fluid/framework/convert_utils.h"
-#include "paddle/fluid/framework/ir/fuse_pass_base.h"
-#include "paddle/fluid/framework/ir/graph_pattern_detector.h"
-#include "paddle/fluid/platform/enforce.h"
-
-namespace paddle {
-namespace framework {
-namespace ir {
-
-class Graph;
-
-class DeleteFillConstantOpPass : public FusePassBase {
- protected:
-  void ApplyImpl(ir::Graph* graph) const override;
-
- private:
-  virtual ~DeleteFillConstantOpPass() = default;
-};
-
-}  // namespace ir
-}  // namespace framework
-}  // namespace paddle
diff --git a/paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass.cc b/paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass.cc
deleted file mode 100644
index 85d709c20db21..0000000000000
--- a/paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass.cc
+++ /dev/null
@@ -1,190 +0,0 @@
-/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include "paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass.h"
-
-#include <string>
-
-#include "glog/logging.h"
-#include "paddle/fluid/framework/ir/graph_pattern_detector.h"
-#include "paddle/fluid/framework/ir/pass.h"
-#include "paddle/fluid/framework/op_version_registry.h"
-#include "paddle/fluid/platform/enforce.h"
-
-namespace paddle {
-namespace framework {
-namespace ir {
-class Node;
-}  // namespace ir
-}  // namespace framework
-}  // namespace paddle
-
-namespace paddle {
-namespace framework {
-namespace ir {
-namespace patterns {
-
-struct UnsqueezeEltwise : public PatternBase {
-  UnsqueezeEltwise(PDPattern *pattern, const std::string &name_scope)
-      : PatternBase(pattern, name_scope, "unsqueeze2_eltwise_fuse_pass") {}
-
-  PDNode *operator()(PDNode *x, PDNode *y);
-
-  // declare operator node's name
-  PATTERN_DECL_NODE(unsqz);
-  PATTERN_DECL_NODE(elementwise);
-  // declare variable node's name
-  PATTERN_DECL_NODE(eltwise_in_x);
-  PATTERN_DECL_NODE(unsqz_in);
-  PATTERN_DECL_NODE(unsqz_out);
-  PATTERN_DECL_NODE(eltwise_out);
-};
-
-PDNode *UnsqueezeEltwise::operator()(PDNode *x, PDNode *y) {
-  x->assert_is_op_input("elementwise_mul", "X");
-  y->assert_is_op_input("unsqueeze2", "X");
-
-  auto *unsqz = pattern->NewNode(unsqz_repr())->assert_is_op("unsqueeze2");
-  auto *unsqz_out = pattern->NewNode(unsqz_out_repr())
-                        ->assert_is_op_output("unsqueeze2", "Out")
-                        ->assert_is_op_input("elementwise_mul", "Y");
-  unsqz->LinksFrom({y}).LinksTo({unsqz_out});
-
-  auto *elementwise =
-      pattern->NewNode(elementwise_repr())->assert_is_op("elementwise_mul");
-  auto *eltwise_out = pattern->NewNode(eltwise_out_repr())
-                          ->AsOutput()
-                          ->assert_is_op_output("elementwise_mul");
-
-  elementwise->LinksFrom({x, unsqz_out}).LinksTo({eltwise_out});
-  return eltwise_out;
-}
-
-}  // namespace patterns
-
-UnsqueezeEltwiseFusePass::UnsqueezeEltwiseFusePass() {
-  AddOpCompat(OpCompat("unsqueeze2"))
-      .AddInput("X")
-      .IsTensor()
-      .End()
-      .AddInput("AxesTensor")
-      .IsOptional()
-      .IsTensor()
-      .End()
-      .AddInput("AxesTensorList")
-      .IsOptional()
-      .IsTensor()
-      .End()
-      .AddOutput("XShape")
-      .IsOptional()
-      .IsTensor()
-      .End()
-      .AddOutput("Out")
-      .IsTensor()
-      .End()
-      .AddAttr("axes")
-      .IsType<std::vector<int>>()
-      .End();
-
-  AddOpCompat(OpCompat("elementwise_mul"))
-      .AddInput("X")
-      .IsTensor()
-      .End()
-      .AddInput("Y")
-      .IsTensor()
-      .End()
-      .AddOutput("Out")
-      .IsTensor()
-      .End()
-      // The attribute value is - 1 before fusion and 0 after fusion
-      .AddAttr("axis")
-      .IsIntIn({-1, 0})
-      .End();
-}
-
-void UnsqueezeEltwiseFusePass::ApplyImpl(ir::Graph *graph) const {
-  PADDLE_ENFORCE_NOT_NULL(
-      graph, platform::errors::PreconditionNotMet("graph should not be null."));
-  FusePassBase::Init("unsqueeze2_eltwise_fuse_pass", graph);
-  int found_subgraph_count = 0;
-
-  GraphPatternDetector gpd;
-  auto *x = gpd.mutable_pattern()
-                ->NewNode("unsqueeze2_eltwise_fuse_pass/x")
-                ->AsInput()
-                ->assert_is_op_input("elementwise_mul", "X")
-                ->assert_var_not_persistable();
-  auto *y = gpd.mutable_pattern()
-                ->NewNode("unsqueeze2_eltwise_fuse_pass/y")
-                ->AsInput()
-                ->assert_is_op_input("unsqueeze2", "X")
-                ->assert_var_not_persistable();
-  patterns::UnsqueezeEltwise fused_pattern(gpd.mutable_pattern(),
-                                           "unsqueeze2_eltwise_fuse_pass");
-  fused_pattern(x, y);
-
-  auto handler = [&](const GraphPatternDetector::subgraph_t &subgraph,
-                     Graph *graph) {
-    if (subgraph.count(x) <= 0 || subgraph.count(y) <= 0) {
-      LOG(WARNING) << "The subgraph is empty.";
-      return;
-    }
-    if (!IsCompat(subgraph, graph)) {
-      LOG(WARNING) << "Pass in op compat failed.";
-      return;
-    }
-    VLOG(4) << "handle UnsqueezeEltwise fuse";
-    GET_IR_NODE_FROM_SUBGRAPH(eltwise_op, elementwise, fused_pattern);
-    GET_IR_NODE_FROM_SUBGRAPH(eltwise_out, eltwise_out, fused_pattern);
-    GET_IR_NODE_FROM_SUBGRAPH(unsqz_op, unsqz, fused_pattern);
-    GET_IR_NODE_FROM_SUBGRAPH(unsqz_out, unsqz_out, fused_pattern);
-
-    size_t eltwise_in_x_rank = (subgraph.at(x)->Var()->GetShape()).size();
-    size_t unsqz_in_rank = (subgraph.at(y)->Var()->GetShape()).size();
-    std::vector<int> unsqz_op_axes =
-        PADDLE_GET_CONST(std::vector<int>, unsqz_op->Op()->GetAttr("axes"));
-    int eltwise_op_axis =
-        PADDLE_GET_CONST(int, eltwise_op->Op()->GetAttr("axis"));
-
-    if (eltwise_in_x_rank == 4 && unsqz_in_rank == 2 &&
-        unsqz_op_axes == std::vector<int>{2, 3} && eltwise_op_axis == -1) {
-      eltwise_op->Op()->SetAttr("axis", 0);
-      eltwise_op->Op()->SetInput("Y", {subgraph.at(y)->Name()});
-      IR_NODE_LINK_TO(subgraph.at(x), eltwise_op);
-      IR_NODE_LINK_TO(subgraph.at(y), eltwise_op);
-      IR_NODE_LINK_TO(eltwise_op, eltwise_out);
-      GraphSafeRemoveNodes(graph, {unsqz_op, unsqz_out});
-      found_subgraph_count++;
-      if (!IsCompat(*eltwise_op->Op())) {
-        LOG(WARNING) << "unsqueeze2_eltwise_fuse_pass op compat failed.";
-        return;
-      }
-    }
-  };
-
-  gpd(graph, handler);
-  AddStatis(found_subgraph_count);
-}
-
-}  // namespace ir
-}  // namespace framework
-}  // namespace paddle
-
-REGISTER_PASS(unsqueeze2_eltwise_fuse_pass,
-              paddle::framework::ir::UnsqueezeEltwiseFusePass);
-REGISTER_PASS_CAPABILITY(unsqueeze2_eltwise_fuse_pass)
-    .AddCombination(
-        paddle::framework::compatible::OpVersionComparatorCombination()
-            .EQ("unsqueeze2", 0)
-            .LE("elementwise_mul", 1));
diff --git a/paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass.h b/paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass.h
deleted file mode 100644
index 0410e5b3f330c..0000000000000
--- a/paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#pragma once
-
-#include "paddle/fluid/framework/ir/fuse_pass_base.h"
-
-namespace paddle {
-namespace framework {
-namespace ir {
-
-class Graph;
-
-//     |(rank 4)   |(rank 2)                    |(rank 4)    |(rank 2)
-//     |       unsqueeze2(axes=[2,3])           |            |
-//     |           |                    fuse     \          /
-//     |------elementwise_mul(axis=-1)   ->   elementwise_mul(axis=0)
-//                 |                                   |
-//                 |                                   |
-//
-// Notice:
-// the rank of input is obtained from var_desc,
-// it maybe change in runtime.
-class UnsqueezeEltwiseFusePass : public FusePassBase {
- public:
-  UnsqueezeEltwiseFusePass();
-  virtual ~UnsqueezeEltwiseFusePass() {}
-
- protected:
-  void ApplyImpl(ir::Graph* graph) const override;
-};
-
-}  // namespace ir
-}  // namespace framework
-}  // namespace paddle
diff --git a/paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass_tester.cc b/paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass_tester.cc
deleted file mode 100644
index 490186a9df31e..0000000000000
--- a/paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass_tester.cc
+++ /dev/null
@@ -1,67 +0,0 @@
-/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include <gtest/gtest.h>
-
-#include "paddle/fluid/framework/ir/pass_tester_helper.h"
-#include "paddle/fluid/framework/ir/unsqueeze2_eltwise_fuse_pass.h"
-#include "paddle/fluid/framework/op_version_registry.h"
-
-namespace paddle {
-namespace framework {
-namespace ir {
-
-TEST(UnsqueezeEltwiseFusePass, basic) {
-  Layers layers;
-  auto* x = layers.data("x", {1, 92, 28, 28});
-  auto* y = layers.data("y", {1, 92});
-  std::vector<int> axes{2, 3};
-  auto* unsqz_out = layers.unsqueeze2(y, axes);
-  AttributeMap attrs;
-  attrs["axis"] = -1;
-  layers.elementwise_mul(x, unsqz_out, nullptr, &attrs);
-
-  std::unique_ptr<ir::Graph> graph(new ir::Graph(layers.main_program()));
-  auto pass = PassRegistry::Instance().Get("unsqueeze2_eltwise_fuse_pass");
-  int num_nodes_before = graph->Nodes().size();
-  VLOG(3) << DebugString(graph);
-
-  graph.reset(pass->Apply(graph.release()));
-  int num_nodes_after = graph->Nodes().size();
-  int num_fused_nodes_after = GetNumOpNodes(graph, "elementwise_mul");
-  VLOG(3) << DebugString(graph);
-
-  PADDLE_ENFORCE_EQ(num_nodes_before,
-                    num_nodes_after + 2,
-                    platform::errors::PreconditionNotMet(
-                        "The number of nodes before and after the fuse does "
-                        "not meet expectations"));
-  PADDLE_ENFORCE_EQ(
-      num_fused_nodes_after,
-      1,
-      platform::errors::PreconditionNotMet(
-          "The number of fusion nodes does not meet expectations after fuse"));
-}
-
-TEST(UnsqueezeEltwiseFusePass, pass_op_version_check) {
-  ASSERT_TRUE(
-      paddle::framework::compatible::PassVersionCheckerRegistrar::GetInstance()
-          .IsPassCompatible("unsqueeze2_eltwise_fuse_pass"));
-}
-
-}  // namespace ir
-}  // namespace framework
-}  // namespace paddle
-
-USE_PASS(unsqueeze2_eltwise_fuse_pass);
diff --git a/paddle/fluid/inference/api/paddle_pass_builder.cc b/paddle/fluid/inference/api/paddle_pass_builder.cc
index 580578f2a04ad..bea0b82ecd494 100644
--- a/paddle/fluid/inference/api/paddle_pass_builder.cc
+++ b/paddle/fluid/inference/api/paddle_pass_builder.cc
@@ -90,7 +90,6 @@ const std::vector<std::string> kTRTSubgraphPasses({
       "trt_map_ops_to_matrix_multiply_pass",          //
       "shuffle_channel_detect_pass",                  //
       "quant_conv2d_dequant_fuse_pass",               //
-      "delete_fill_constant_op_pass",                 //
       "delete_quant_dequant_op_pass",                 //
       "delete_quant_dequant_filter_op_pass",          //
       "trt_delete_weight_dequant_linear_op_pass",     //
@@ -123,7 +122,6 @@ const std::vector<std::string> kTRTSubgraphPasses({
       "preln_layernorm_x_fuse_pass",     //
       "reverse_roll_fuse_pass",          //
       "conv_bn_fuse_pass",               //
-      "unsqueeze2_eltwise_fuse_pass",    //
       "conv_elementwise_add_fuse_pass",  //
 #if defined _WIN32  // Windows CI is TensorRT7.0. Remove this after upgrading.
 #else
diff --git a/test/ir/inference/test_unsqueeze2_eltwise_fuse_pass.py b/test/ir/inference/test_unsqueeze2_eltwise_fuse_pass.py
deleted file mode 100644
index 163f18d813375..0000000000000
--- a/test/ir/inference/test_unsqueeze2_eltwise_fuse_pass.py
+++ /dev/null
@@ -1,105 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-
-import hypothesis.strategies as st
-from auto_scan_test import PassAutoScanTest
-from program_config import OpConfig, ProgramConfig, TensorConfig
-
-import paddle.inference as paddle_infer
-
-
-class TestUnsqueezeEltwiseFusePass(PassAutoScanTest):
-    r"""
-        y_var
-          |
-       unsqueeze2
-          \
-    unsqueeze2_out_var    x_var
-             \           /
-            elementwise_mul
-    """
-
-    def sample_predictor_configs(self, program_config):
-        # TRT
-        config = self.create_trt_inference_config()
-        config.enable_tensorrt_engine(
-            max_batch_size=10,
-            workspace_size=102400,
-            min_subgraph_size=0,
-            precision_mode=paddle_infer.PrecisionType.Float32,
-            use_static=False,
-            use_calib_mode=False,
-        )
-        yield config, [
-            'elementwise_mul',
-        ], (1e-5, 1e-5)
-
-    def sample_program_config(self, draw):
-        # 1. Generate shape and attr of mul
-        x_shape = draw(
-            st.lists(
-                st.integers(min_value=1, max_value=10), min_size=4, max_size=4
-            )
-        )
-        axis = -1
-
-        # 2. Generate legal shape and attr of input:Y of unsqueeze2
-        y_shape = x_shape[:2]
-        unsqueeze2_axes = [2, 3]
-
-        unsqueeze2_op = OpConfig(
-            "unsqueeze2",
-            inputs={
-                "X": ["unsqueeze2_x"],
-                "AxesTensor": [],
-                "AxesTensorList": [],
-            },
-            axes=unsqueeze2_axes,
-            outputs={"Out": ["unsqueeze2_out"], "XShape": ["xshape"]},
-        )
-        mul_op = OpConfig(
-            "elementwise_mul",
-            inputs={"Y": ["unsqueeze2_out"], "X": ["mul_x"]},
-            axis=axis,
-            outputs={"Out": ["mul_out"]},
-        )
-
-        ops = [
-            unsqueeze2_op,
-            mul_op,
-        ]
-
-        program_config = ProgramConfig(
-            ops=ops,
-            weights={},
-            inputs={
-                "mul_x": TensorConfig(shape=x_shape),
-                "unsqueeze2_x": TensorConfig(shape=y_shape),
-            },
-            outputs=ops[-1].outputs["Out"],
-        )
-        return program_config
-
-    def test(self):
-        self.run_and_statis(
-            quant=False,
-            max_examples=300,
-            passes=["unsqueeze2_eltwise_fuse_pass"],
-        )
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/tools/parallel_UT_rule.py b/tools/parallel_UT_rule.py
index 854dc82d27337..ec9130ff6d91d 100755
--- a/tools/parallel_UT_rule.py
+++ b/tools/parallel_UT_rule.py
@@ -131,7 +131,6 @@
     'test_conv_concat_relu_mkldnn_fuse_pass',
     'test_bf16_utils',
     'test_sum_bf16_mkldnn_op',
-    'test_unsqueeze2_eltwise_fuse_pass_cc',
     'dense_table_test',
     'test_collective_optimizer',
     'test_origin_info',
@@ -2145,7 +2144,6 @@
     'test_recv_save_op',
     'heter_listen_and_server_test',
     'test_analyzer_capi_ner',
-    'test_unsqueeze2_eltwise_fuse_pass_cc',
     'test_dgc_optimizer',
     'heter_server_test',
     'test_custom_conj',
diff --git a/tools/windows/run_unittests.sh b/tools/windows/run_unittests.sh
index 5300eae9b1921..03fb4b0254362 100644
--- a/tools/windows/run_unittests.sh
+++ b/tools/windows/run_unittests.sh
@@ -178,7 +178,6 @@ disable_win_inference_test="^trt_quant_int8_yolov3_r50_test$|\
 ^test_trt_convert_multihead_matmul$|\
 ^test_trt_convert_prelu$|\
 ^test_trt_fc_fuse_quant_dequant_pass$|\
-^test_unsqueeze2_eltwise_fuse_pass$|\
 ^test_parallel_executor_seresnext_with_fuse_all_reduce_gpu$|\
 ^test_parallel_executor_seresnext_with_reduce_gpu$|\
 ^test_api_impl$|\

From eb12e62776e86b46dbca38fba24c6b8f4b795c1d Mon Sep 17 00:00:00 2001
From: cyber-pioneer <116002591+cyber-pioneer@users.noreply.github.com>
Date: Tue, 9 May 2023 11:28:41 +0800
Subject: [PATCH 229/405] fix eval branch of prim vjp of batch_norm in amp mode
 (#53598)

---
 .../composite_backward_api.h                  |  6 +++
 .../test_composite_batch_norm.py              | 38 ++++++++++++++++---
 2 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
index 979be562db93a..a9d91b6e85489 100644
--- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
+++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
@@ -1699,6 +1699,9 @@ void batch_norm_grad(const Tensor& x,
         if (use_global_stats) {
           auto nhwc_x_grad = scale * rsqrt_var * nhwc_out_grad;
           auto nchw_x_grad = transpose<T>(nhwc_x_grad, nhwc_to_nchw_dim);
+          if (x.dtype() == phi::DataType::FLOAT16) {
+            nchw_x_grad = cast<T>(nchw_x_grad, x.dtype());
+          }
           set_output<T>(nchw_x_grad, x_grad);
         } else {
           auto part1 = scale * rsqrt_var;
@@ -1732,6 +1735,9 @@ void batch_norm_grad(const Tensor& x,
             sum<T>(out_grad_data * x_sub_mean, reduce_axis, dtype, false);
         if (use_global_stats) {
           auto x_grad_data = scale * rsqrt_var * out_grad_data;
+          if (x.dtype() == phi::DataType::FLOAT16) {
+            x_grad_data = cast<T>(x_grad_data, x.dtype());
+          }
           set_output<T>(x_grad_data, x_grad);
         } else {
           auto part1 = scale * rsqrt_var;
diff --git a/test/prim/composite_ops/test_composite_batch_norm.py b/test/prim/composite_ops/test_composite_batch_norm.py
index e61269c8f22ee..497d8e4d0a505 100644
--- a/test/prim/composite_ops/test_composite_batch_norm.py
+++ b/test/prim/composite_ops/test_composite_batch_norm.py
@@ -386,10 +386,12 @@ def apply_to_static(net, use_cinn):
 
 
 class PrimeNet(paddle.nn.Layer):
-    def __init__(self, data_layout='NCHW'):
+    def __init__(self, data_layout='NCHW', is_test=False):
         super().__init__()
         self.conv = nn.Conv2D(2, 4, (3, 3), bias_attr=False)
-        self.bn = BatchNorm(4, act="relu", data_layout=data_layout)
+        self.bn = BatchNorm(
+            4, act="relu", data_layout=data_layout, is_test=is_test
+        )
 
     def forward(self, x):
         y = self.conv(x)
@@ -408,10 +410,10 @@ def setUp(self):
         self.x = paddle.randn([4, 2, 6, 6], dtype="float32")
         self.x.stop_gradient = False
 
-    def train(self, use_prim, data_layout="NCHW"):
+    def train(self, use_prim, data_layout="NCHW", is_test=False):
         core._set_prim_all_enabled(use_prim)
         paddle.seed(2022)
-        net = PrimeNet(data_layout)
+        net = PrimeNet(data_layout=data_layout, is_test=is_test)
         sgd = paddle.optimizer.SGD(
             learning_rate=0.1, parameters=net.parameters()
         )
@@ -429,8 +431,19 @@ def train(self, use_prim, data_layout="NCHW"):
 
     def test_amp_nchw(self):
         if not isinstance(framework._current_expected_place(), core.CPUPlace):
-            expected = self.train(False)
-            actual = self.train(True)
+            expected = self.train(use_prim=False)
+            actual = self.train(use_prim=True)
+            np.testing.assert_allclose(
+                expected,
+                actual,
+                rtol=1e-3,
+                atol=1e-3,
+            )
+
+    def test_amp_nchw_eval(self):
+        if not isinstance(framework._current_expected_place(), core.CPUPlace):
+            expected = self.train(use_prim=False, is_test=True)
+            actual = self.train(use_prim=True, is_test=True)
             np.testing.assert_allclose(
                 expected,
                 actual,
@@ -449,6 +462,19 @@ def test_amp_nhwc(self):
                 atol=1e-3,
             )
 
+    def test_amp_nhwc_eval(self):
+        if not isinstance(framework._current_expected_place(), core.CPUPlace):
+            expected = self.train(
+                use_prim=False, data_layout="NHWC", is_test=True
+            )
+            actual = self.train(use_prim=True, data_layout="NHWC", is_test=True)
+            np.testing.assert_allclose(
+                expected,
+                actual,
+                rtol=1e-3,
+                atol=1e-3,
+            )
+
 
 class TestPrimEvalBranch(unittest.TestCase):
     """

From 6029e02aa718ef7d5b60f6d75a56a6c1ae79990e Mon Sep 17 00:00:00 2001
From: GGBond8488 <33050871+GGBond8488@users.noreply.github.com>
Date: Tue, 9 May 2023 13:00:53 +0800
Subject: [PATCH 230/405] [Zero-Dim] add 0D test for linalg.norm/linalg.cond
 (#53592)

* add 0D test for linalg and linalg.cond

* remove p_norm test

* Update test_zero_dim_tensor.py

* Update test_zero_dim_tensor, test=allcase

* add 0D op test for cond and pnorm,test=allcase

* fix conda error
---
 .../tests/unittests/test_zero_dim_tensor.py   | 398 +++++++++++++-----
 1 file changed, 296 insertions(+), 102 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
index 9482f252f876b..76f0ce1791a7b 100644
--- a/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
+++ b/python/paddle/fluid/tests/unittests/test_zero_dim_tensor.py
@@ -2658,57 +2658,156 @@ def test_dist(self):
         self.assertEqual(x.grad.shape, [2, 2])
         self.assertEqual(y.grad.shape, [2, 2])
 
+    def test_linalg_norm(self):
+        # 1D input, p = fro ,axis = None, using reduceInferMeta
+        x_1 = paddle.arange(24, dtype="float32") - 12
+        x_1.stop_gradient = False
+        out_1 = paddle.linalg.norm(x_1)
+        out_1.retain_grads()
+        out_1.backward()
+
+        self.assertEqual(out_1.shape, [])
+        self.assertTrue(x_1.grad.shape, [24])
+
+        # 1D input, p = 1 ,axis = None,
+        # using p_nrom, as_vector = True
+        x_2 = paddle.arange(24, dtype="float32") - 12
+        x_2.stop_gradient = False
+        out_2 = paddle.linalg.norm(x_2, p=1)
+        out_2.retain_grads()
+        out_2.backward()
+
+        self.assertEqual(out_2.shape, [])
+        self.assertEqual(x_2.grad.shape, [24])
+
+        # 1D input, p = 1 ,axis = 0,
+        # using p_nrom, as_vector = False
+        x_2_p = paddle.arange(24, dtype="float32") - 12
+        x_2_p.stop_gradient = False
+        out_2_p = paddle.linalg.norm(x_2_p, p=1, axis=0)
+        out_2_p.retain_grads()
+        out_2_p.backward()
+
+        self.assertEqual(out_2_p.shape, [])
+        self.assertEqual(x_2_p.grad.shape, [24])
+
+        # 1D input, p = fro ,axis = 0,
+        # using p_nrom, as_vector = False
+        x_2_fro = paddle.arange(24, dtype="float32") - 12
+        x_2_fro.stop_gradient = False
+        out_2_fro = paddle.linalg.norm(x_2_fro, p="fro", axis=0)
+        out_2_fro.retain_grads()
+        out_2_fro.backward()
+
+        self.assertEqual(out_2_fro.shape, [])
+        self.assertEqual(x_2_fro.grad.shape, [24])
+
+        # 2D input, p = 1, axis = [0, 1]
+        # using p_matrix_norm ,depends on paddle.sum
+        x_3 = paddle.arange(24, dtype="float32").reshape([4, 6])
+        x_3.stop_gradient = False
+        out_3 = paddle.linalg.norm(x_3, p=1, axis=[0, 1])
+        out_3.retain_grads()
+        out_3.backward()
+        self.assertEqual(out_3.shape, [])
+        self.assertEqual(x_3.grad.shape, [4, 6])
+
+        # 2D input, p = 1, axis = None
+        # using p_matrix_norm, depends on paddle.sum
+        x_4 = paddle.arange(24, dtype="float32").reshape([4, 6])
+        x_4.stop_gradient = False
+        out_4 = paddle.linalg.norm(x_4)
+        out_4.retain_grads()
+        out_4.backward()
+        self.assertEqual(out_4.shape, [])
+        self.assertEqual(x_4.grad.shape, [4, 6])
+
+        # 2D input, p = inf, axis = [0, 1]
+        # using p_matrix_norm, depends on paddle.sum
+        x_5 = paddle.arange(24, dtype="float32").reshape([4, 6])
+        x_5.stop_gradient = False
+        out_5 = paddle.linalg.norm(x_5, p=2, axis=[0, 1])
+        out_5.retain_grads()
+        out_5.backward()
+
+        self.assertEqual(out_5.shape, [])
+        self.assertEqual(x_5.grad.shape, [4, 6])
+
+        # 2D input, p = -inf, axis = [0, 1]
+        x_6 = paddle.arange(24, dtype="float32").reshape([4, 6])
+        x_6.stop_gradient = False
+        out_6 = paddle.linalg.norm(x_6, p=-float("inf"), axis=[0, 1])
+        out_6.retain_grads()
+        out_6.backward()
+
+        self.assertEqual(out_6.shape, [])
+        self.assertEqual(x_6.grad.shape, [4, 6])
+
     def test_linalg_cond(self):
         def assert_shape(out):
             self.assertEqual(out.shape, [])
 
-        # x1 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x1.stop_gradient = False
+        x1 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x1.stop_gradient = False
         # p = 2 : use paddle.sum
-        # out = paddle.linalg.cond(x1)
-        # assert_shape(out)
+        out = paddle.linalg.cond(x1)
+        out.backward()
+        assert_shape(out)
+        self.assertEqual(x1.grad.shape, [3, 3])
 
         # p = fro : use paddle.sum
-        # x2 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x2.stop_gradient = False
-        # out_fro = paddle.linalg.cond(x2, p='fro')
-        # assert_shape(out_fro)
+        x2 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x2.stop_gradient = False
+        out_fro = paddle.linalg.cond(x2, p='fro')
+        out_fro.backward()
+        assert_shape(out_fro)
+        self.assertEqual(x2.grad.shape, [3, 3])
 
         # p = nuc : use paddle.sum
-        # x3 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x3.stop_gradient = False
-        # out_nuc = paddle.linalg.cond(x3, p='nuc')
-        # assert_shape(out_nuc)
+        x3 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x3.stop_gradient = False
+        out_nuc = paddle.linalg.cond(x3, p='nuc')
+        out_nuc.backward()
+        assert_shape(out_nuc)
+        self.assertEqual(x3.grad.shape, [3, 3])
 
         # p in (-1, 1) : use paddle.sum
-        # x4 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x4.stop_gradient = False
-        # out_1 = paddle.linalg.cond(x4, p=1)
-        # assert_shape(out_1)
-        # x5 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x5.stop_gradient = False
-        # out_minus_1 = paddle.linalg.cond(x5, p=-1)
-        # assert_shape(out_minus_1)
+        x4 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x4.stop_gradient = False
+        out_1 = paddle.linalg.cond(x4, p=1)
+        out_1.backward()
+        assert_shape(out_1)
+        self.assertEqual(x4.grad.shape, [3, 3])
+
+        x5 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x5.stop_gradient = False
+        out_minus_1 = paddle.linalg.cond(x5, p=-1)
+        out_minus_1.backward()
+        assert_shape(out_minus_1)
+        self.assertEqual(x5.grad.shape, [3, 3])
 
         # p in (-2, 2)  depends on paddle.sum
-        # x6 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x6.stop_gradient = False
-        # out_2 = paddle.linalg.cond(x6, p=2)
-        # assert_shape(out_2)
+        x6 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x6.stop_gradient = False
+        out_2 = paddle.linalg.cond(x6, p=2)
+        out_2.backward()
+        assert_shape(out_2)
+        self.assertEqual(x6.grad.shape, [3, 3])
 
         # p in (-inf, inf):use paddle.sum
-        # x8 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x8.stop_gradient = False
-        # out_inf = paddle.linalg.cond(x8, p=float("inf"))
-        # assert_shape(out_inf)
-
-        # depends on paddle.sum
-        # a = paddle.randn([2, 4, 4])
-        # a.stop_gradient = False
-        # a_cond_fro = paddle.linalg.cond(a, p='fro')
-        # a_cond_fro.backward()
-        # self.assertEqual(len(a_cond_fro.shape), 1)
-        # self.assertEqual(a.grad.shape, [2, 4, 4])
+        x8 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x8.stop_gradient = False
+        out_inf = paddle.linalg.cond(x8, p=float("inf"))
+        out_inf.backward()
+        assert_shape(out_inf)
+        self.assertEqual(x8.grad.shape, [3, 3])
+
+        a = paddle.randn([2, 4, 4])
+        a.stop_gradient = False
+        a_cond_fro = paddle.linalg.cond(a, p='fro')
+        a_cond_fro.backward()
+        self.assertEqual(len(a_cond_fro.shape), 1)
+        self.assertEqual(a.grad.shape, [2, 4, 4])
 
     def test_trace(self):
         x = paddle.to_tensor([[3, 2], [1, 9]], dtype="float32")
@@ -4861,103 +4960,198 @@ def test_dist(self):
         self.assertEqual(res[1].shape, (2, 2))
         np.testing.assert_array_equal(res[0], np.array(2).astype(np.float32))
 
+    @prog_scope()
+    def test_linalg_norm(self):
+        # 1D input, p = fro ,axis = None, using reduceInferMeta
+        x_1 = paddle.arange(24, dtype="float32") - 12
+        x_1.stop_gradient = False
+        out_1 = paddle.linalg.norm(x_1)
+        paddle.static.append_backward(out_1)
+
+        prog = paddle.static.default_main_program()
+
+        res = self.exe.run(prog, fetch_list=[out_1, x_1.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (24,))
+
+        # 1D input, p = 1 ,axis = None,
+        # using p_nrom, as_vector = True
+        x_2 = paddle.arange(24, dtype="float32") - 12
+        x_2.stop_gradient = False
+        out_2 = paddle.linalg.norm(x_2, p=1)
+        paddle.static.append_backward(out_2.sum())
+
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out_2, x_2.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (24,))
+
+        # 1D input, p = 1 ,axis = 0,
+        # using p_nrom, as_vector = False
+        x_2_p = paddle.arange(24, dtype="float32") - 12
+        x_2_p.stop_gradient = False
+        out_2_p = paddle.linalg.norm(x_2_p, p=1, axis=0)
+        paddle.static.append_backward(out_2_p.sum())
+
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out_2_p, x_2_p.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (24,))
+
+        # 1D input, p = fro ,axis = 0,
+        # using p_nrom, as_vector = False
+        x_2_fro = paddle.arange(24, dtype="float32") - 12
+        x_2_fro.stop_gradient = False
+        out_2_fro = paddle.linalg.norm(x_2_fro, p="fro", axis=0)
+        paddle.static.append_backward(out_2_fro.sum())
+
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out_2_fro, x_2_fro.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (24,))
+
+        # 2D input, p = 1, axis = [0, 1]
+        # using p_matrix_norm ,depends on  paddle.sum
+        x_3 = paddle.arange(24, dtype="float32").reshape([4, 6])
+        x_3.stop_gradient = False
+        out_3 = paddle.linalg.norm(x_3, p=1, axis=[0, 1])
+        paddle.static.append_backward(out_3.sum())
+
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out_3, x_3.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (4, 6))
+
+        # 2D input, p = 1, axis = None
+        # using p_matrix_norm, depends on paddle.sum
+        x_4 = paddle.arange(24, dtype="float32").reshape([4, 6])
+        x_4.stop_gradient = False
+        out_4 = paddle.linalg.norm(x_4)
+        paddle.static.append_backward(out_4.sum())
+
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out_4, x_4.grad_name])
+
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (4, 6))
+
+        # 2D input, p = inf, axis = None
+        x_5 = paddle.arange(24, dtype="float32").reshape([4, 6])
+        x_5.stop_gradient = False
+        out_5 = paddle.linalg.norm(x_5)
+        paddle.static.append_backward(out_5.sum())
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out_5, x_5.grad_name])
+
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (4, 6))
+
+        # 2D input, p = -inf, axis = [0, 1]
+        x_6 = paddle.arange(24, dtype="float32").reshape([4, 6])
+        x_6.stop_gradient = False
+        out_6 = paddle.linalg.norm(x_6, p=-float("inf"), axis=[0, 1])
+        paddle.static.append_backward(out_6.sum())
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out_6, x_6.grad_name])
+
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (4, 6))
+
     @prog_scope()
     def test_linalg_cond(self):
-        pass
         # use paddle.sum
-        # x = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x.stop_gradient = False
-        # out = paddle.linalg.cond(x)
-        # paddle.static.append_backward(out)
-
-        # prog = paddle.static.default_main_program()
-        # res = self.exe.run(prog, fetch_list=[out, x.grad_name])
+        x = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x.stop_gradient = False
+        out = paddle.linalg.cond(x)
+        paddle.static.append_backward(out)
 
-        # self.assertTrue(res[0].shape, ())
-        # self.assertTrue(res[1].shape, (3, 3))
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out, x.grad_name])
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (3, 3))
 
         # p = fro : use paddle.sum
-        # x2 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x2.stop_gradient = False
-        # out_fro = paddle.linalg.cond(x2, p='fro')
-        # paddle.static.append_backward(out_fro)
+        x2 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x2.stop_gradient = False
+        out_fro = paddle.linalg.cond(x2, p='fro')
+        paddle.static.append_backward(out_fro)
 
-        # prog = paddle.static.default_main_program()
-        # res = self.exe.run(prog, fetch_list=[out_fro, x.grad_name])
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out_fro, x.grad_name])
 
-        # self.assertTrue(res[0].shape, ())
-        # self.assertTrue(res[1].shape, (3, 3))
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (3, 3))
 
         # p = nuc : use paddle.sum
-        # x3 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x3.stop_gradient = False
-        # out_nuc = paddle.linalg.cond(x3, p='nuc')
-        # paddle.static.append_backward(out_nuc)
+        x3 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x3.stop_gradient = False
+        out_nuc = paddle.linalg.cond(x3, p='nuc')
+        paddle.static.append_backward(out_nuc)
 
-        # prog = paddle.static.default_main_program()
-        # res = self.exe.run(prog, fetch_list=[out_nuc, x.grad_name])
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out_nuc, x.grad_name])
 
-        # self.assertTrue(res[0].shape, ())
-        # self.assertTrue(res[1].shape, (3, 3))
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (3, 3))
 
         # p in (-1, 1) : use paddle.sum
-        # x4 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x4.stop_gradient = False
-        # out_1 = paddle.linalg.cond(x4, p=1)
-        # paddle.static.append_backward(out_1)
+        x4 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x4.stop_gradient = False
+        out_1 = paddle.linalg.cond(x4, p=1)
+        paddle.static.append_backward(out_1)
 
-        # prog = paddle.static.default_main_program()
-        # res = self.exe.run(prog, fetch_list=[out_1, x.grad_name])
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out_1, x.grad_name])
 
-        # self.assertTrue(res[0].shape, ())
-        # self.assertTrue(res[1].shape, (3, 3))
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (3, 3))
 
-        # x5 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x5.stop_gradient = False
-        # out_minus_1 = paddle.linalg.cond(x5, p=-1)
-        # paddle.static.append_backward(out_minus_1)
+        x5 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x5.stop_gradient = False
+        out_minus_1 = paddle.linalg.cond(x5, p=-1)
+        paddle.static.append_backward(out_minus_1)
 
-        # prog = paddle.static.default_main_program()
-        # res = self.exe.run(prog, fetch_list=[out_minus_1, x.grad_name])
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out_minus_1, x.grad_name])
 
-        # self.assertTrue(res[0].shape, ())
-        # self.assertTrue(res[1].shape, (3, 3))
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (3, 3))
 
         # p in (-2, 2)  depends on paddle.sum
-        # x6 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x6.stop_gradient = False
-        # out_2 = paddle.linalg.cond(x6, p=2)
-        # paddle.static.append_backward(out_2)
+        x6 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x6.stop_gradient = False
+        out_2 = paddle.linalg.cond(x6, p=2)
+        paddle.static.append_backward(out_2)
 
-        # prog = paddle.static.default_main_program()
-        # res = self.exe.run(prog, fetch_list=[out_2, x.grad_name])
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out_2, x.grad_name])
 
-        # self.assertTrue(res[0].shape, ())
-        # self.assertTrue(res[1].shape, (3, 3))
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (3, 3))
 
         # p in (-inf, inf):use paddle.sum
-        # x8 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
-        # x8.stop_gradient = False
-        # out_inf = paddle.linalg.cond(x8, p=float("inf"))
-        # paddle.static.append_backward(out_inf)
+        x8 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x8.stop_gradient = False
+        out_inf = paddle.linalg.cond(x8, p=float("inf"))
+        paddle.static.append_backward(out_inf)
 
-        # prog = paddle.static.default_main_program()
-        # res = self.exe.run(prog, fetch_list=[out_inf, x.grad_name])
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[out_inf, x.grad_name])
 
-        # self.assertTrue(res[0].shape, ())
-        # self.assertTrue(res[1].shape, (3, 3))
+        self.assertEqual(res[0].shape, ())
+        self.assertEqual(res[1].shape, (3, 3))
 
         # depends on paddle.sum
-        # a = paddle.randn([2, 4, 4])
-        # a.stop_gradient = False
-        # a_cond_fro = paddle.linalg.cond(a, p='fro')
-        # paddle.static.append_backward(a_cond_fro)
+        a = paddle.randn([2, 4, 4])
+        a.stop_gradient = False
+        a_cond_fro = paddle.linalg.cond(a, p='fro')
+        paddle.static.append_backward(a_cond_fro.sum())
 
-        # prog = paddle.static.default_main_program()
-        # res = self.exe.run(prog, fetch_list=[a_cond_fro, a.grad_name])
+        prog = paddle.static.default_main_program()
+        res = self.exe.run(prog, fetch_list=[a_cond_fro, a.grad_name])
 
-        # self.assertEqual(res[0].shape, (2,))
-        # self.assertEqual(res[1].shape, (2, 4, 4))
+        self.assertEqual(res[0].shape, (2,))
+        self.assertEqual(res[1].shape, (2, 4, 4))
 
     @prog_scope()
     def test_trace(self):

From 9682b04ed7758dcf553866fd918ed07e33fa2250 Mon Sep 17 00:00:00 2001
From: umiswing <umiswing@foxmail.com>
Date: Tue, 9 May 2023 14:39:08 +0800
Subject: [PATCH 231/405] [cutlass] Add generated files to .gitignore (#53607)

---
 .gitignore | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index a4690ba442520..6fef519e976c0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -75,7 +75,8 @@ tools/nvcc_lazy
 # TODO(zhiqiang) Move this file to build directory.
 paddle/fluid/pybind/eager_op_function.cc
 tools/nvcc_lazy
-
+paddle/phi/kernels/sparse/gpu/cutlass_generator/all_gemm_operations.h
+paddle/phi/kernels/sparse/gpu/cutlass_generator/configurations.h
 
 # these files (directories) are generated before build system generation
 paddle/fluid/operators/generated_op*.cc

From 72cb09e3a3198a98be54210f386299daf4dc2ee2 Mon Sep 17 00:00:00 2001
From: zhiboniu <31800336+zhiboniu@users.noreply.github.com>
Date: Tue, 9 May 2023 15:03:53 +0800
Subject: [PATCH 232/405] add logaddexp api (#52284)

* add logaddexp

* update
---
 python/paddle/__init__.py                     |  2 +
 .../fluid/tests/unittests/test_logaddexp.py   | 71 +++++++++++++++++++
 python/paddle/tensor/__init__.py              |  2 +
 python/paddle/tensor/math.py                  | 58 +++++++++++++++
 4 files changed, 133 insertions(+)
 create mode 100644 python/paddle/fluid/tests/unittests/test_logaddexp.py

diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py
index d155dce987c08..e9a4d5e35be44 100644
--- a/python/paddle/__init__.py
+++ b/python/paddle/__init__.py
@@ -258,6 +258,7 @@
 from .tensor.math import add  # noqa: F401
 from .tensor.math import subtract  # noqa: F401
 from .tensor.math import logsumexp  # noqa: F401
+from .tensor.math import logaddexp  # noqa: F401
 from .tensor.math import inverse  # noqa: F401
 from .tensor.math import log1p  # noqa: F401
 from .tensor.math import erf  # noqa: F401
@@ -437,6 +438,7 @@
     'eye',
     'cumsum',
     'cumprod',
+    'logaddexp',
     'logcumsumexp',
     'logit',
     'LazyGuard',
diff --git a/python/paddle/fluid/tests/unittests/test_logaddexp.py b/python/paddle/fluid/tests/unittests/test_logaddexp.py
new file mode 100644
index 0000000000000..434fae3d8f3f2
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_logaddexp.py
@@ -0,0 +1,71 @@
+#  Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+import paddle
+
+
+def ref_logaddexp_old(x, y):
+    y = np.broadcast_to(y, x.shape)
+    out = np.log1p(np.exp(-np.absolute(x - y))) + np.maximum(x, y)
+    return out
+
+
+def ref_logaddexp(x, y):
+    return np.logaddexp(x, y)
+
+
+class TestLogsumexpAPI(unittest.TestCase):
+    def setUp(self):
+        self.place = (
+            paddle.CUDAPlace(0)
+            if paddle.fluid.core.is_compiled_with_cuda()
+            else paddle.CPUPlace()
+        )
+
+    def api_case(self):
+        self.x = np.random.uniform(-1, 1, self.xshape).astype(self.dtype)
+        self.y = np.random.uniform(-1, 1, self.yshape).astype(self.dtype)
+        out_ref = ref_logaddexp(self.x, self.y)
+
+        # paddle.disable_static(self.place)
+        x = paddle.to_tensor(self.x)
+        y = paddle.to_tensor(self.y)
+        out = paddle.logaddexp(x, y)
+        np.testing.assert_allclose(out.numpy(), out_ref, atol=1e-06)
+
+    def test_api(self):
+        self.xshape = [1, 2, 3, 4]
+        self.yshape = [1, 2, 3, 4]
+        self.dtype = np.float64
+        self.api_case()
+
+    def test_api_broadcast(self):
+        self.xshape = [1, 2, 3, 4]
+        self.yshape = [1, 2, 3, 1]
+        self.dtype = np.float32
+        self.api_case()
+
+    def test_api_bigdata(self):
+        self.xshape = [10, 200, 300]
+        self.yshape = [10, 200, 300]
+        self.dtype = np.float32
+        self.api_case()
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/python/paddle/tensor/__init__.py b/python/paddle/tensor/__init__.py
index bea2fd7323d9f..4cd1614a1d60b 100644
--- a/python/paddle/tensor/__init__.py
+++ b/python/paddle/tensor/__init__.py
@@ -201,6 +201,7 @@
 from .math import subtract_  # noqa: F401
 from .math import atan2  # noqa: F401
 from .math import logsumexp  # noqa: F401
+from .math import logaddexp  # noqa: F401
 from .math import inverse  # noqa: F401
 from .math import log2  # noqa: F401
 from .math import log10  # noqa: F401
@@ -341,6 +342,7 @@
     'floor',
     'floor_',
     'increment',
+    'logaddexp',
     'log',
     'log2',
     'log10',
diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py
index 4a257822e5b0d..40163ddf5d9d6 100644
--- a/python/paddle/tensor/math.py
+++ b/python/paddle/tensor/math.py
@@ -636,6 +636,64 @@ def add_(x, y, name=None):
     return _C_ops.add_(x, y)
 
 
+def logaddexp(x, y, name=None):
+    """
+    Elementwise LogAddExp Operator.
+    Add of exponentiations of the inputs
+    The equation is:
+
+    ..  math::
+
+        Out=log(X.exp()+Y.exp())
+
+    $X$ the tensor of any dimension.
+    $Y$ the tensor whose dimensions must be less than or equal to the dimensions of $X$.
+
+    There are two cases for this operator:
+
+    1. The shape of $Y$ is the same with $X$.
+    2. The shape of $Y$ is a continuous subsequence of $X$.
+
+    For case 2:
+
+    1. Broadcast $Y$ to match the shape of $X$, where axis is the start dimension index for broadcasting $Y$ onto $X$.
+    2. If $axis$ is -1 (default), $axis$=rank($X$)-rank($Y$).
+    3. The trailing dimensions of size 1 for $Y$ will be ignored for the consideration of subsequence, such as shape($Y$) = (2, 1) => (2).
+
+        For example:
+
+        ..  code-block:: python
+
+            shape(X) = (2, 3, 4, 5), shape(Y) = (,)
+            shape(X) = (2, 3, 4, 5), shape(Y) = (5,)
+            shape(X) = (2, 3, 4, 5), shape(Y) = (4, 5), with axis=-1(default) or axis=2
+            shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1
+            shape(X) = (2, 3, 4, 5), shape(Y) = (2), with axis=0
+            shape(X) = (2, 3, 4, 5), shape(Y) = (2, 1), with axis=0
+
+    Args:
+        x (Tensor): Tensor or LoDTensor of any dimensions. Its dtype should be float32, float64, float16.
+        y (Tensor): Tensor or LoDTensor of any dimensions. Its dtype should be float32, float64, float16.
+        name (string, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
+
+    Returns:
+        N-D Tensor. A location into which the result is stored. It's dimension equals with x.
+
+    Examples:
+
+        ..  code-block:: python
+
+            import paddle
+
+            x = paddle.to_tensor([-1, -2, -3], 'float64')
+            y = paddle.to_tensor([-1], 'float64')
+            z = paddle.logaddexp(x, y)
+            print(z)  # [-0.30685282, -0.68673831, -0.87307199]
+    """
+
+    return paddle.log1p(paddle.exp(-paddle.abs(x - y))) + paddle.maximum(x, y)
+
+
 def subtract(x, y, name=None):
     """
     Substract two tensors element-wise. The equation is:

From ea0abf93d4978ffc3806556ce9ecd1aede8d6f73 Mon Sep 17 00:00:00 2001
From: Wilber <jiweibo@baidu.com>
Date: Tue, 9 May 2023 16:20:47 +0800
Subject: [PATCH 233/405] Support trt cuda graph. (#53406)

---
 paddle/fluid/inference/analysis/argument.h    |  1 +
 .../inference/analysis/ir_pass_manager.cc     |  2 +
 .../ir_passes/tensorrt_subgraph_pass.cc       | 75 ++++++++++++++++---
 .../ir_passes/tensorrt_subgraph_pass.h        |  9 ++-
 paddle/fluid/inference/api/analysis_config.cc | 12 ++-
 .../fluid/inference/api/analysis_predictor.cc |  1 +
 .../inference/api/paddle_analysis_config.h    |  7 +-
 paddle/fluid/inference/tensorrt/engine.cc     | 53 ++++++++++++-
 paddle/fluid/inference/tensorrt/engine.h      | 73 ++++++++++++++++++
 .../fluid/inference/tensorrt/test_engine.cc   |  1 +
 paddle/fluid/pybind/inference_api.cc          |  3 +-
 test/cpp/inference/api/CMakeLists.txt         |  2 +-
 test/cpp/inference/api/trt_mobilenet_test.cc  | 26 +++++++
 13 files changed, 245 insertions(+), 20 deletions(-)

diff --git a/paddle/fluid/inference/analysis/argument.h b/paddle/fluid/inference/analysis/argument.h
index 88e6749223008..8a43229af7971 100644
--- a/paddle/fluid/inference/analysis/argument.h
+++ b/paddle/fluid/inference/analysis/argument.h
@@ -231,6 +231,7 @@ struct Argument {
                       TensorRtUseStaticEngine,
                       bool);
   DECL_ARGUMENT_FIELD(tensorrt_use_calib_mode, TensorRtUseCalibMode, bool);
+  DECL_ARGUMENT_FIELD(tensorrt_use_cuda_graph, TensorRtUseCudaGraph, bool);
   DECL_ARGUMENT_FIELD(tensorrt_use_varseqlen, TensorRtUseOSS, bool);
   DECL_ARGUMENT_FIELD(tensorrt_with_interleaved, TensorRtWithInterleaved, bool);
   DECL_ARGUMENT_FIELD(tensorrt_transformer_posid,
diff --git a/paddle/fluid/inference/analysis/ir_pass_manager.cc b/paddle/fluid/inference/analysis/ir_pass_manager.cc
index 1d87edcd3404c..4051511906b1b 100644
--- a/paddle/fluid/inference/analysis/ir_pass_manager.cc
+++ b/paddle/fluid/inference/analysis/ir_pass_manager.cc
@@ -165,6 +165,8 @@ void IRPassManager::CreatePasses(Argument *argument,
                 new AnalysisConfig::Precision(precision_mode));
       pass->Set("context_memory_sharing",
                 new bool(argument->trt_engine_memory_sharing()));
+      pass->Set("use_cuda_graph",
+                new bool(argument->tensorrt_use_cuda_graph()));
       bool use_static_engine = argument->tensorrt_use_static_engine();
       bool model_from_memory = argument->model_from_memory();
       std::string optim_cache_dir = argument->optim_cache_dir();
diff --git a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc
index b64d2bd16ce1a..33e5622436a0f 100644
--- a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc
+++ b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc
@@ -101,6 +101,22 @@ void OutputProcess(framework::ir::Graph *graph,
   }
 }
 
+// Determine whether the whole graph offload to tensorrt. If so we can try to
+// enable optimization such as cudaGraph.
+bool AllNodesLowerToTrtPostProcess(framework::ir::Graph *graph) {
+  std::unordered_set<std::string> trt_nodes_set{
+      "feed", "fetch", "tensorrt_engine"};
+  bool all_nodes_offload_to_trt = true;
+  for (auto *node : graph->Nodes()) {
+    if (node->IsOp()) {
+      if (!trt_nodes_set.count(node->Op()->Type())) {
+        all_nodes_offload_to_trt = false;
+        break;
+      }
+    }
+  }
+  return all_nodes_offload_to_trt;
+}
 }  // namespace
 
 using framework::ir::Node;
@@ -124,6 +140,7 @@ void analysis::TensorRtSubgraphPass::ApplyImpl(
 
   auto enable_int8 = Get<bool>("enable_int8");
   auto use_calib_mode = Get<bool>("use_calib_mode");
+  bool use_cuda_graph = Get<bool>("use_cuda_graph");
   bool no_calib_int8 = enable_int8 && !(use_calib_mode);
   auto trt_disabled_ops = Get<std::vector<std::string>>("trt_disabled_ops");
   auto with_dynamic_shape = Get<bool>("with_dynamic_shape");
@@ -165,13 +182,11 @@ void analysis::TensorRtSubgraphPass::ApplyImpl(
   // those parameter already exist in trt, and should not have another copy in
   // fluid.
   std::vector<std::string> repetitive_params;
+  std::vector<std::string> engine_names;
   for (auto *node : graph->Nodes()) {
     if (node->IsOp() && !framework::ir::Agent(node).subgraph()->empty()) {
-      CreateTensorRTOp(node, graph, graph_param_names, &repetitive_params);
-      std::unordered_set<const Node *> nodes2remove(
-          framework::ir::Agent(node).subgraph()->begin(),
-          framework::ir::Agent(node).subgraph()->end());
-      framework::ir::GraphSafeRemoveNodes(graph, nodes2remove);
+      engine_names.push_back(CreateTensorRTOp(
+          node, graph, graph_param_names, &repetitive_params, use_cuda_graph));
     }
   }
 
@@ -184,6 +199,32 @@ void analysis::TensorRtSubgraphPass::ApplyImpl(
   framework::ir::GraphSafeRemoveNodes(graph, nodes2remove);
   graph->Set(framework::ir::kRepetitiveParamAttr,
              new std::vector<std::string>(repetitive_params));
+
+  bool all_nodes_offload_to_trt = AllNodesLowerToTrtPostProcess(graph);
+  if (all_nodes_offload_to_trt) {
+    LOG(INFO) << "The entire graph is offloaded to TensorRT.";
+  }
+  if (use_cuda_graph && !all_nodes_offload_to_trt) {
+    LOG_FIRST_N(WARNING, 1)
+        << "You have enabled CudaGraph, but not the entire graph offload to "
+           "trt, now return to normal mode.";
+    use_cuda_graph = false;
+  }
+  if (use_cuda_graph && all_nodes_offload_to_trt) {
+    for (auto &name : engine_names) {
+      PADDLE_ENFORCE_EQ(
+          paddle::inference::Singleton<
+              inference::tensorrt::TRTEngineManager>::Global()
+              .Has(name),
+          true,
+          platform::errors::PreconditionNotMet(
+              "TRTEnegineManager shoud has engine %s, but not found.", name));
+      paddle::inference::Singleton<
+          inference::tensorrt::TRTEngineManager>::Global()
+          .Get(name)
+          ->SetAllNodesLowerToTrt(use_cuda_graph);
+    }
+  }
 }
 
 std::string GenerateEngineKey(const std::set<std::string> &engine_inputs,
@@ -191,6 +232,7 @@ std::string GenerateEngineKey(const std::set<std::string> &engine_inputs,
                               const std::string &predictor_id,
                               const std::string &max_batch_size,
                               const std::string &precision,
+                              bool use_cuda_graph,
                               const bool for_calibration) {
   std::string engine_hash_key = "";
   for (auto name : engine_inputs) {
@@ -209,17 +251,21 @@ std::string GenerateEngineKey(const std::set<std::string> &engine_inputs,
   engine_hash_key += "#";
   engine_hash_key += precision;
 
+  engine_hash_key += "#";
+  engine_hash_key += use_cuda_graph;
+
   auto engine_key = std::to_string(std::hash<std::string>()(engine_hash_key));
   VLOG(2) << "TRT engine hash key: " << engine_hash_key;
   VLOG(2) << "TRT engine key: " << engine_key;
   return engine_key;
 }
 
-void TensorRtSubgraphPass::CreateTensorRTOp(
+std::string TensorRtSubgraphPass::CreateTensorRTOp(
     framework::ir::Node *node,
     framework::ir::Graph *graph,
     const std::vector<std::string> &graph_params,
-    std::vector<std::string> *repetitive_params) const {
+    std::vector<std::string> *repetitive_params,
+    bool use_cuda_graph) const {
   auto *op_desc = node->Op();
   auto &subgraph = *framework::ir::Agent(node).subgraph();
   PADDLE_ENFORCE_EQ(subgraph.empty(),
@@ -506,6 +552,7 @@ void TensorRtSubgraphPass::CreateTensorRTOp(
                         std::to_string(0),
                         std::to_string(max_batch_size),
                         std::to_string(static_cast<int>(precision_mode)),
+                        use_cuda_graph,
                         false);
   auto calibration_engine_key =
       GenerateEngineKey(input_names_with_id,
@@ -513,6 +560,7 @@ void TensorRtSubgraphPass::CreateTensorRTOp(
                         std::to_string(0),
                         std::to_string(max_batch_size),
                         std::to_string(static_cast<int>(precision_mode)),
+                        use_cuda_graph,
                         true);
   auto predictor_id = Get<int>("predictor_id");
 
@@ -547,7 +595,7 @@ void TensorRtSubgraphPass::CreateTensorRTOp(
       (enable_int8 && calibration_data.size() == 0 && use_calib_mode);
   if (calibration_mode) {
     // calibraion mode means generate int8 calibration table data process.
-    return;
+    return calibration_engine_key;
   }
 
   std::copy(params_not_shared.begin(),
@@ -582,6 +630,11 @@ void TensorRtSubgraphPass::CreateTensorRTOp(
            "recommend using the same TRT version at runtime.";
   }
 
+  std::unordered_set<const Node *> nodes2remove(
+      framework::ir::Agent(node).subgraph()->begin(),
+      framework::ir::Agent(node).subgraph()->end());
+  framework::ir::GraphSafeRemoveNodes(graph, nodes2remove);
+
   // Setting the disable_trt_plugin_fp16 to true means that TRT plugin will not
   // run fp16.
   // When running fp16, the output accuracy of the model will be affected,
@@ -628,7 +681,7 @@ void TensorRtSubgraphPass::CreateTensorRTOp(
         LOG(INFO) << "Load TRT Optimized Info from "
                   << GetTrtEngineSerializedPath(
                          Get<std::string>("model_opt_cache_dir"), engine_key);
-        return;
+        return engine_key + std::to_string(predictor_id);
       } catch (const std::exception &exp) {
         LOG(WARNING)
             << "Fail to load TRT Optimized Info from "
@@ -643,7 +696,7 @@ void TensorRtSubgraphPass::CreateTensorRTOp(
   // If with_dynamic_shape is configured，but min_input_shape is empty,
   // create trt engine in runtime instead of in pass.
   if (with_dynamic_shape && min_input_shape.empty()) {
-    return;
+    return engine_key + std::to_string(predictor_id);
   }
 
   // the following code will NOT run in following situation:
@@ -676,6 +729,8 @@ void TensorRtSubgraphPass::CreateTensorRTOp(
               << GetTrtEngineSerializedPath(
                      Get<std::string>("model_opt_cache_dir"), engine_key);
   }
+
+  return engine_key + std::to_string(predictor_id);
 }
 
 }  // namespace analysis
diff --git a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.h b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.h
index 1bc86d554b50f..a79c41f6a075f 100644
--- a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.h
+++ b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.h
@@ -42,10 +42,11 @@ class TensorRtSubgraphPass : public framework::ir::FusePassBase {
   void ApplyImpl(framework::ir::Graph *graph) const override;
 
  private:
-  void CreateTensorRTOp(framework::ir::Node *x,
-                        framework::ir::Graph *graph,
-                        const std::vector<std::string> &graph_params,
-                        std::vector<std::string> *repetitive_params) const;
+  std::string CreateTensorRTOp(framework::ir::Node *x,
+                               framework::ir::Graph *graph,
+                               const std::vector<std::string> &graph_params,
+                               std::vector<std::string> *repetitive_params,
+                               bool use_cuda_graph) const;
   void CleanIntermediateOutputs(framework::ir::Node *node);
 };
 
diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc
index 3fa947ce27daf..4f73fb23c6a5a 100644
--- a/paddle/fluid/inference/api/analysis_config.cc
+++ b/paddle/fluid/inference/api/analysis_config.cc
@@ -16,6 +16,7 @@
 #include <string>
 #include <tuple>
 
+#include "glog/logging.h"
 #include "paddle/fluid/inference/api/helper.h"
 #include "paddle/fluid/inference/api/paddle_analysis_config.h"
 #include "paddle/fluid/inference/api/paddle_pass_builder.h"
@@ -442,6 +443,7 @@ AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) {
   CP_MEMBER(trt_dla_core_);
   CP_MEMBER(trt_use_static_engine_);
   CP_MEMBER(trt_use_calib_mode_);
+  CP_MEMBER(trt_use_cuda_graph_);
   CP_MEMBER(trt_use_varseqlen_);
   CP_MEMBER(trt_with_interleaved_);
   CP_MEMBER(tensorrt_transformer_posid_);
@@ -719,7 +721,8 @@ void AnalysisConfig::EnableTensorRtEngine(
     int min_subgraph_size,
     AnalysisConfig::Precision precision_mode,
     bool use_static,
-    bool use_calib_mode) {
+    bool use_calib_mode,
+    bool use_cuda_graph) {
 #ifdef PADDLE_WITH_TENSORRT
   if (!use_gpu()) {
     LOG(ERROR) << "To use TensorRT engine, please call EnableUseGpu() first";
@@ -733,6 +736,11 @@ void AnalysisConfig::EnableTensorRtEngine(
   tensorrt_precision_mode_ = precision_mode;
   trt_use_static_engine_ = use_static;
   trt_use_calib_mode_ = use_calib_mode;
+  trt_use_cuda_graph_ = use_cuda_graph;
+  if (use_cuda_graph) {
+    LOG_FIRST_N(INFO, 1) << "You have enabled Trt Cuda Graph, you must ensure "
+                            "that the input Shape remains unchanged.";
+  }
 
   Update();
 #else
@@ -1313,6 +1321,8 @@ std::string AnalysisConfig::Summary() {
                     trt_use_static_engine_ ? "true" : "false"});
       os.InsertRow(
           {"tensorrt_use_calib_mode", trt_use_calib_mode_ ? "true" : "false"});
+      os.InsertRow(
+          {"tensorrt_use_cuda_graph", trt_use_cuda_graph_ ? "true" : "false"});
 
       // dynamic_shape
       os.InsertRow({"tensorrt_enable_dynamic_shape",
diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc
index 466eee7d9414f..5495f929e8895 100644
--- a/paddle/fluid/inference/api/analysis_predictor.cc
+++ b/paddle/fluid/inference/api/analysis_predictor.cc
@@ -1352,6 +1352,7 @@ void AnalysisPredictor::PrepareArgument() {
     argument_->SetTensorRtDLACore(config_.trt_dla_core_);
     argument_->SetTensorRtUseStaticEngine(config_.trt_use_static_engine_);
     argument_->SetTensorRtUseCalibMode(config_.trt_use_calib_mode_);
+    argument_->SetTensorRtUseCudaGraph(config_.trt_use_cuda_graph_);
     argument_->SetCloseTrtPluginFp16(config_.disable_trt_plugin_fp16_);
     argument_->SetTensorRtShapeRangeInfoPath(config_.shape_range_info_path());
     argument_->SetTensorRtAllowBuildAtRuntime(
diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h
index 3300fe8a9b8dc..d810442810af7 100644
--- a/paddle/fluid/inference/api/paddle_analysis_config.h
+++ b/paddle/fluid/inference/api/paddle_analysis_config.h
@@ -586,6 +586,9 @@ struct PD_INFER_DECL AnalysisConfig {
   /// \param use_static Serialize optimization information to disk for reusing.
   /// \param use_calib_mode Use TRT int8 calibration(post training
   /// quantization).
+  /// \param use_cuda_graph Use CudaGraph to reduce the time consumption of
+  /// enqueue. Note that this option can only be enabled when your input is
+  /// constant (including the batch dimension).
   ///
   ///
   void EnableTensorRtEngine(int64_t workspace_size = 1 << 30,
@@ -593,7 +596,8 @@ struct PD_INFER_DECL AnalysisConfig {
                             int min_subgraph_size = 3,
                             Precision precision = Precision::kFloat32,
                             bool use_static = false,
-                            bool use_calib_mode = true);
+                            bool use_calib_mode = true,
+                            bool use_cuda_graph = false);
   ///
   /// \brief A boolean state telling whether the TensorRT engine is used.
   ///
@@ -1114,6 +1118,7 @@ struct PD_INFER_DECL AnalysisConfig {
   Precision tensorrt_precision_mode_{Precision::kFloat32};
   bool trt_use_static_engine_{false};
   bool trt_use_calib_mode_{true};
+  bool trt_use_cuda_graph_{false};
   bool trt_use_varseqlen_{false};
   bool trt_with_interleaved_{false};
   std::string tensorrt_transformer_posid_{""};
diff --git a/paddle/fluid/inference/tensorrt/engine.cc b/paddle/fluid/inference/tensorrt/engine.cc
index 4710966030d67..b2ee3f4796595 100644
--- a/paddle/fluid/inference/tensorrt/engine.cc
+++ b/paddle/fluid/inference/tensorrt/engine.cc
@@ -25,6 +25,7 @@ limitations under the License. */
 #include "paddle/fluid/platform/device/gpu/gpu_info.h"
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/phi/common/data_type.h"
+#include "paddle/phi/core/enforce.h"
 
 namespace paddle {
 namespace inference {
@@ -129,12 +130,60 @@ void TensorRTEngine::Execute(int batch_size,
                 phi::Stream(reinterpret_cast<phi::StreamId>(stream)));
     infer_context->setDeviceMemory(context_memory);
   }
+
+  // TODO(wilber): Is cudaGraph has conflict with memory sharing?
+  if (startup_with_cudagraph_ && !cudagraph_inited_) {
+    // Avoid capturing initialization calls by executing the enqueue function at
+    // least once before starting CUDA graph capture.
+    const auto ret = Enqueue(infer_context, buffers, batch_size, stream);
+    PADDLE_ENFORCE_EQ(
+        ret,
+        true,
+        phi::errors::PreconditionNotMet("Trt CudaGraph test run failed."));
+    cudaStreamSynchronize(stream);
+
+    cuda_graph_.BeginCapture(stream);
+    // The built TRT engine may contain operations that are not permitted under
+    // CUDA graph capture mode. When the stream is capturing, the call may
+    // return false if the current CUDA graph capture fails.
+    if (Enqueue(infer_context, buffers, batch_size, stream)) {
+      cuda_graph_.EndCapture(stream);
+      cudagraph_inited_ = true;
+    } else {
+      cuda_graph_.EndCaptureOnError(stream);
+      // Ensure any CUDA error has been cleaned up.
+      PADDLE_ENFORCE_GPU_SUCCESS(cudaGetLastError());
+      LOG(WARNING) << "The built TensorRT engine contains operations that are "
+                      "not permitted under "
+                      "CUDA graph capture mode. The specified UseCudaGraph "
+                      "flag has been ignored. The inference will be "
+                      "launched without using CUDA graph launch.";
+      cudagraph_inited_ = false;
+    }
+    startup_with_cudagraph_ = false;
+  }
+
+  Enqueue(infer_context, buffers, batch_size, stream);
+}
+
+bool TensorRTEngine::Enqueue(nvinfer1::IExecutionContext *context,
+                             std::vector<void *> *buffers,
+                             int batch_size,
+                             cudaStream_t stream) {
+  if (cudagraph_inited_) {
+    VLOG(1) << "cuda_graph init success, so we will use cuda graph launch the "
+               "entire graph.";
+    return cuda_graph_.Launch(stream);
+  }
+
+  bool ret;
   if (!with_dynamic_shape()) {
-    infer_context->enqueue(batch_size, buffers->data(), stream, nullptr);
+    ret = context->enqueue(batch_size, buffers->data(), stream, nullptr);
   } else {
-    infer_context->enqueueV2(buffers->data(), stream, nullptr);
+    ret = context->enqueueV2(buffers->data(), stream, nullptr);
   }
   SetRuntimeBatch(batch_size);
+  return ret;
 }
 
 void TensorRTEngine::FreezeNetwork() {
diff --git a/paddle/fluid/inference/tensorrt/engine.h b/paddle/fluid/inference/tensorrt/engine.h
index 8e1531352137f..1906b8be5171e 100644
--- a/paddle/fluid/inference/tensorrt/engine.h
+++ b/paddle/fluid/inference/tensorrt/engine.h
@@ -49,6 +49,64 @@ namespace paddle {
 namespace inference {
 namespace tensorrt {
 
+// The code is mainly from TensorRT, thanks to the project.
+class TrtCudaGraph {
+ public:
+  TrtCudaGraph() = default;
+  ~TrtCudaGraph() {
+    if (cuda_graph_exec_) {
+      cudaGraphExecDestroy(cuda_graph_exec_);
+    }
+  }
+
+  void BeginCapture(cudaStream_t stream) {
+    PADDLE_ENFORCE_GPU_SUCCESS(
+        cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal));
+  }
+
+  bool Launch(cudaStream_t stream) {
+    return cudaGraphLaunch(cuda_graph_exec_, stream);
+  }
+
+  void EndCapture(cudaStream_t stream) {
+    PADDLE_ENFORCE_GPU_SUCCESS(cudaStreamEndCapture(stream, &cuda_graph_));
+    PADDLE_ENFORCE_GPU_SUCCESS(cudaGraphInstantiate(
+        &cuda_graph_exec_, cuda_graph_, nullptr, nullptr, 0));
+    PADDLE_ENFORCE_GPU_SUCCESS(cudaGraphDestroy(cuda_graph_));
+  }
+
+  void EndCaptureOnError(cudaStream_t stream) {
+    // There are two possibilities why stream capture would fail:
+    // (1) stream is in cudaErrorStreamCaptureInvalidated state.
+    // (2) TRT reports a failure.
+    // In case (1), the returning cuda_graph_ should be nullptr.
+    // In case (2), the returning cuda_graph_ is not nullptr, but it should not
+    // be used.
+    const auto ret = cudaStreamEndCapture(stream, &cuda_graph_);
+    if (ret == cudaErrorStreamCaptureInvalidated) {
+      PADDLE_ENFORCE_EQ(cuda_graph_ == nullptr,
+                        true,
+                        platform::errors::PreconditionNotMet(
+                            "CudaGraph capture stream failed."));
+    } else {
+      PADDLE_ENFORCE_GPU_SUCCESS(ret);
+      PADDLE_ENFORCE_NOT_NULL(
+          cuda_graph_,
+          phi::errors::PreconditionNotMet("CudaGraph capture stream failed."));
+      PADDLE_ENFORCE_GPU_SUCCESS(cudaGraphDestroy(cuda_graph_));
+      cuda_graph_ = nullptr;
+    }
+    // Clean up any cuda error.
+    cudaGetLastError();
+    LOG(WARNING) << "The TRT CUDA graph capture on the stream has failed.";
+  }
+
+ private:
+  DISABLE_COPY_AND_ASSIGN(TrtCudaGraph);
+  cudaGraph_t cuda_graph_{};
+  cudaGraphExec_t cuda_graph_exec_{};
+};
+
 namespace plugin {
 class PluginTensorRT;
 }  // namespace plugin
@@ -445,6 +503,11 @@ class TensorRTEngine {
                std::vector<void*>* buffers,
                cudaStream_t stream = nullptr);
 
+  bool Enqueue(nvinfer1::IExecutionContext* context,
+               std::vector<void*>* buffers,
+               int batch,
+               cudaStream_t stream);
+
   nvinfer1::INetworkDefinition* network() { return infer_network_.get(); }
 
   ShapeMapType min_input_shape() { return min_input_shape_; }
@@ -682,6 +745,11 @@ class TensorRTEngine {
     context_memory_sharing_ = context_memory_sharing;
   }
 
+  void SetAllNodesLowerToTrt(bool all_nodes_offload_to_trt) {
+    // all nodes are in trt, so we can use cudaGraph to optimize runtime.
+    startup_with_cudagraph_ = all_nodes_offload_to_trt;
+  }
+
  private:
   // Each ICudaEngine object is bound to a specific GPU when it is instantiated,
   // ensure that the thread is associated with the correct device by calling
@@ -744,6 +812,11 @@ class TensorRTEngine {
   infer_ptr<nvinfer1::IHostMemory> ihost_memory_;
   std::unordered_map<nvinfer1::ITensor*, float> quant_dynamic_range_;
 
+  // cudagraph related
+  TrtCudaGraph cuda_graph_;
+  bool cudagraph_inited_{false};
+  bool startup_with_cudagraph_{false};
+
   std::unordered_map<std::string, paddle::any> attrs_;
   std::unordered_map<std::string, std::function<void(void)>> attr_dels_;
 #if IS_TRT_VERSION_GE(6000)
diff --git a/paddle/fluid/inference/tensorrt/test_engine.cc b/paddle/fluid/inference/tensorrt/test_engine.cc
index 9a06b2e65ef10..23a0df7316724 100644
--- a/paddle/fluid/inference/tensorrt/test_engine.cc
+++ b/paddle/fluid/inference/tensorrt/test_engine.cc
@@ -274,6 +274,7 @@ TEST_F(TensorRTEngineTest, test_pool2d) {
   buffers[0] = reinterpret_cast<void *>(x_v_gpu_data);
   buffers[1] = reinterpret_cast<void *>(y_gpu_data);
 
+  engine_->SetAllNodesLowerToTrt(true);
   engine_->Execute(2, &buffers, ctx_->stream());
 
   LOG(INFO) << "to get output";
diff --git a/paddle/fluid/pybind/inference_api.cc b/paddle/fluid/pybind/inference_api.cc
index a8bfd5a9917e7..e861c5b5bbe90 100644
--- a/paddle/fluid/pybind/inference_api.cc
+++ b/paddle/fluid/pybind/inference_api.cc
@@ -840,7 +840,8 @@ void BindAnalysisConfig(py::module *m) {
            py::arg("min_subgraph_size") = 3,
            py::arg("precision_mode") = AnalysisConfig::Precision::kFloat32,
            py::arg("use_static") = false,
-           py::arg("use_calib_mode") = true)
+           py::arg("use_calib_mode") = true,
+           py::arg("use_cuda_graph") = false)
       .def("enable_tensorrt_memory_optim",
            &AnalysisConfig::EnableTensorRTMemoryOptim,
            py::arg("engine_memory_sharing") = true,
diff --git a/test/cpp/inference/api/CMakeLists.txt b/test/cpp/inference/api/CMakeLists.txt
index 01010715c8104..c4045e9ca603a 100644
--- a/test/cpp/inference/api/CMakeLists.txt
+++ b/test/cpp/inference/api/CMakeLists.txt
@@ -1377,7 +1377,7 @@ if(WITH_TESTING AND WITH_INFERENCE_API_TEST)
     set_tests_properties(test_analyzer_ernie PROPERTIES TIMEOUT 120)
   endif()
   if(WITH_GPU AND TENSORRT_FOUND)
-    set_tests_properties(trt_mobilenet_test PROPERTIES TIMEOUT 120)
+    set_tests_properties(trt_mobilenet_test PROPERTIES TIMEOUT 240)
     if(WITH_MKLDNN)
       set_tests_properties(test_analyzer_bfloat16_resnet50 PROPERTIES TIMEOUT
                                                                       120)
diff --git a/test/cpp/inference/api/trt_mobilenet_test.cc b/test/cpp/inference/api/trt_mobilenet_test.cc
index 5c0519c067f56..7cae99e0d3479 100644
--- a/test/cpp/inference/api/trt_mobilenet_test.cc
+++ b/test/cpp/inference/api/trt_mobilenet_test.cc
@@ -99,4 +99,30 @@ TEST(PredictorPool, use_gpu) {
   predictor->Run();
 }
 
+TEST(PredictorPool, use_trt_cuda_graph) {
+  std::string model_dir = FLAGS_infer_model + "/" + "mobilenet";
+  Config config;
+  config.EnableUseGpu(100, 0);
+  config.SetModel(model_dir);
+  config.EnableTensorRtEngine(
+      1 << 20, 1, 3, PrecisionType::kFloat32, false, false, true);
+  config.Exp_DisableTensorRtOPs({"fc"});
+  config.EnableTensorRtDLA(0);
+  services::PredictorPool pred_pool(config, 1);
+
+  auto predictor = pred_pool.Retrive(0);
+  auto input_names = predictor->GetInputNames();
+  auto input_t = predictor->GetInputHandle(input_names[0]);
+  std::vector<int> in_shape = {1, 3, 224, 224};
+  int in_num =
+      std::accumulate(in_shape.begin(), in_shape.end(), 1, [](int &a, int &b) {
+        return a * b;
+      });
+
+  std::vector<float> input(in_num, 0);
+  input_t->Reshape(in_shape);
+  input_t->CopyFromCpu(input.data());
+  predictor->Run();
+}
+
 }  // namespace paddle_infer

From dd90f10252fa47aabebf4ddcf6975b8f37bd5687 Mon Sep 17 00:00:00 2001
From: Sanbu <96160062+sanbuphy@users.noreply.github.com>
Date: Tue, 9 May 2023 16:23:45 +0800
Subject: [PATCH 234/405] Support static graph code-gen for unpool3d (#53479)

---
 paddle/fluid/operators/unpool_op.cc      | 179 -----------------------
 paddle/phi/api/yaml/backward.yaml        |  11 ++
 paddle/phi/api/yaml/legacy_backward.yaml |  11 --
 paddle/phi/api/yaml/legacy_ops.yaml      |  10 --
 paddle/phi/api/yaml/op_compat.yaml       |   6 +
 paddle/phi/api/yaml/ops.yaml             |  10 ++
 paddle/phi/ops/compat/unpool3d_sig.cc    |  37 -----
 7 files changed, 27 insertions(+), 237 deletions(-)
 delete mode 100644 paddle/fluid/operators/unpool_op.cc
 delete mode 100644 paddle/phi/ops/compat/unpool3d_sig.cc

diff --git a/paddle/fluid/operators/unpool_op.cc b/paddle/fluid/operators/unpool_op.cc
deleted file mode 100644
index 62441dd6127e3..0000000000000
--- a/paddle/fluid/operators/unpool_op.cc
+++ /dev/null
@@ -1,179 +0,0 @@
-/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-Indicesou may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/phi/infermeta/backward.h"
-#include "paddle/phi/infermeta/binary.h"
-
-namespace paddle {
-namespace operators {
-
-class Unpool3dOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput(
-        "X",
-        "(Tensor) The input tensor of unpool operator. "
-        "The format of input tensor is NCDHW. Where N is batch size, C is the "
-        "number of channels, D, H and W is the depth, height and width of "
-        "feature.");
-    AddInput(
-        "Indices",
-        "(Tensor) The input tensor of the indices given out by MaxPool3d. "
-        "The format of input tensor is NCDHW. Where N is batch size, C is the "
-        "number of channels, D, H and W is the depth, height and width of "
-        "feature.");
-    AddOutput("Out",
-              "(Tensor) The output tensor of unpool operator."
-              "The format of output tensor is also NCDHW."
-              "Where N is batch size, C is "
-              "the number of channels, D, H and W is the depth, height and "
-              "width of feature.");
-    AddAttr<std::vector<int>>(
-        "ksize",
-        "(vector), the unpooling window size(depth, height, width) "
-        "of unpooling operator.");
-    AddAttr<std::vector<int>>(
-        "strides",
-        "(vector, default:{1, 1, 1}), "
-        "strides (depth, height, width) of unpooling operator.")
-        .SetDefault({1, 1, 1});
-    AddAttr<std::vector<int>>(
-        "paddings",
-        "(vector default:{0, 0,0}), "
-        "paddings (depth, height, width) of unpooling operator.")
-        .SetDefault({0, 0, 0});
-    AddAttr<std::string>(
-        "unpooling_type",
-        "(string), unpooling type, can be \"max\" for max-unpooling ")
-        .InEnum({"max"});
-    AddAttr<std::vector<int>>("output_size",
-                              "(vector, optional). The shape of output.")
-        .SetDefault({0, 0, 0});
-    AddAttr<std::string>(
-        "data_format",
-        "(string, default NCDHW)"
-        "Defaults to \"NCDHW\". Specify the data format of the output data, ")
-        .SetDefault("NCDHW");
-    AddComment(R"DOC(
-Input shape is: $(N, C_{in}, D_{in}, H_{in}, W_{in})$, Output shape is:
-$(N, C_{out}, D_{out}, H_{out}, W_{out})$, where
-$$
-D_{out} = (D_{in}-1) * strides[0] - 2 * paddings[0] + ksize[0] \\
-H_{out} = (H_{in}-1) * strides[1] - 2 * paddings[1] + ksize[1] \\
-W_{out} = (W_{in}-1) * strides[2] - 2 * paddings[2] + ksize[2]
-$$
-)DOC");
-  }
-};
-
-int UnpoolOutputSize(int input_size, int ksize, int padding, int stride) {
-  int output_size = (input_size - 1) * stride - 2 * padding + ksize;
-  return output_size;
-}
-
-class UnpoolOp : public framework::OperatorWithKernel {
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(ctx, "X"),
-                          ctx.GetPlace());
-  }
-
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-};
-
-class Unpool3dOp : public framework::OperatorWithKernel {
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(ctx, "X"),
-                          ctx.GetPlace());
-  }
-
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-};
-
-template <typename T>
-class UnpoolOpGradMaker : public framework::SingleGradOpMaker<T> {
- public:
-  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
-  void Apply(GradOpPtr<T> op) const override {
-    op->SetType(this->ForwardOpType() + "_grad");
-    op->SetInput("X", this->Input("X"));
-    op->SetInput("Indices", this->Input("Indices"));
-    op->SetInput("Out", this->Output("Out"));
-    op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out"));
-    op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
-    op->SetAttrMap(this->Attrs());
-  }
-};
-
-template <typename T>
-class Unpool3dOpGradMaker : public framework::SingleGradOpMaker<T> {
- public:
-  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
-  void Apply(GradOpPtr<T> op) const override {
-    op->SetType(this->ForwardOpType() + "_grad");
-    op->SetInput("X", this->Input("X"));
-    op->SetInput("Indices", this->Input("Indices"));
-    op->SetInput("Out", this->Output("Out"));
-    op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out"));
-    op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
-    op->SetAttrMap(this->Attrs());
-  }
-};
-
-class Unpool3dOpGrad : public framework::OperatorWithKernel {
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(ctx, "X"),
-                          ctx.GetPlace());
-  }
-
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-
-DECLARE_INFER_SHAPE_FUNCTOR(unpool,
-                            Unpool3dInferShapeFunctor,
-                            PD_INFER_META(phi::Unpool3dInferMeta));
-
-REGISTER_OPERATOR(unpool3d,
-                  ops::Unpool3dOp,
-                  ops::Unpool3dOpMaker,
-                  ops::Unpool3dOpGradMaker<paddle::framework::OpDesc>,
-                  ops::Unpool3dOpGradMaker<paddle::imperative::OpBase>,
-                  Unpool3dInferShapeFunctor);
-
-DECLARE_INFER_SHAPE_FUNCTOR(unpool3d_grad,
-                            Unpool3dGradInferShapeFunctor,
-                            PD_INFER_META(phi::UnchangedInferMeta));
-
-REGISTER_OPERATOR(unpool3d_grad,
-                  ops::Unpool3dOpGrad,
-                  Unpool3dGradInferShapeFunctor);
diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index 0e348847554c2..7b3b0d225af10 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -2065,6 +2065,17 @@
     func : where_grad
   no_need_buffer : x, y
 
+- backward_op: unpool3d_grad
+  forward: unpool3d (Tensor x, Tensor indices, int[] ksize, int[] strides={1,1,1}, int[] paddings={0,0,0}, int[] output_size={0,0,0}, str data_format="NCDHW") -> Tensor(out)
+  args: (Tensor x, Tensor indices, Tensor out, Tensor out_grad, int[] ksize, int[] strides, int[] paddings, int[] output_size, str data_format)
+  output: Tensor(x_grad)
+  infer_meta:
+    func: UnchangedInferMeta
+    param : [x]
+  kernel:
+    func: unpool3d_grad
+    data_type: x
+
 - backward_op: unpool_grad
   forward: unpool (Tensor x, Tensor indices, int[] ksize, int[] strides = {1,1}, int[] paddings ={0,0} ,IntArray output_size = {0,0}, str data_format="NCHW") -> Tensor(out)
   args: (Tensor x, Tensor indices, Tensor out, Tensor out_grad, int[] ksize, int[] strides, int[] paddings, IntArray output_size, str data_format)
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index 8a3f0c15b677f..b844f76290c0e 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -1042,14 +1042,3 @@
   kernel :
     func : yolo_loss_grad
   optional : gt_score
-
-- backward_op: unpool3d_grad
-  forward: unpool3d (Tensor x, Tensor indices, int[] ksize, int[] strides, int[] padding, int[] output_size, str data_format) -> Tensor(out)
-  args: (Tensor x, Tensor indices, Tensor out, Tensor out_grad, int[] ksize, int[] strides, int[] padding, int[] output_size, str data_format)
-  output: Tensor(x_grad)
-  infer_meta:
-    func: UnchangedInferMeta
-    param : [x]
-  kernel:
-    func: unpool3d_grad
-    data_type: x
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index e941b54fe5030..266e981254f36 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -1207,16 +1207,6 @@
     func : unique
     data_type : x
 
-- op : unpool3d
-  args: (Tensor x, Tensor indices, int[] ksize, int[] strides, int[] padding, int[] output_size, str data_format)
-  output: Tensor(out)
-  infer_meta:
-    func: Unpool3dInferMeta
-  kernel:
-    func: unpool3d
-    data_type: x
-  backward: unpool3d_grad
-
 - op : yolo_loss
   args : (Tensor x, Tensor gt_box, Tensor gt_label, Tensor gt_score, int[] anchors, int[] anchor_mask, int class_num, float ignore_thresh, int downsample_ratio, bool use_label_smooth=true, float scale_x_y=1.0)
   output : Tensor(loss), Tensor(objectness_mask), Tensor(gt_match_mask)
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index 23e509e5dd1a4..97cca3ced8078 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -2431,6 +2431,12 @@
       data_type : int
       support_tensor : true
 
+- op : unpool3d
+  inputs :
+    {x : X, indices: Indices}
+  outputs :
+    out : Out
+
 - op : unsqueeze (unsqueeze2)
   backward : unsqueeze_grad (unsqueeze2_grad), unsqueeze_double_grad(unsqueeze2_double_grad)
   inputs :
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index b2c262861ba16..2f0960e9d30c5 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -2153,6 +2153,16 @@
     data_type: x
   backward: unpool_grad
 
+- op : unpool3d
+  args: (Tensor x, Tensor indices, int[] ksize, int[] strides={1,1,1}, int[] paddings={0,0,0}, int[] output_size={0,0,0}, str data_format="NCDHW")
+  output: Tensor(out)
+  infer_meta:
+    func: Unpool3dInferMeta
+  kernel:
+    func: unpool3d
+    data_type: x
+  backward: unpool3d_grad
+
 - op : unsqueeze
   args : (Tensor x, IntArray axis = {})
   output : Tensor(out), Tensor(xshape)
diff --git a/paddle/phi/ops/compat/unpool3d_sig.cc b/paddle/phi/ops/compat/unpool3d_sig.cc
deleted file mode 100644
index c73aca837d57b..0000000000000
--- a/paddle/phi/ops/compat/unpool3d_sig.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/core/compat/op_utils.h"
-namespace phi {
-
-KernelSignature Unpool3dOpArgumentMapping(const ArgumentMappingContext& ctx) {
-  return KernelSignature(
-      "unpool3d",
-      {"X", "Indices"},
-      {"ksize", "strides", "paddings", "output_size", "data_format"},
-      {"Out"});
-}
-
-KernelSignature Unpool3dGradOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature(
-      "unpool3d_grad",
-      {"X", "Indices", "Out", "Out@GRAD"},
-      {"ksize", "strides", "paddings", "output_size", "data_format"},
-      {"X@GRAD"});
-}
-
-}  // namespace phi
-PD_REGISTER_ARG_MAPPING_FN(unpool3d, phi::Unpool3dOpArgumentMapping);
-PD_REGISTER_ARG_MAPPING_FN(unpool3d_grad, phi::Unpool3dGradOpArgumentMapping);

From 0f1b077b4f371b34c716d397c0f7f83785feaffd Mon Sep 17 00:00:00 2001
From: YUNSHEN XIE <1084314248@qq.com>
Date: Tue, 9 May 2023 16:24:07 +0800
Subject: [PATCH 235/405] fix ut timeout, test=document_fix (#53629)

---
 python/paddle/fluid/tests/unittests/collective/CMakeLists.txt | 2 +-
 python/paddle/fluid/tests/unittests/collective/testslist.csv  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt b/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt
index ae17fbb193829..285aa0d8d9ef6 100644
--- a/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt
@@ -225,7 +225,7 @@ if((WITH_GPU OR WITH_ROCM) AND (LINUX))
     test_collective_reduce_scatter_api ENVS
     "http_proxy=;https_proxy=;PYTHONPATH=..:${PADDLE_BINARY_DIR}/python")
   set_tests_properties(test_collective_reduce_scatter_api
-                       PROPERTIES TIMEOUT "120" LABELS "RUN_TYPE=DIST")
+                       PROPERTIES TIMEOUT "150" LABELS "RUN_TYPE=DIST")
 endif()
 if((WITH_GPU OR WITH_ROCM) AND (LINUX))
   py_test_modules(
diff --git a/python/paddle/fluid/tests/unittests/collective/testslist.csv b/python/paddle/fluid/tests/unittests/collective/testslist.csv
index 7d8e32998232e..aeeb981d67f0a 100644
--- a/python/paddle/fluid/tests/unittests/collective/testslist.csv
+++ b/python/paddle/fluid/tests/unittests/collective/testslist.csv
@@ -24,7 +24,7 @@ test_collective_process_group,linux,gpu;rocm,350,DIST,../dist_test.sh,2,,http_pr
 test_collective_reduce,linux,gpu;rocm,300,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=..,
 test_collective_reduce_api,linux,gpu;rocm,500,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=..,
 test_collective_reduce_scatter,linux,gpu;rocm,350,DIST,../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=..,
-test_collective_reduce_scatter_api,linux,gpu;rocm,120,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=..,
+test_collective_reduce_scatter_api,linux,gpu;rocm,150,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=..,
 test_collective_scatter,linux,gpu;rocm,300,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=..,
 test_collective_scatter_api,linux,gpu;rocm,180,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=..,
 test_collective_scatter_object_list_api,linux,gpu;rocm,120,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=..,

From 14c642cb97250e3c0d40e714df2d12184329d37e Mon Sep 17 00:00:00 2001
From: Zhan Rongrui <46243324+zrr1999@users.noreply.github.com>
Date: Tue, 9 May 2023 17:15:56 +0800
Subject: [PATCH 236/405] =?UTF-8?q?=E3=80=90Hackathon=204th=20No.30?=
 =?UTF-8?q?=E3=80=91=E4=B8=BA=20Paddle=20=E6=96=B0=E5=A2=9E=20paddle.spars?=
 =?UTF-8?q?e.sum=20=E7=A8=80=E7=96=8F=20API=20=20(#51406)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 paddle/phi/api/yaml/sparse_backward.yaml      |  11 +
 paddle/phi/api/yaml/sparse_ops.yaml           |  11 +
 .../phi/kernels/cpu/reduce_sum_grad_kernel.cc |   1 +
 .../phi/kernels/gpu/reduce_sum_grad_kernel.cu |   1 +
 .../phi/kernels/sparse/cpu/sum_grad_kernel.cc | 219 +++++++++
 paddle/phi/kernels/sparse/cpu/sum_kernel.cc   | 283 +++++++++++
 .../phi/kernels/sparse/gpu/sum_grad_kernel.cu | 235 +++++++++
 paddle/phi/kernels/sparse/gpu/sum_kernel.cu   | 456 ++++++++++++++++++
 paddle/phi/kernels/sparse/unary_grad_kernel.h |  17 +
 paddle/phi/kernels/sparse/unary_kernel.h      |  16 +
 .../tests/unittests/test_sparse_sum_op.py     | 204 ++++++++
 python/paddle/sparse/__init__.py              |   2 +
 python/paddle/sparse/unary.py                 |  89 +++-
 13 files changed, 1544 insertions(+), 1 deletion(-)
 create mode 100644 paddle/phi/kernels/sparse/cpu/sum_grad_kernel.cc
 create mode 100644 paddle/phi/kernels/sparse/cpu/sum_kernel.cc
 create mode 100644 paddle/phi/kernels/sparse/gpu/sum_grad_kernel.cu
 create mode 100644 paddle/phi/kernels/sparse/gpu/sum_kernel.cu
 create mode 100644 python/paddle/fluid/tests/unittests/test_sparse_sum_op.py

diff --git a/paddle/phi/api/yaml/sparse_backward.yaml b/paddle/phi/api/yaml/sparse_backward.yaml
index c541129f7ffbb..a18157ce8f7e3 100644
--- a/paddle/phi/api/yaml/sparse_backward.yaml
+++ b/paddle/phi/api/yaml/sparse_backward.yaml
@@ -367,6 +367,17 @@
     func : subtract_coo_coo_grad{sparse_coo, sparse_coo, sparse_coo -> sparse_coo, sparse_coo},
            subtract_csr_csr_grad{sparse_csr, sparse_csr, sparse_csr -> sparse_csr, sparse_csr}
 
+- backward_op : sum_grad
+  forward : sum(Tensor x, IntArray axis={}, DataType dtype=DataType::UNDEFINED, bool keepdim=false) -> Tensor(out)
+  args : (Tensor x, Tensor out_grad, IntArray axis={}, bool keepdim=false)
+  output : Tensor(x_grad)
+  infer_meta :
+    func : UnchangedInferMeta
+    param : [x]
+  kernel :
+    func : sum_coo_grad {sparse_coo, sparse_coo -> sparse_coo},
+           sum_csr_grad {sparse_csr, sparse_csr -> sparse_csr}
+
 - backward_op : sync_batch_norm_grad
   forward : sync_batch_norm_(Tensor x, Tensor mean, Tensor variance, Tensor scale, Tensor bias, bool is_test, float momentum, float epsilon, str data_layout, bool use_global_stats, bool trainable_statistics) -> Tensor(out), Tensor(mean_out), Tensor(variance_out), Tensor(saved_mean), Tensor(saved_variance), Tensor(reserve_space)
   args : (Tensor x, Tensor scale, Tensor bias, Tensor saved_mean, Tensor saved_variance, Tensor reserve_space, Tensor out_grad, float momentum, float epsilon, str data_layout, bool is_test, bool use_global_stats, bool trainable_statistics)
diff --git a/paddle/phi/api/yaml/sparse_ops.yaml b/paddle/phi/api/yaml/sparse_ops.yaml
index 85b64e867eef8..41d4aedd66d1b 100644
--- a/paddle/phi/api/yaml/sparse_ops.yaml
+++ b/paddle/phi/api/yaml/sparse_ops.yaml
@@ -334,6 +334,17 @@
     layout : x
   backward : subtract_grad
 
+- op : sum
+  args : (Tensor x, IntArray axis={}, DataType dtype=DataType::UNDEFINED, bool keepdim=false)
+  output : Tensor(out)
+  infer_meta :
+    func : SumInferMeta
+  kernel :
+    func : sum_coo{sparse_coo -> sparse_coo},
+           sum_csr{sparse_csr -> sparse_csr}
+    data_type : x
+  backward : sum_grad
+
 - op : sync_batch_norm_
   args : (Tensor x, Tensor mean, Tensor variance, Tensor scale, Tensor bias, bool is_test, float momentum, float epsilon, str data_layout, bool use_global_stats, bool trainable_statistics)
   output : Tensor(out), Tensor(mean_out), Tensor(variance_out), Tensor(saved_mean), Tensor(saved_variance), Tensor(reserve_space)
diff --git a/paddle/phi/kernels/cpu/reduce_sum_grad_kernel.cc b/paddle/phi/kernels/cpu/reduce_sum_grad_kernel.cc
index 8a5e3812950ec..0771a3e9e72f4 100644
--- a/paddle/phi/kernels/cpu/reduce_sum_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/reduce_sum_grad_kernel.cc
@@ -51,6 +51,7 @@ PD_REGISTER_KERNEL(sum_grad,
                    float,
                    double,
                    phi::dtype::float16,
+                   int16_t,
                    int,
                    int64_t,
                    phi::dtype::complex<float>,
diff --git a/paddle/phi/kernels/gpu/reduce_sum_grad_kernel.cu b/paddle/phi/kernels/gpu/reduce_sum_grad_kernel.cu
index 9ee6d530374db..3e88506f723d3 100644
--- a/paddle/phi/kernels/gpu/reduce_sum_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/reduce_sum_grad_kernel.cu
@@ -67,6 +67,7 @@ PD_REGISTER_KERNEL(sum_grad,
                    double,
                    phi::dtype::float16,
                    phi::dtype::bfloat16,
+                   int16_t,
                    int,
                    int64_t,
                    phi::dtype::complex<float>,
diff --git a/paddle/phi/kernels/sparse/cpu/sum_grad_kernel.cc b/paddle/phi/kernels/sparse/cpu/sum_grad_kernel.cc
new file mode 100644
index 0000000000000..05eb975b7e24e
--- /dev/null
+++ b/paddle/phi/kernels/sparse/cpu/sum_grad_kernel.cc
@@ -0,0 +1,219 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/sparse/unary_grad_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/visit_type.h"
+#include "paddle/phi/kernels/funcs/math_function.h"
+#include "paddle/phi/kernels/reduce_sum_grad_kernel.h"
+#include "paddle/phi/kernels/sparse/empty_kernel.h"
+#include "paddle/phi/kernels/sparse/impl/unary_grad_kernel_impl.h"
+
+namespace phi {
+namespace sparse {
+
+template <typename T, typename IntT, typename Context>
+void SumCooGradCPUKernel(const Context& dev_ctx,
+                         const SparseCooTensor& x,
+                         const SparseCooTensor& dout,
+                         const IntArray& axis,
+                         bool keep_dim,
+                         SparseCooTensor* dx) {
+  EmptyLikeCooKernel<T, Context>(dev_ctx, x, dx);
+  unsigned int n_dim = axis.size();
+
+  const DenseTensor& x_indices = x.indices();
+  const DenseTensor& dout_indices = dout.indices();
+  const DenseTensor& dout_values = dout.values();
+  const auto* dout_indices_data = dout_indices.data<int64_t>();
+  const auto* dout_values_data = dout_values.data<T>();
+
+  DenseTensor* dx_indices = dx->mutable_indices();
+  DenseTensor* dx_values = dx->mutable_values();
+  *dx_indices = x_indices;
+
+  const auto* dx_indices_data = dx_indices->data<int64_t>();
+  auto* dx_values_data = dx_values->data<T>();
+
+  phi::funcs::SetConstant<Context, T> set_constant;
+  if (n_dim == 0) {
+    T value = dout_values.data<T>()[0];
+    set_constant(dev_ctx, dx_values, value);
+    if (dx_values->dtype() != dx->dtype()) {
+      *dx_values = phi::Cast<T, Context>(dev_ctx, *dx_values, dx->dtype());
+    }
+    return;
+  }
+
+  auto dim = axis[0] < 0 ? x.dims().size() + axis[0] : axis[0];
+  auto sparse_dim = x.sparse_dim();
+  if (dim >= sparse_dim) {
+    dim = dim - sparse_dim + 1;
+    phi::ReduceSumGradKernel<T, Context>(
+        dev_ctx, x.values(), dout.values(), {dim}, keep_dim, false, dx_values);
+    if (dx_values->dtype() != dx->dtype()) {
+      *dx_values = phi::Cast<T, Context>(dev_ctx, *dx_values, dx->dtype());
+    }
+    return;
+  }
+  // Ensure the sparse_dim is not less than 1.
+  if (sparse_dim == 1) {
+    keep_dim = true;
+  }
+
+  int64_t dense_dim = 1;
+  for (auto i = 1; i < x.values().dims().size(); ++i) {
+    dense_dim *= x.values().dims()[i];
+  }
+
+  std::map<std::vector<IntT>, int64_t> indices_map;
+  for (auto j = 0; j < dout_indices.dims()[1]; ++j) {
+    std::vector<IntT> pos;
+    for (int i = 0; i < dout_indices.dims()[0]; ++i) {
+      pos.push_back(dout_indices_data[j + i * dout_indices.dims()[1]]);
+    }
+    indices_map[pos] = j;
+  }
+
+  for (auto j = 0; j < dx_indices->dims()[1]; ++j) {
+    std::vector<IntT> pos;
+    for (int i = 0; i < dx_indices->dims()[0]; ++i) {
+      if (i != dim) {
+        pos.push_back(dx_indices_data[j + i * dx_indices->dims()[1]]);
+      } else if (keep_dim) {
+        pos.push_back(0);
+      }
+    }
+    for (int i = 0; i < dense_dim; ++i) {
+      dx_values_data[i + j * dense_dim] =
+          dout_values_data[i + indices_map[pos] * dense_dim];
+    }
+  }
+  if (dx_values->dtype() != dx->dtype()) {
+    *dx_values = phi::Cast<T, Context>(dev_ctx, *dx_values, dx->dtype());
+  }
+}
+
+template <typename T, typename Context>
+void SumCsrGradKernel(const Context& dev_ctx,
+                      const SparseCsrTensor& x,
+                      const SparseCsrTensor& dout,
+                      const IntArray& axis,
+                      bool keep_dim,
+                      SparseCsrTensor* dx) {
+  EmptyLikeCsrKernel<T, Context>(dev_ctx, x, dx);
+  unsigned int n_dim = axis.size();
+
+  const DenseTensor& x_crows = x.crows();
+  const DenseTensor& x_cols = x.cols();
+  const DenseTensor& dout_values = dout.values();
+  const auto* x_crows_data = x_crows.data<int64_t>();
+
+  DenseTensor* dx_crows = dx->mutable_crows();
+  DenseTensor* dx_cols = dx->mutable_cols();
+  DenseTensor* dx_values = dx->mutable_values();
+
+  *dx_crows = x_crows;
+  *dx_cols = x_cols;
+
+  phi::funcs::SetConstant<Context, T> set_constant;
+  if (n_dim == 0) {
+    T value = dout_values.data<T>()[0];
+    set_constant(dev_ctx, dx_values, value);
+    if (dx_values->dtype() != dx->dtype()) {
+      *dx_values = phi::Cast<T, Context>(dev_ctx, *dx_values, dx->dtype());
+    }
+    return;
+  }
+  PADDLE_ENFORCE_EQ(axis[0],
+                    -1,
+                    phi::errors::Unimplemented(
+                        "`axis` of SumCsrKernel only support None or -1 now."
+                        "More number will be supported in the future."));
+
+  if (x.dims().size() == 2) {
+    int value_index = 0;
+    for (int k = 0; k < x.dims()[0]; ++k) {
+      if (x_crows_data[k] == x_crows_data[k + 1]) {
+        continue;
+      }
+      T value = dout_values.data<T>()[value_index];
+      set_constant(dev_ctx, dx_values, value);
+      value_index += 1;
+    }
+  } else {
+    int dout_value_index = 0;
+    int dx_value_index = 0;
+    for (auto batch = 0; batch < x.dims()[0]; ++batch) {
+      for (auto k = batch * (x.dims()[1] + 1);
+           k < batch * (x.dims()[1] + 1) + x.dims()[1];
+           ++k) {
+        if (x_crows_data[k] == x_crows_data[k + 1]) {
+          continue;
+        }
+        T value = dout_values.data<T>()[dout_value_index];
+        for (auto i = x_crows_data[k]; i < x_crows_data[k + 1]; ++i) {
+          dx_values->data<T>()[dx_value_index] = value;
+          dx_value_index++;
+        }
+        dout_value_index++;
+      }
+    }
+  }
+
+  if (dx_values->dtype() != dx->dtype()) {
+    *dx_values = phi::Cast<T, Context>(dev_ctx, *dx_values, dx->dtype());
+  }
+}
+
+template <typename T, typename Context>
+void SumCooGradKernel(const Context& dev_ctx,
+                      const SparseCooTensor& x,
+                      const SparseCooTensor& dout,
+                      const IntArray& axis,
+                      bool keep_dim,
+                      SparseCooTensor* dx) {
+  PD_VISIT_BASE_INTEGRAL_TYPES(
+      x.indices().dtype(), "SumCooGradCPUKernel", ([&] {
+        SumCooGradCPUKernel<T, data_t, Context>(
+            dev_ctx, x, dout, axis, keep_dim, dx);
+      }));
+}
+
+}  // namespace sparse
+}  // namespace phi
+
+PD_REGISTER_KERNEL(sum_coo_grad,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::sparse::SumCooGradKernel,
+                   float,
+                   double,
+                   int16_t,
+                   int,
+                   int64_t,
+                   bool) {}
+
+PD_REGISTER_KERNEL(sum_csr_grad,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::sparse::SumCsrGradKernel,
+                   float,
+                   double,
+                   int16_t,
+                   int,
+                   int64_t,
+                   bool) {}
diff --git a/paddle/phi/kernels/sparse/cpu/sum_kernel.cc b/paddle/phi/kernels/sparse/cpu/sum_kernel.cc
new file mode 100644
index 0000000000000..2b4b11bea89e4
--- /dev/null
+++ b/paddle/phi/kernels/sparse/cpu/sum_kernel.cc
@@ -0,0 +1,283 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/sparse/unary_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/visit_type.h"
+#include "paddle/phi/kernels/cast_kernel.h"
+#include "paddle/phi/kernels/empty_kernel.h"
+#include "paddle/phi/kernels/reduce_sum_kernel.h"
+#include "paddle/phi/kernels/sparse/empty_kernel.h"
+
+namespace phi {
+namespace sparse {
+
+template <typename T, typename IntT, typename Context>
+void SumCooCPUKernel(const Context& dev_ctx,
+                     const SparseCooTensor& x,
+                     const IntArray& axis,
+                     DataType dtype,
+                     bool keep_dim,
+                     SparseCooTensor* out) {
+  size_t n_dim = axis.size();
+  auto sparse_dim = x.sparse_dim();
+  // create out sparse tensor
+  const auto& x_dims = x.dims();
+  const auto& x_indices = x.indices();
+  const auto& x_values = x.values();
+  DDim out_dims;
+  DenseTensor out_indices;
+  DenseTensor out_values;
+  if (n_dim == 0) {
+    std::vector<int64_t> out_indices_shape;
+    if (keep_dim) {
+      out_dims = make_ddim(std::vector<int64_t>(x_dims.size(), 1));
+      out_indices_shape = {sparse_dim, 1};
+    } else {
+      out_dims = make_ddim({1});
+      out_indices_shape = {1};
+    }
+    out_indices = Empty<IntT, Context>(dev_ctx, out_indices_shape);
+    auto* out_indices_data = out_indices.data<IntT>();
+    std::fill(out_indices_data, out_indices_data + out_indices.numel(), 0);
+    out_values = phi::Sum<T>(dev_ctx, x.values(), {}, dtype, keep_dim);
+    out->SetMember(out_indices, out_values, out_dims, x.coalesced());
+    return;
+  }
+
+  auto dim = axis[0] < 0 ? x_dims.size() + axis[0] : axis[0];
+  const auto* x_indices_data = x_indices.data<IntT>();
+  const auto* x_values_data = x_values.data<T>();
+
+  std::vector<int64_t> dims;
+  for (int i = 0; i < x.dims().size(); ++i) {
+    if (i != dim) {
+      dims.emplace_back(x.dims()[i]);
+    } else if (keep_dim || (dim < sparse_dim && sparse_dim == 1)) {
+      dims.emplace_back(1);
+    }
+  }
+  out_dims = make_ddim(dims);
+
+  if (dim >= sparse_dim) {
+    out_indices = x_indices;
+    dim = dim - sparse_dim + 1;
+    out_values = phi::Sum<T>(dev_ctx, x.values(), {dim}, dtype, keep_dim);
+    out->SetMember(out_indices, out_values, out_dims, x.coalesced());
+    return;
+  }
+
+  // Ensure the sparse_dim is not less than 1.
+  if (sparse_dim == 1) {
+    keep_dim = true;
+  }
+  // if axis in sparse_dim and keep_dim, sparse_dim will be reduced.
+  if (!keep_dim) {
+    sparse_dim -= 1;
+  }
+
+  // indices_map is a mapping from output's position to values to be summed.
+  std::map<std::vector<IntT>, std::vector<int64_t>> indices_map;
+  for (int64_t j = 0; j < x_indices.dims()[1]; ++j) {
+    std::vector<IntT> pos;
+    for (int64_t i = 0; i < x_indices.dims()[0]; ++i) {
+      if (dim != i) {
+        pos.emplace_back(x_indices_data[j + i * x_indices.dims()[1]]);
+      } else if (keep_dim) {
+        pos.emplace_back(0);
+      }
+    }
+    indices_map[pos].emplace_back(j);
+  }
+
+  std::vector<int> out_values_dims;
+  out_values_dims.push_back(static_cast<int>(indices_map.size()));
+  for (auto i = 1; i < x.values().dims().size(); ++i) {
+    out_values_dims.push_back(static_cast<int>(x.values().dims()[i]));
+  }
+  int64_t dense_dim = std::accumulate(out_values_dims.begin() + 1,
+                                      out_values_dims.end(),
+                                      1,
+                                      std::multiplies<int64_t>());
+
+  out_indices = Empty<IntT, Context>(
+      dev_ctx, {sparse_dim, static_cast<int>(indices_map.size())});
+  out_values = Empty<T, Context>(dev_ctx, out_values_dims);
+
+  auto* out_indices_data = out_indices.data<IntT>();
+  auto* out_values_data = out_values.data<T>();
+
+  auto iter_indices_map = indices_map.begin();
+  for (size_t j = 0; j < indices_map.size(); ++j) {
+    std::vector<IntT> pos = iter_indices_map->first;
+    std::vector<int64_t> values_index = iter_indices_map->second;
+    iter_indices_map++;
+    for (auto i = 0; i < sparse_dim; ++i) {
+      out_indices_data[j + i * indices_map.size()] = pos[i];
+    }
+    for (auto i = 0; i < dense_dim; ++i) {
+      T out_value = 0;
+      for (auto index : values_index) {
+        out_value += x_values_data[i + index * dense_dim];
+      }
+      out_values_data[i + j * dense_dim] = out_value;
+    }
+  }
+
+  if (dtype != phi::DataType::UNDEFINED && dtype != x.dtype()) {
+    out_values = phi::Cast<T, Context>(dev_ctx, out_values, dtype);
+  }
+  out->SetMember(out_indices, out_values, out_dims, x.coalesced());
+}
+
+template <typename T, typename Context>
+void SumCsrKernel(const Context& dev_ctx,
+                  const SparseCsrTensor& x,
+                  const IntArray& axis,
+                  DataType dtype,
+                  bool keep_dim,
+                  SparseCsrTensor* out) {
+  size_t n_dim = axis.size();
+  const auto& x_crows = x.crows();
+  const auto& x_values = x.values();
+  const auto* x_crows_data = x_crows.data<int64_t>();
+  const auto* x_values_data = x_values.data<T>();
+
+  DenseTensor out_crows, out_cols, out_values;
+  DDim out_dims;
+  if (n_dim == 0) {
+    if (keep_dim && x.dims().size() == 3) {
+      out_dims = make_ddim({1, 1, 1});
+    } else {
+      out_dims = make_ddim({1, 1});
+    }
+    out_crows = Empty<int64_t, Context>(dev_ctx, {2});  // crows = [0, 1]
+    auto* out_crows_data = out_crows.data<int64_t>();
+    out_crows_data[0] = 0;
+    out_crows_data[1] = 1;
+
+    out_cols = Empty<int64_t, Context>(dev_ctx, {1});  // crows = [0]
+    auto* out_cols_data = out_cols.data<int64_t>();
+    out_cols_data[0] = 0;
+    out_values = phi::Sum<T>(dev_ctx, x.values(), {}, dtype, true);
+  } else {
+    PADDLE_ENFORCE_EQ(axis[0],
+                      -1,
+                      phi::errors::Unimplemented(
+                          "`axis` of SumCsrKernel only support None or -1 now."
+                          "More number will be supported in the future."));
+    out_crows = EmptyLike<int64_t, Context>(dev_ctx, x.crows());
+    auto* out_crows_data = out_crows.data<int64_t>();
+    std::vector<T> out_data;
+    if (x.dims().size() == 2) {
+      out_crows_data[0] = 0;
+      out_dims = make_ddim({x.dims()[0], 1});
+      for (int i = 0; i < x.dims()[0]; ++i) {
+        if (x_crows_data[i] != x_crows_data[i + 1]) {
+          T sum_value = 0;
+          for (auto j = x_crows_data[i]; j < x_crows_data[i + 1]; ++j) {
+            sum_value += x_values_data[j];
+          }
+          out_crows_data[i + 1] = out_crows_data[i] + 1;
+          out_data.emplace_back(sum_value);
+        } else {
+          out_crows_data[i + 1] = out_crows_data[i];
+        }
+      }
+    } else {
+      if (keep_dim) {
+        out_dims = make_ddim({x.dims()[0], x.dims()[1], 1});
+      } else {
+        out_dims = make_ddim({x.dims()[0], x.dims()[1]});
+      }
+      int j = 0;
+      for (int batch = 0; batch < x.dims()[0]; ++batch) {
+        auto* cur_x_crows_data = x_crows_data + batch * x.dims()[2];
+        auto* cur_out_crows_data = out_crows_data + batch * x.dims()[2];
+        for (int i = 0; i < x.dims()[1]; ++i) {
+          cur_out_crows_data[0] = 0;
+          if (cur_x_crows_data[i] != cur_x_crows_data[i + 1]) {
+            T sum_value = 0;
+            for (auto k = cur_x_crows_data[i]; k < cur_x_crows_data[i + 1];
+                 ++k) {
+              sum_value += x_values_data[j++];
+            }
+            out_data.emplace_back(sum_value);
+            cur_out_crows_data[i + 1] = cur_out_crows_data[i] + 1;
+          } else {
+            cur_out_crows_data[i + 1] = cur_out_crows_data[i];
+          }
+        }
+      }
+    }
+    out_cols =
+        Empty<int64_t, Context>(dev_ctx, {static_cast<int>(out_data.size())});
+    out_values =
+        Empty<T, Context>(dev_ctx, {static_cast<int>(out_data.size())});
+    auto* out_cols_data = out_cols.data<int64_t>();
+    T* out_values_data = out_values.data<T>();
+    for (size_t i = 0; i < out_data.size(); ++i) {
+      out_cols_data[i] = 0;
+      out_values_data[i] = out_data[i];
+    }
+    if (dtype != phi::DataType::UNDEFINED && dtype != x.dtype()) {
+      out_values = phi::Cast<T, Context>(dev_ctx, out_values, dtype);
+    }
+  }
+  out->SetMember(out_crows, out_cols, out_values, out_dims);
+}
+
+template <typename T, typename Context>
+void SumCooKernel(const Context& dev_ctx,
+                  const SparseCooTensor& x,
+                  const IntArray& axis,
+                  DataType dtype,
+                  bool keep_dim,
+                  SparseCooTensor* out) {
+  PD_VISIT_BASE_INTEGRAL_TYPES(x.indices().dtype(), "SumCooCPUKernel", ([&] {
+                                 SumCooCPUKernel<T, data_t, Context>(
+                                     dev_ctx, x, axis, dtype, keep_dim, out);
+                               }));
+}
+
+}  // namespace sparse
+}  // namespace phi
+
+PD_REGISTER_KERNEL(sum_coo,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::sparse::SumCooKernel,
+                   float,
+                   double,
+                   int16_t,
+                   int,
+                   int64_t,
+                   bool) {
+  kernel->OutputAt(0).SetDataType(paddle::DataType::UNDEFINED);
+}
+
+PD_REGISTER_KERNEL(sum_csr,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::sparse::SumCsrKernel,
+                   float,
+                   double,
+                   int16_t,
+                   int,
+                   int64_t,
+                   bool) {
+  kernel->OutputAt(0).SetDataType(paddle::DataType::UNDEFINED);
+}
diff --git a/paddle/phi/kernels/sparse/gpu/sum_grad_kernel.cu b/paddle/phi/kernels/sparse/gpu/sum_grad_kernel.cu
new file mode 100644
index 0000000000000..b0da1e7ab42f0
--- /dev/null
+++ b/paddle/phi/kernels/sparse/gpu/sum_grad_kernel.cu
@@ -0,0 +1,235 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/visit_type.h"
+#include "paddle/phi/kernels/cast_kernel.h"
+#include "paddle/phi/kernels/funcs/elementwise_base.h"
+#include "paddle/phi/kernels/reduce_sum_grad_kernel.h"
+#include "paddle/phi/kernels/reduce_sum_kernel.h"
+#include "paddle/phi/kernels/sparse/empty_kernel.h"
+#include "paddle/phi/kernels/sparse/unary_grad_kernel.h"
+#include "paddle/phi/kernels/sparse/unary_kernel.h"
+
+namespace phi {
+namespace sparse {
+
+template <typename T>
+__global__ void SetValueCudaKernel(const T* value,
+                                   const int64_t length,
+                                   T* data) {
+  CUDA_KERNEL_LOOP_TYPE(index, length, int64_t) { data[index] = value[0]; }
+}
+
+template <typename T>
+__global__ void SumCsr2DGradCudaKernel(const int64_t* x_crows_data,
+                                       const T* dout_values_data,
+                                       const int64_t x_dim0,
+                                       T* dx_values_data) {
+  // dout_crows_data[index] should be equal to index;
+  CUDA_KERNEL_LOOP_TYPE(index, x_dim0, int64_t) {
+    T value = dout_values_data[index];
+    for (auto i = x_crows_data[index]; i < x_crows_data[index + 1]; ++i) {
+      dx_values_data[i] = value;
+    }
+  }
+}
+
+template <typename T>
+__global__ void SumCsr3DGradCudaKernel(const int64_t* x_crows_data,
+                                       const T* dout_values_data,
+                                       const int64_t x_dim0,
+                                       const int64_t x_dim1,
+                                       T* dx_values_data) {
+  // dout_crows_data[index] should be equal to number;
+  CUDA_KERNEL_LOOP_TYPE(index, x_dim0 * (x_dim1 + 1), int64_t) {
+    int64_t batch = index / (x_dim1 + 1);
+    int64_t number = index % (x_dim1 + 1);
+
+    // compute offset of dx_values_data in every batch
+    int64_t batch_offset = 0;
+    for (int64_t b = 1; b <= batch; ++b) {
+      batch_offset += x_crows_data[b * (x_dim1 + 1) - 1];
+    }
+
+    T value = dout_values_data[index - batch];
+    for (auto i = x_crows_data[index]; i < x_crows_data[index + 1]; ++i) {
+      dx_values_data[i + batch_offset] = value;
+    }
+  }
+}
+
+template <typename T, typename IntT, typename Context>
+void SumCooGradGPUKernel(const Context& dev_ctx,
+                         const SparseCooTensor& x,
+                         const SparseCooTensor& dout,
+                         const IntArray& axis,
+                         bool keep_dim,
+                         SparseCooTensor* dx) {
+  EmptyLikeCooKernel<T, Context>(dev_ctx, x, dx);
+  unsigned int n_dim = axis.size();
+
+  const DenseTensor& x_indices = x.indices();
+  const DenseTensor& dout_indices = dout.indices();
+  const DenseTensor& dout_values = dout.values();
+  const auto* dout_indices_data = dout_indices.data<IntT>();
+  const auto* dout_values_data = dout_values.data<T>();
+
+  DenseTensor* dx_indices = dx->mutable_indices();
+  DenseTensor* dx_values = dx->mutable_values();
+  *dx_indices = x_indices;
+
+  const auto* dx_indices_data = dx_indices->data<IntT>();
+  auto* dx_values_data = dx_values->data<T>();
+
+  if (n_dim == 0) {
+    auto length = dx->nnz();
+    for (auto i = 1; i < x.values().dims().size(); ++i) {
+      length *= x.values().dims()[i];
+    }
+    auto config = phi::backends::gpu::GetGpuLaunchConfig1D(dev_ctx, length, 1);
+
+    SetValueCudaKernel<T>
+        <<<config.block_per_grid.x,
+           config.thread_per_block.x,
+           0,
+           dev_ctx.stream()>>>(dout_values_data, length, dx_values_data);
+
+    if (dx_values->dtype() != dx->dtype()) {
+      *dx_values = phi::Cast<T, Context>(dev_ctx, *dx_values, dx->dtype());
+    }
+    return;
+  }
+
+  auto dim = axis[0] < 0 ? x.dims().size() + axis[0] : axis[0];
+  auto sparse_dim = x.sparse_dim();
+  if (dim >= sparse_dim) {
+    dim = dim - sparse_dim + 1;
+    phi::ReduceSumGradKernel<T, Context>(
+        dev_ctx, x.values(), dout.values(), {dim}, keep_dim, false, dx_values);
+  } else {
+    *dx_values = dout_values;
+  }
+  if (dx_values->dtype() != dx->dtype()) {
+    *dx_values = phi::Cast<T, Context>(dev_ctx, *dx_values, dx->dtype());
+  }
+}
+
+template <typename T, typename Context>
+void SumCsrGradKernel(const Context& dev_ctx,
+                      const SparseCsrTensor& x,
+                      const SparseCsrTensor& dout,
+                      const IntArray& axis,
+                      bool keep_dim,
+                      SparseCsrTensor* dx) {
+  EmptyLikeCsrKernel<T, Context>(dev_ctx, x, dx);
+  size_t n_dim = axis.size();
+
+  const DenseTensor& x_crows = x.crows();
+  const DenseTensor& x_cols = x.cols();
+  const DenseTensor& dout_values = dout.values();
+
+  DenseTensor* dx_crows = dx->mutable_crows();
+  DenseTensor* dx_cols = dx->mutable_cols();
+  DenseTensor* dx_values = dx->mutable_values();
+
+  const auto* x_crows_data = x_crows.data<int64_t>();
+  const auto* dout_values_data = dout_values.data<T>();
+  auto* dx_values_data = dx_values->data<T>();
+
+  *dx_crows = x_crows;
+  *dx_cols = x_cols;
+
+  if (n_dim == 0) {
+    auto config =
+        phi::backends::gpu::GetGpuLaunchConfig1D(dev_ctx, dx->nnz(), 1);
+    SetValueCudaKernel<T>
+        <<<config.block_per_grid.x,
+           config.thread_per_block.x,
+           0,
+           dev_ctx.stream()>>>(dout_values_data, dx->nnz(), dx_values_data);
+
+    if (dx_values->dtype() != dx->dtype()) {
+      *dx_values = phi::Cast<T, Context>(dev_ctx, *dx_values, dx->dtype());
+    }
+    return;
+  }
+  PADDLE_ENFORCE_EQ(axis[0],
+                    -1,
+                    phi::errors::Unimplemented(
+                        "`axis` of SumCsrKernel only support None or -1 now."
+                        "More number will be supported in the future."));
+  if (x.dims().size() == 2) {
+    auto config =
+        phi::backends::gpu::GetGpuLaunchConfig1D(dev_ctx, x.dims()[0], 1);
+    SumCsr2DGradCudaKernel<T><<<config.block_per_grid.x,
+                                config.thread_per_block.x,
+                                0,
+                                dev_ctx.stream()>>>(
+        x_crows_data, dout_values_data, x.dims()[0], dx_values_data);
+  } else {
+    auto config = phi::backends::gpu::GetGpuLaunchConfig1D(
+        dev_ctx, x.dims()[0] * (x.dims()[1] + 1), 1);
+    SumCsr3DGradCudaKernel<T><<<config.block_per_grid.x,
+                                config.thread_per_block.x,
+                                0,
+                                dev_ctx.stream()>>>(x_crows_data,
+                                                    dout_values_data,
+                                                    x.dims()[0],
+                                                    x.dims()[1],
+                                                    dx_values_data);
+  }
+  if (dx_values->dtype() != dx->dtype()) {
+    *dx_values = phi::Cast<T, Context>(dev_ctx, *dx_values, dx->dtype());
+  }
+}
+
+template <typename T, typename Context>
+void SumCooGradKernel(const Context& dev_ctx,
+                      const SparseCooTensor& x,
+                      const SparseCooTensor& dout,
+                      const IntArray& axis,
+                      bool keep_dim,
+                      SparseCooTensor* dx) {
+  PD_VISIT_BASE_INTEGRAL_TYPES(
+      x.indices().dtype(), "SumCooGradGPUKernel", ([&] {
+        SumCooGradGPUKernel<T, data_t, Context>(
+            dev_ctx, x, dout, axis, keep_dim, dx);
+      }));
+}
+}  // namespace sparse
+}  // namespace phi
+
+PD_REGISTER_KERNEL(sum_coo_grad,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::sparse::SumCooGradKernel,
+                   float,
+                   double,
+                   int16_t,
+                   int,
+                   int64_t,
+                   bool) {}
+
+PD_REGISTER_KERNEL(sum_csr_grad,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::sparse::SumCsrGradKernel,
+                   float,
+                   double,
+                   int16_t,
+                   int,
+                   int64_t,
+                   bool) {}
diff --git a/paddle/phi/kernels/sparse/gpu/sum_kernel.cu b/paddle/phi/kernels/sparse/gpu/sum_kernel.cu
new file mode 100644
index 0000000000000..594e1ec48b2e1
--- /dev/null
+++ b/paddle/phi/kernels/sparse/gpu/sum_kernel.cu
@@ -0,0 +1,456 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/sparse/unary_kernel.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/backends/gpu/gpu_launch_config.h"
+#include "paddle/phi/backends/gpu/gpu_primitives.h"
+#include "paddle/phi/common/scalar.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/visit_type.h"
+#include "paddle/phi/kernels/cast_kernel.h"
+#include "paddle/phi/kernels/cum_kernel.h"
+#include "paddle/phi/kernels/empty_kernel.h"
+#include "paddle/phi/kernels/funcs/elementwise_base.h"
+#include "paddle/phi/kernels/index_select_kernel.h"
+#include "paddle/phi/kernels/reduce_sum_kernel.h"
+#include "paddle/phi/kernels/reshape_kernel.h"
+#include "paddle/phi/kernels/sparse/empty_kernel.h"
+#include "paddle/phi/kernels/sparse/sparse_utils_kernel.h"
+
+namespace phi {
+namespace sparse {
+
+template <typename T, typename IntT>
+__global__ void SumCooCudaKernel(const IntT* x_indices_data,
+                                 const T* x_values_data,
+                                 const int64_t x_nnz,
+                                 const int64_t dense_dim,
+                                 const int64_t sparse_dim,
+                                 const int64_t axis,
+                                 const bool keep_dim,
+                                 IntT* out_indices_data,
+                                 T* out_values_data) {
+  CUDA_KERNEL_LOOP_TYPE(index_i, x_nnz, int64_t) {
+    int64_t i = 0;
+    for (int j = 0; j < dense_dim; ++j) {
+      out_values_data[j + index_i * dense_dim] = 0;
+    }
+
+    int64_t _index_j_ =
+        static_cast<int64_t>(blockIdx.y) * blockDim.y + threadIdx.y;
+    for (auto index_j = _index_j_; index_j < x_nnz;
+         index_j += static_cast<int64_t>(blockDim.y) * gridDim.y) {
+      // Determine whether the index_i and index_j elements have the same
+      // indices in all dimensions except for the specified axis dimension.
+      bool same = true;
+      for (int j = 0; j < sparse_dim + !keep_dim; ++j) {
+        if (j != axis && x_indices_data[index_i + j * x_nnz] !=
+                             x_indices_data[index_j + j * x_nnz]) {
+          same = false;
+          break;
+        }
+      }
+      if (same) {
+        for (int j = 0; j < dense_dim; ++j) {
+          phi::CudaAtomicAdd(&out_values_data[j + index_i * dense_dim],
+                             x_values_data[j + index_j * dense_dim]);
+        }
+      }
+    }
+    if (_index_j_ != 0) {
+      return;
+    }
+    if (keep_dim) {
+      for (int j = 0; j < sparse_dim; ++j) {
+        if (j == axis) {
+          out_indices_data[index_i + j * x_nnz] = 0;
+        } else {
+          out_indices_data[index_i + j * x_nnz] =
+              x_indices_data[index_i + j * x_nnz];
+        }
+      }
+      return;
+    }
+    for (int j = 0; j < sparse_dim; ++j) {
+      // out_indices_data [sparse_dim, x.nnz()]
+      int64_t x_indices_data_offset;
+      if (j < axis) {
+        x_indices_data_offset = index_i + j * x_nnz;
+      } else {
+        x_indices_data_offset = index_i + (j + 1) * x_nnz;
+      }
+      out_indices_data[index_i + j * x_nnz] =
+          x_indices_data[x_indices_data_offset];
+    }
+  }
+}
+
+__global__ void SumAllCsrCudaKernel(int64_t* out_crows_data,
+                                    int64_t* out_cols_data) {
+  CUDA_KERNEL_LOOP_TYPE(index, 2, int64_t) {
+    out_crows_data[index] = index;
+    if (index == 0) {
+      out_cols_data[0] = 0;
+    }
+  }
+}
+
+template <typename T>
+__global__ void SumCsr2DCudaKernel(const int64_t* x_crows_data,
+                                   const T* x_values_data,
+                                   const int64_t x_dim0,
+                                   int64_t* out_crows_data,
+                                   int64_t* out_cols_data,
+                                   T* out_values_data) {
+  CUDA_KERNEL_LOOP_TYPE(index, x_dim0 + 1, int64_t) {
+    out_crows_data[index] = index;
+    if (index != x_dim0) {
+      out_cols_data[index] = 0;
+      T sum_value = 0;
+      for (auto j = x_crows_data[index]; j < x_crows_data[index + 1]; ++j) {
+        sum_value += x_values_data[j];
+      }
+      out_values_data[index] = sum_value;
+    }
+  }
+}
+
+template <typename T>
+__global__ void SumCsr3DCudaKernel(const int64_t* x_crows_data,
+                                   const T* x_values_data,
+                                   const int64_t x_dim0,
+                                   const int64_t x_dim1,
+                                   const int64_t* batch_nnz_data,
+                                   int64_t* out_crows_data,
+                                   int64_t* out_cols_data,
+                                   T* out_values_data) {
+  CUDA_KERNEL_LOOP_TYPE(index, x_dim0 * (x_dim1 + 1), int64_t) {
+    int64_t batch = index / (x_dim1 + 1);
+    int64_t number = index % (x_dim1 + 1);
+    out_crows_data[index] = number;
+    out_cols_data[index] = 0;
+
+    if (number != x_dim1) {
+      T sum_value = 0;
+      int64_t x_values_data_offset;
+      if (batch == 0) {
+        x_values_data_offset = 0;
+      } else {
+        x_values_data_offset = batch_nnz_data[batch - 1];
+      }
+      for (int64_t j = x_crows_data[index]; j < x_crows_data[index + 1]; ++j) {
+        sum_value += x_values_data[j + x_values_data_offset];
+      }
+      out_values_data[index - batch] = sum_value;
+    }
+  }
+}
+
+template <typename T, typename IntT, typename Context>
+void SumCooGPU0Kernel(const Context& dev_ctx,
+                      const SparseCooTensor& x,
+                      const IntArray& axis,
+                      DataType dtype,
+                      bool keep_dim,
+                      SparseCooTensor* out) {
+  auto sparse_dim = x.sparse_dim();
+  // create out sparse tensor
+  const auto& x_dims = x.dims();
+  const auto& x_indices = x.indices();
+  const auto& x_values = x.values();
+  DDim out_dims;
+  DenseTensor out_indices;
+  DenseTensor out_values;
+  if (keep_dim) {
+    out_dims = make_ddim(std::vector<int64_t>(x_dims.size(), 1));
+    out_indices = Empty<IntT, Context>(dev_ctx, {sparse_dim, 1});
+  } else {
+    out_dims = make_ddim({1});
+    out_indices = Empty<IntT, Context>(dev_ctx, {1, 1});
+  }
+  phi::funcs::SetConstant<Context, IntT> set_out_indices;
+  set_out_indices(dev_ctx, &out_indices, static_cast<IntT>(0));
+  out_values = phi::Sum<T>(dev_ctx, x.values(), {}, dtype, keep_dim);
+  out->SetMember(out_indices, out_values, out_dims, x.coalesced());
+}
+
+template <typename T, typename IntT, typename Context>
+void SumCooGPU1Kernel(const Context& dev_ctx,
+                      const SparseCooTensor& x,
+                      const IntArray& axis,
+                      DataType dtype,
+                      bool keep_dim,
+                      SparseCooTensor* out) {
+  auto sparse_dim = x.sparse_dim();
+  // create out sparse tensor
+  const auto& x_dims = x.dims();
+  const auto& x_indices = x.indices();
+  const auto& x_values = x.values();
+  DDim out_dims;
+  DenseTensor out_indices;
+  DenseTensor out_values;
+  auto n_dim = x.dims().size();
+  auto dim = axis[0] < 0 ? x_dims.size() + axis[0] : axis[0];
+
+  std::vector<int64_t> dims;
+  for (int i = 0; i < n_dim; ++i) {
+    if (i != dim) {
+      dims.emplace_back(x.dims()[i]);
+    } else if (keep_dim || (dim < sparse_dim && sparse_dim == 1)) {
+      dims.emplace_back(1);
+    }
+  }
+  out_dims = make_ddim(dims);
+
+  if (dim >= sparse_dim) {
+    out_indices = x_indices;
+    dim = dim - sparse_dim + 1;
+    out_values = phi::Sum<T>(dev_ctx, x.values(), {dim}, dtype, keep_dim);
+    out->SetMember(out_indices, out_values, out_dims, x.coalesced());
+    return;
+  }
+
+  // Ensure the sparse_dim is not less than 1.
+  if (sparse_dim == 1) {
+    keep_dim = true;
+  }
+  // if axis in sparse_dim and keep_dim, sparse_dim will be reduced.
+  if (!keep_dim) {
+    sparse_dim -= 1;
+  }
+
+  std::vector<int> out_values_dims;
+  out_values_dims.push_back(x.nnz());
+  for (auto i = 1; i < x.values().dims().size(); ++i) {
+    out_values_dims.push_back(static_cast<int>(x.values().dims()[i]));
+  }
+  int64_t dense_dim = std::accumulate(out_values_dims.begin() + 1,
+                                      out_values_dims.end(),
+                                      1,
+                                      std::multiplies<int64_t>());
+
+  out_indices = Empty<IntT, Context>(dev_ctx, {sparse_dim, x.nnz()});
+  out_values = Empty<T, Context>(dev_ctx, out_values_dims);
+
+  const auto* x_indices_data = x_indices.data<IntT>();
+  const auto* x_values_data = x_values.data<T>();
+  auto* out_indices_data = out_indices.data<IntT>();
+  auto* out_values_data = out_values.data<T>();
+
+  auto config =
+      phi::backends::gpu::GetGpuLaunchConfig2D(dev_ctx, x.nnz(), x.nnz());
+  SumCooCudaKernel<T, IntT><<<config.block_per_grid.x,
+                              config.thread_per_block.x,
+                              0,
+                              dev_ctx.stream()>>>(x_indices_data,
+                                                  x_values_data,
+                                                  x.nnz(),
+                                                  dense_dim,
+                                                  sparse_dim,
+                                                  dim,
+                                                  keep_dim,
+                                                  out_indices_data,
+                                                  out_values_data);
+  if (dtype != phi::DataType::UNDEFINED && dtype != x.dtype()) {
+    out_values = phi::Cast<T, Context>(dev_ctx, out_values, dtype);
+  }
+  out->SetMember(out_indices, out_values, out_dims, x.coalesced());
+}
+
+template <typename T, typename Context>
+void SumCooKernel(const Context& dev_ctx,
+                  const SparseCooTensor& x,
+                  const IntArray& axis,
+                  DataType dtype,
+                  bool keep_dim,
+                  SparseCooTensor* out) {
+  const size_t n_dim = axis.size();
+  if (n_dim == 0) {
+    PD_VISIT_BASE_INTEGRAL_TYPES(x.indices().dtype(), "SumCooGPUKernel", ([&] {
+                                   SumCooGPU0Kernel<T, data_t, Context>(
+                                       dev_ctx, x, axis, dtype, keep_dim, out);
+                                 }));
+  } else {
+    PD_VISIT_BASE_INTEGRAL_TYPES(x.indices().dtype(), "SumCooGPUKernel", ([&] {
+                                   SumCooGPU1Kernel<T, data_t, Context>(
+                                       dev_ctx, x, axis, dtype, keep_dim, out);
+                                 }));
+  }
+}
+
+template <typename T, typename Context>
+void SumCsr0Kernel(const Context& dev_ctx,
+                   const SparseCsrTensor& x,
+                   const IntArray& axis,
+                   DataType dtype,
+                   bool keep_dim,
+                   SparseCsrTensor* out) {
+  auto x_dim0 = x.dims()[0];
+  auto x_dim1 = x.dims()[1];
+  const auto& x_crows = x.crows();
+  const auto& x_values = x.values();
+  const auto* x_crows_data = x_crows.data<int64_t>();
+  const auto* x_values_data = x_values.data<T>();
+
+  DenseTensor out_crows, out_cols, out_values;
+  DDim out_dims;
+  if (keep_dim && x.dims().size() == 3) {
+    out_dims = make_ddim({1, 1, 1});
+  } else {
+    out_dims = make_ddim({1, 1});
+  }
+  out_crows = Empty<int64_t, Context>(dev_ctx, {2});  // crows = [0, 1]
+  out_cols = Empty<int64_t, Context>(dev_ctx, {1});   // crows = [0]
+  auto* out_crows_data = out_crows.data<int64_t>();
+  auto* out_cols_data = out_cols.data<int64_t>();
+
+  auto config = phi::backends::gpu::GetGpuLaunchConfig1D(dev_ctx, 2, 1);
+  SumAllCsrCudaKernel<<<config.block_per_grid.x,
+                        config.thread_per_block.x,
+                        0,
+                        dev_ctx.stream()>>>(out_crows_data, out_cols_data);
+
+  out_values = phi::Sum<T>(dev_ctx, x.values(), {}, dtype, true);
+  out->SetMember(out_crows, out_cols, out_values, out_dims);
+}
+
+template <typename T, typename Context>
+void SumCsr1Kernel(const Context& dev_ctx,
+                   const SparseCsrTensor& x,
+                   const IntArray& axis,
+                   DataType dtype,
+                   bool keep_dim,
+                   SparseCsrTensor* out) {
+  auto x_dim0 = x.dims()[0];
+  auto x_dim1 = x.dims()[1];
+  const auto& x_crows = x.crows();
+  const auto& x_values = x.values();
+  const auto* x_crows_data = x_crows.data<int64_t>();
+  const auto* x_values_data = x_values.data<T>();
+
+  DenseTensor out_crows, out_cols, out_values;
+  DDim out_dims;
+  out_crows = EmptyLike<int64_t, Context>(dev_ctx, x.crows());
+  auto* out_crows_data = out_crows.data<int64_t>();
+
+  if (x.dims().size() == 2) {
+    out_cols = Empty<int64_t, Context>(dev_ctx, {x_dim0});
+    out_values = Empty<T, Context>(dev_ctx, {x_dim0});
+    auto* out_cols_data = out_cols.data<int64_t>();
+    auto* out_values_data = out_values.data<T>();
+    out_dims = make_ddim({x_dim0, 1});
+    auto config =
+        phi::backends::gpu::GetGpuLaunchConfig1D(dev_ctx, x_dim0 + 1, 1);
+    SumCsr2DCudaKernel<T><<<config.block_per_grid.x,
+                            config.thread_per_block.x,
+                            0,
+                            dev_ctx.stream()>>>(x_crows_data,
+                                                x_values_data,
+                                                x_dim0,
+                                                out_crows_data,
+                                                out_cols_data,
+                                                out_values_data);
+
+  } else {
+    out_cols = Empty<int64_t, Context>(dev_ctx, {x_dim0 * x_dim1});
+    out_values = Empty<T, Context>(dev_ctx, {x_dim0 * x_dim1});
+    auto* out_cols_data = out_cols.data<int64_t>();
+    auto* out_values_data = out_values.data<T>();
+    if (keep_dim) {
+      out_dims = make_ddim({x_dim0, x_dim1, 1});
+    } else {
+      out_dims = make_ddim({x_dim0, x_dim1});
+    }
+
+    DenseTensor x_crows_reshape =
+        Reshape<int64_t, Context>(dev_ctx, x_crows, {x_dim0, x_dim1 + 1});
+    DenseTensor last_indices = Empty<int64_t, Context>(dev_ctx, {1});
+    phi::funcs::SetConstant<Context, int64_t> set_constant;
+    set_constant(dev_ctx, &last_indices, x_dim1);
+
+    DenseTensor x_crows_last = Empty<int64_t, Context>(dev_ctx, {x_dim0, 1});
+    IndexSelectKernel<int64_t, Context>(
+        dev_ctx, x_crows_reshape, last_indices, 1, &x_crows_last);
+
+    DenseTensor batch_nnz = Empty<int64_t, Context>(dev_ctx, {x_dim0, 1});
+    CumsumKernel<int64_t, Context>(
+        dev_ctx, x_crows_last, Scalar(0), false, false, false, &batch_nnz);
+    auto* batch_nnz_data = batch_nnz.data<int64_t>();
+
+    auto config = phi::backends::gpu::GetGpuLaunchConfig1D(
+        dev_ctx, x.dims()[0] * (x.dims()[1] + 1), 1);
+    SumCsr3DCudaKernel<T><<<config.block_per_grid.x,
+                            config.thread_per_block.x,
+                            0,
+                            dev_ctx.stream()>>>(x_crows_data,
+                                                x_values_data,
+                                                x_dim0,
+                                                x_dim1,
+                                                batch_nnz_data,
+                                                out_crows_data,
+                                                out_cols_data,
+                                                out_values_data);
+  }
+  if (dtype != phi::DataType::UNDEFINED && dtype != x.dtype()) {
+    out_values = phi::Cast<T, Context>(dev_ctx, out_values, dtype);
+  }
+  out->SetMember(out_crows, out_cols, out_values, out_dims);
+}
+
+template <typename T, typename Context>
+void SumCsrKernel(const Context& dev_ctx,
+                  const SparseCsrTensor& x,
+                  const IntArray& axis,
+                  DataType dtype,
+                  bool keep_dim,
+                  SparseCsrTensor* out) {
+  size_t n_dim = axis.size();
+  if (n_dim == 0) {
+    SumCsr0Kernel<T, Context>(dev_ctx, x, axis, dtype, keep_dim, out);
+  } else {
+    PADDLE_ENFORCE_EQ(axis[0],
+                      -1,
+                      phi::errors::Unimplemented(
+                          "`axis` of SumCsrKernel only support None or -1 now."
+                          "More number will be supported in the future."));
+    SumCsr1Kernel<T, Context>(dev_ctx, x, axis, dtype, keep_dim, out);
+  }
+}
+
+}  // namespace sparse
+}  // namespace phi
+
+PD_REGISTER_KERNEL(sum_coo,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::sparse::SumCooKernel,
+                   float,
+                   double,
+                   int,
+                   int64_t) {
+  kernel->OutputAt(0).SetDataType(paddle::DataType::UNDEFINED);
+}
+
+PD_REGISTER_KERNEL(sum_csr,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::sparse::SumCsrKernel,
+                   float,
+                   double,
+                   int,
+                   int64_t) {
+  kernel->OutputAt(0).SetDataType(paddle::DataType::UNDEFINED);
+}
diff --git a/paddle/phi/kernels/sparse/unary_grad_kernel.h b/paddle/phi/kernels/sparse/unary_grad_kernel.h
index 7440533057022..5893b16f6ba3d 100644
--- a/paddle/phi/kernels/sparse/unary_grad_kernel.h
+++ b/paddle/phi/kernels/sparse/unary_grad_kernel.h
@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include "paddle/phi/common/int_array.h"
 #include "paddle/phi/core/sparse_coo_tensor.h"
 #include "paddle/phi/core/sparse_csr_tensor.h"
 
@@ -92,6 +93,22 @@ void TransposeCsrGradKernel(const Context& dev_ctx,
                             const std::vector<int>& perm,
                             SparseCsrTensor* dx);
 
+template <typename T, typename Context>
+void SumCooGradKernel(const Context& dev_ctx,
+                      const SparseCooTensor& x,
+                      const SparseCooTensor& dout,
+                      const IntArray& axis,
+                      bool keep_dim,
+                      SparseCooTensor* dx);
+
+template <typename T, typename Context>
+void SumCsrGradKernel(const Context& dev_ctx,
+                      const SparseCsrTensor& x,
+                      const SparseCsrTensor& dout,
+                      const IntArray& axis,
+                      bool keep_dim,
+                      SparseCsrTensor* dx);
+
 template <typename T, typename Context>
 void ReshapeCooGradKernel(const Context& dev_ctx,
                           const SparseCooTensor& x,
diff --git a/paddle/phi/kernels/sparse/unary_kernel.h b/paddle/phi/kernels/sparse/unary_kernel.h
index b219ec07236df..d692f75b59408 100644
--- a/paddle/phi/kernels/sparse/unary_kernel.h
+++ b/paddle/phi/kernels/sparse/unary_kernel.h
@@ -157,6 +157,22 @@ SparseCsrTensor TransposeCsr(const Context& dev_ctx,
   return csr;
 }
 
+template <typename T, typename Context>
+void SumCooKernel(const Context& dev_ctx,
+                  const SparseCooTensor& x,
+                  const IntArray& axis,
+                  DataType dtype,
+                  bool keep_dim,
+                  SparseCooTensor* out);
+
+template <typename T, typename Context>
+void SumCsrKernel(const Context& dev_ctx,
+                  const SparseCsrTensor& x,
+                  const IntArray& axis,
+                  DataType dtype,
+                  bool keep_dim,
+                  SparseCsrTensor* out);
+
 template <typename T, typename Context>
 SparseCooTensor ReluCoo(const Context& dev_ctx, const SparseCooTensor& x) {
   SparseCooTensor coo;
diff --git a/python/paddle/fluid/tests/unittests/test_sparse_sum_op.py b/python/paddle/fluid/tests/unittests/test_sparse_sum_op.py
new file mode 100644
index 0000000000000..3690341c51dc0
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_sparse_sum_op.py
@@ -0,0 +1,204 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+import paddle
+
+devices = ['cpu']
+if paddle.device.get_device() != "cpu":
+    devices.append(paddle.device.get_device())
+
+
+class TestSparseSum(unittest.TestCase):
+    """
+    Test the API paddle.sparse.sum on some sparse tensors.
+    x: sparse tensor, out: sparse tensor
+    """
+
+    def to_sparse(self, x, format, sparse_dim=None):
+        if format == 'coo':
+            if sparse_dim:
+                return x.detach().to_sparse_coo(sparse_dim=sparse_dim)
+            else:
+                return x.detach().to_sparse_coo(sparse_dim=x.ndim)
+        elif format == 'csr':
+            return x.detach().to_sparse_csr()
+
+    def check_result(
+        self, x_shape, dims, keepdim, format, sparse_dim=None, dtype=None
+    ):
+        for device in devices:
+            paddle.device.set_device(device)
+            if sparse_dim:
+                mask_shape = [*x_shape[:sparse_dim]] + [1] * (
+                    len(x_shape) - sparse_dim
+                )
+                mask = paddle.randint(0, 2, mask_shape)
+            else:
+                mask = paddle.randint(0, 2, x_shape)
+
+            while paddle.sum(mask) == 0:
+                if sparse_dim:
+                    mask_shape = [*x_shape[:sparse_dim]] + [1] * (
+                        len(x_shape) - sparse_dim
+                    )
+                    mask = paddle.randint(0, 2, mask_shape)
+                else:
+                    mask = paddle.randint(0, 2, x_shape)
+            # "+ 1" to make sure that all zero elements in "origin_x" is caused by multiplying by "mask",
+            # or the backward checks may fail.
+            origin_x = (paddle.rand(x_shape, dtype='float64') + 1) * mask
+            dense_x = origin_x.detach()
+            dense_x.stop_gradient = False
+            dense_out = paddle.sum(dense_x, dims, keepdim=keepdim, dtype=dtype)
+            sp_x = self.to_sparse(origin_x, format, sparse_dim)
+            sp_x.stop_gradient = False
+            sp_out = paddle.sparse.sum(sp_x, dims, keepdim=keepdim, dtype=dtype)
+            np.testing.assert_allclose(
+                sp_out.to_dense().numpy(), dense_out.numpy(), rtol=1e-05
+            )
+            dense_out.backward()
+            sp_out.backward()
+            np.testing.assert_allclose(
+                sp_x.grad.to_dense().numpy(),
+                (dense_x.grad * mask).numpy(),
+                rtol=1e-05,
+            )
+
+    def test_sum_1d(self):
+        self.check_result([5], None, False, 'coo')
+        self.check_result([5], None, True, 'coo')
+        self.check_result([5], 0, False, 'coo')
+        self.check_result([5], 0, True, 'coo')
+
+    def test_sum_2d(self):
+        self.check_result([2, 5], None, False, 'coo', dtype="float32")
+        self.check_result([2, 5], None, True, 'coo')
+        self.check_result([2, 5], 0, True, 'coo', dtype="float32")
+        self.check_result([2, 5], 0, False, 'coo')
+        self.check_result([2, 5], 1, False, 'coo')
+        self.check_result([2, 5], None, True, 'csr', dtype="float32")
+        self.check_result([2, 5], -1, True, 'csr', dtype="float32")
+        self.check_result([2, 5], 0, False, 'coo')
+        self.check_result([2, 5], -1, True, 'csr')
+
+    def test_sum_3d(self):
+        self.check_result([6, 2, 3], -1, True, 'csr')
+        for i in [0, 1, -2, None]:
+            self.check_result([6, 2, 3], i, False, 'coo')
+            self.check_result([6, 2, 3], i, True, 'coo')
+
+    def test_sum_nd(self):
+        for i in range(6):
+            self.check_result([8, 3, 4, 4, 5, 3], i, False, 'coo')
+            self.check_result([8, 3, 4, 4, 5, 3], i, True, 'coo')
+            # Randint now only supports access to dimension 0 to 9.
+            self.check_result([2, 3, 4, 2, 3, 4, 2, 3, 4], i, False, 'coo')
+
+    def test_sum_sparse_dim(self):
+        for i in range(6):
+            self.check_result([8, 3, 4, 4, 5, 3], i, False, 'coo', sparse_dim=3)
+            self.check_result([8, 3, 4, 4, 5, 3], i, True, 'coo', sparse_dim=3)
+
+
+class TestSparseSumStatic(unittest.TestCase):
+    def check_result_coo(self, x_shape, dims, keepdim, dtype=None):
+        for device in devices:
+            paddle.device.set_device(device)
+            mask = paddle.randint(0, 2, x_shape)
+            while paddle.sum(mask) == 0:
+                mask = paddle.randint(0, 2, x_shape)
+            origin_data = (paddle.rand(x_shape, dtype='float32') + 1) * mask
+            sparse_data = origin_data.detach().to_sparse_coo(
+                sparse_dim=len(x_shape)
+            )
+            indices_data = sparse_data.indices()
+            values_data = sparse_data.values()
+
+            dense_x = origin_data
+            dense_out = paddle.sum(dense_x, dims, keepdim=keepdim, dtype=dtype)
+
+            paddle.enable_static()
+            with paddle.static.program_guard(
+                paddle.static.Program(), paddle.static.Program()
+            ):
+                indices = paddle.static.data(
+                    name='indices',
+                    shape=indices_data.shape,
+                    dtype=indices_data.dtype,
+                )
+                values = paddle.static.data(
+                    name='values',
+                    shape=values_data.shape,
+                    dtype=values_data.dtype,
+                )
+                sp_x = paddle.sparse.sparse_coo_tensor(
+                    indices,
+                    values,
+                    shape=origin_data.shape,
+                    dtype=origin_data.dtype,
+                )
+                sp_out = paddle.sparse.sum(
+                    sp_x, dims, keepdim=keepdim, dtype=dtype
+                )
+                sp_dense_out = sp_out.to_dense()
+
+                sparse_exe = paddle.static.Executor()
+                sparse_fetch = sparse_exe.run(
+                    feed={
+                        'indices': indices_data.numpy(),
+                        "values": values_data.numpy(),
+                    },
+                    fetch_list=[sp_dense_out],
+                    return_numpy=True,
+                )
+
+                np.testing.assert_allclose(
+                    dense_out.numpy(), sparse_fetch[0], rtol=1e-5
+                )
+            paddle.disable_static()
+
+    def test_sum(self):
+        # 1d
+        self.check_result_coo([5], None, False)
+        self.check_result_coo([5], None, True)
+        self.check_result_coo([5], 0, True)
+        self.check_result_coo([5], 0, False)
+
+        # 2d
+        self.check_result_coo([2, 5], None, False, dtype="float32")
+        self.check_result_coo([2, 5], None, True)
+        self.check_result_coo([2, 5], 0, True, dtype="float32")
+        self.check_result_coo([2, 5], 0, False)
+        self.check_result_coo([2, 5], 1, False)
+        self.check_result_coo([2, 5], 0, False)
+
+        # 3d
+        for i in [0, 1, -2, None]:
+            self.check_result_coo([6, 2, 3], i, False)
+            self.check_result_coo([6, 2, 3], i, True)
+
+        # nd
+        for i in range(6):
+            self.check_result_coo([8, 3, 4, 4, 5, 3], i, False)
+            self.check_result_coo([8, 3, 4, 4, 5, 3], i, True)
+            # Randint now only supports access to dimension 0 to 9.
+            self.check_result_coo([2, 3, 4, 2, 3, 4, 2, 3, 4], i, False)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/python/paddle/sparse/__init__.py b/python/paddle/sparse/__init__.py
index e92a5936c4cfc..99051f7cc6702 100644
--- a/python/paddle/sparse/__init__.py
+++ b/python/paddle/sparse/__init__.py
@@ -35,6 +35,7 @@
 from .unary import rad2deg
 from .unary import expm1
 from .unary import transpose
+from .unary import sum
 from .unary import reshape
 from .unary import isnan
 
@@ -79,6 +80,7 @@
     'add',
     'subtract',
     'transpose',
+    'sum',
     'multiply',
     'divide',
     'coalesce',
diff --git a/python/paddle/sparse/unary.py b/python/paddle/sparse/unary.py
index da1d0b549aa41..453980225891a 100644
--- a/python/paddle/sparse/unary.py
+++ b/python/paddle/sparse/unary.py
@@ -15,12 +15,14 @@
 import numpy as np
 
 from paddle import _C_ops, in_dynamic_mode
+from paddle.common_ops_import import Variable
+from paddle.fluid.data_feeder import check_type, check_variable_and_dtype
 from paddle.fluid.framework import (
     convert_np_dtype_to_dtype_,
     core,
     dygraph_only,
 )
-from paddle.fluid.layer_helper import LayerHelper
+from paddle.framework import LayerHelper, in_dygraph_mode
 
 __all__ = []
 
@@ -155,6 +157,91 @@ def transpose(x, perm, name=None):
     return _C_ops.sparse_transpose(x, perm)
 
 
+def sum(x, axis=None, dtype=None, keepdim=False, name=None):
+    """
+    Computes the sum of sparse tensor elements over the given dimension, requiring x to be a SparseCooTensor or SparseCsrTensor.
+
+    Args:
+        x (Tensor): An N-D Tensor, the data type is bool, float16, float32, float64, int32 or int64.
+        axis (int|list|tuple, optional): The dimensions along which the sum is performed. If
+            :attr:`None`, sum all elements of :attr:`x` and return a
+            Tensor with a single element, otherwise must be in the
+            range :math:`[-rank(x), rank(x))`. If :math:`axis[i] < 0`,
+            the dimension to reduce is :math:`rank + axis[i]`.
+        dtype (str, optional): The dtype of output Tensor. The default value is None, the dtype
+            of output is the same as input Tensor `x`.
+        keepdim (bool, optional): Whether to reserve the reduced dimension in the
+            output Tensor. The result Tensor will have one fewer dimension
+            than the :attr:`x` unless :attr:`keepdim` is true, default
+            value is False.
+        name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
+
+    Returns:
+        Tensor: Results of summation operation on the specified axis of input Tensor `x`.
+        if `x.dtype='bool'` or `x.dtype='int32'`, it's data type is `'int64'`,
+        otherwise it's data type is the same as `x`.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+
+            dense_x = paddle.to_tensor([[-2., 0.], [1., 2.]])
+            sparse_x = dense_x.to_sparse_coo(1)
+            out1 = paddle.sparse.sum(sparse_x)  # [1.]
+            out2 = paddle.sparse.sum(sparse_x, axis=0)  # [-1., 2.]
+            out3 = paddle.sparse.sum(sparse_x, axis=-1)  # [-2., 3.]
+            out4 = paddle.sparse.sum(sparse_x, axis=1, keepdim=True)  # [[-2.], [3.]]
+    """
+    dtype_flag = False
+    if dtype is not None:
+        dtype_flag = True
+        dtype = convert_np_dtype_to_dtype_(dtype)
+
+    if in_dygraph_mode():
+        return _C_ops.sparse_sum(x, axis, dtype, keepdim)
+    else:
+        if axis is None:
+            axis = []
+        else:
+            axis = [axis]
+        attrs = {'axis': axis, 'dtype': dtype, 'keepdim': keepdim}
+
+        if dtype_flag:
+            attrs.update({'in_dtype': x.dtype, 'out_dtype': dtype})
+
+        check_variable_and_dtype(
+            x,
+            'x',
+            [
+                'bool',
+                'float32',
+                'float64',
+                'int16',
+                'int32',
+                'int64',
+            ],
+            'sparse_sum',
+        )
+
+        check_type(
+            axis, 'axis', (int, list, tuple, type(None), Variable), 'sparse_sum'
+        )
+
+        op_type = 'sparse_sum'
+        helper = LayerHelper(op_type)
+        if dtype_flag:
+            out = helper.create_sparse_variable_for_type_inference(dtype=dtype)
+        else:
+            out = helper.create_sparse_variable_for_type_inference(
+                dtype=x.dtype
+            )
+        helper.append_op(
+            type=op_type, inputs={'x': x}, outputs={'out': out}, attrs=attrs
+        )
+        return out
+
+
 @dygraph_only
 def atan(x, name=None):
     """

From 7e9c87c59e91f84f71027b77ff9fa07db70700df Mon Sep 17 00:00:00 2001
From: RuohengMa <120699764+RuohengMa@users.noreply.github.com>
Date: Tue, 9 May 2023 17:21:09 +0800
Subject: [PATCH 237/405] [PHI kernels] Bind XPU kernels (#53336)

* bind sparse_coo_tensor, reduce_max/max_int32, range/arange_int32, equal_bool, scatter_grad_float32, nearest_interp_int64 kernels

* add more unit tests; modify compilation logic of xpu sparse kernels
---
 paddle/phi/backends/xpu/xpu2_op_list.cc       | 25 ++++-
 paddle/phi/kernels/CMakeLists.txt             | 10 +-
 paddle/phi/kernels/reduce_max_kernel.cc       |  4 +-
 .../kernels/sparse/cpu/sparse_utils_kernel.cc |  2 +-
 .../kernels/sparse/xpu/sparse_utils_kernel.cc | 29 ++++++
 paddle/phi/kernels/xpu/compare_kernel.cc      |  8 +-
 paddle/phi/kernels/xpu/interpolate_kernel.cc  |  3 +-
 paddle/phi/kernels/xpu/reduce_max_kernel.cc   |  4 +-
 paddle/phi/kernels/xpu/scatter_grad_kernel.cc | 98 +++++++++++++++++++
 test/xpu/test_compare_op_xpu.py               | 29 +++++-
 test/xpu/test_reduce_max_op_xpu.py            | 26 ++++-
 test/xpu/test_scatter_op_xpu.py               |  7 +-
 test/xpu/test_sparse_utils_op_xpu.py          | 74 ++++++++++++++
 13 files changed, 299 insertions(+), 20 deletions(-)
 create mode 100644 paddle/phi/kernels/sparse/xpu/sparse_utils_kernel.cc
 create mode 100644 paddle/phi/kernels/xpu/scatter_grad_kernel.cc
 create mode 100644 test/xpu/test_sparse_utils_op_xpu.py

diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc
index c05b51ac7bdd5..9bde77ec8bf0b 100644
--- a/paddle/phi/backends/xpu/xpu2_op_list.cc
+++ b/paddle/phi/backends/xpu/xpu2_op_list.cc
@@ -261,7 +261,8 @@ XPUOpMap& get_kl2_ops() {
        XPUKernelSet({phi::DataType::INT64,
                      phi::DataType::INT32,
                      phi::DataType::FLOAT16,
-                     phi::DataType::FLOAT32})},
+                     phi::DataType::FLOAT32,
+                     phi::DataType::BOOL})},
       {"exp_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"exp", XPUKernelSet({phi::DataType::FLOAT32})},
       {"expand_as_v2",
@@ -513,7 +514,9 @@ XPUOpMap& get_kl2_ops() {
       {"multi_encoder_xpu",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"nearest_interp_v2",
-       XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
+       XPUKernelSet({phi::DataType::FLOAT32,
+                     phi::DataType::FLOAT16,
+                     phi::DataType::INT64})},
       {"nearest_interp_v2_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"not_equal",
        XPUKernelSet({phi::DataType::INT64,
@@ -545,7 +548,10 @@ XPUOpMap& get_kl2_ops() {
       {"prelu_grad",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"prod_raw", XPUKernelSet({phi::DataType::FLOAT32})},
-      {"range", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::INT64})},
+      {"range",
+       XPUKernelSet({phi::DataType::FLOAT32,
+                     phi::DataType::INT64,
+                     phi::DataType::INT32})},
       {"randperm",
        XPUKernelSet({phi::DataType::INT32,
                      phi::DataType::INT64,
@@ -555,7 +561,8 @@ XPUOpMap& get_kl2_ops() {
       {"reciprocal_grad",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"reduce_max_grad", XPUKernelSet({phi::DataType::FLOAT32})},
-      {"reduce_max", XPUKernelSet({phi::DataType::FLOAT32})},
+      {"reduce_max",
+       XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::INT32})},
       {"reduce_mean_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"reduce_mean",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
@@ -615,6 +622,8 @@ XPUOpMap& get_kl2_ops() {
        XPUKernelSet({phi::DataType::INT64,
                      phi::DataType::INT32,
                      phi::DataType::FLOAT32})},
+      {"scatter_grad",
+       XPUKernelSet({phi::DataType::FLOAT16, phi::DataType::FLOAT32})},
       {"scatter_nd_add", XPUKernelSet({phi::DataType::FLOAT32})},
       {"sampling_id",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT64})},
@@ -673,6 +682,14 @@ XPUOpMap& get_kl2_ops() {
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"softplus", XPUKernelSet({phi::DataType::FLOAT32})},
       {"softplus_grad", XPUKernelSet({phi::DataType::FLOAT32})},
+      {"sparse_coo_tensor",
+       XPUKernelSet({phi::DataType::FLOAT64,
+                     phi::DataType::INT64,
+                     phi::DataType::INT32,
+                     phi::DataType::UINT8,
+                     phi::DataType::INT16,
+                     phi::DataType::FLOAT32,
+                     phi::DataType::FLOAT16})},
       {"split",
        XPUKernelSet({phi::DataType::FLOAT32,
                      phi::DataType::FLOAT16,
diff --git a/paddle/phi/kernels/CMakeLists.txt b/paddle/phi/kernels/CMakeLists.txt
index 9b78cbf424618..e617dac99e99c 100644
--- a/paddle/phi/kernels/CMakeLists.txt
+++ b/paddle/phi/kernels/CMakeLists.txt
@@ -184,8 +184,14 @@ else()
     "fusion/cpu/*.cc")
 endif()
 
-file(GLOB kernel_xpu "xpu/*.cc" "legacy/xpu/*.cc" "selected_rows/xpu/*.cc"
-     "fusion/xpu/*.cc")
+file(
+  GLOB
+  kernel_xpu
+  "xpu/*.cc"
+  "legacy/xpu/*.cc"
+  "selected_rows/xpu/*.cc"
+  "fusion/xpu/*.cc"
+  "sparse/xpu/*.cc")
 
 if(WITH_MKLDNN)
   set(COMMON_KERNEL_DEPS ${COMMON_KERNEL_DEPS} get_kerneltype_forvar_utils)
diff --git a/paddle/phi/kernels/reduce_max_kernel.cc b/paddle/phi/kernels/reduce_max_kernel.cc
index 7892fc879c713..8f61ec6623b1c 100644
--- a/paddle/phi/kernels/reduce_max_kernel.cc
+++ b/paddle/phi/kernels/reduce_max_kernel.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -62,5 +62,5 @@ PD_REGISTER_KERNEL(
 #endif
 
 #if defined(PADDLE_WITH_XPU)
-PD_REGISTER_KERNEL(max, XPU, ALL_LAYOUT, phi::MaxKernel, float) {}
+PD_REGISTER_KERNEL(max, XPU, ALL_LAYOUT, phi::MaxKernel, float, int) {}
 #endif
diff --git a/paddle/phi/kernels/sparse/cpu/sparse_utils_kernel.cc b/paddle/phi/kernels/sparse/cpu/sparse_utils_kernel.cc
index 41c39cf387b06..fab853b3c5615 100644
--- a/paddle/phi/kernels/sparse/cpu/sparse_utils_kernel.cc
+++ b/paddle/phi/kernels/sparse/cpu/sparse_utils_kernel.cc
@@ -1,4 +1,4 @@
-/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
diff --git a/paddle/phi/kernels/sparse/xpu/sparse_utils_kernel.cc b/paddle/phi/kernels/sparse/xpu/sparse_utils_kernel.cc
new file mode 100644
index 0000000000000..a254da8b0f770
--- /dev/null
+++ b/paddle/phi/kernels/sparse/xpu/sparse_utils_kernel.cc
@@ -0,0 +1,29 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/sparse/sparse_utils_kernel.h"
+
+#include "paddle/phi/core/kernel_registry.h"
+
+PD_REGISTER_KERNEL(sparse_coo_tensor,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::sparse::SparseCooTensorKernel,
+                   float,
+                   double,
+                   phi::dtype::float16,
+                   uint8_t,
+                   int16_t,
+                   int,
+                   int64_t) {}
diff --git a/paddle/phi/kernels/xpu/compare_kernel.cc b/paddle/phi/kernels/xpu/compare_kernel.cc
index 4c9900cffa5a2..645f881bf381c 100644
--- a/paddle/phi/kernels/xpu/compare_kernel.cc
+++ b/paddle/phi/kernels/xpu/compare_kernel.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -119,7 +119,8 @@ PD_REGISTER_KERNEL(less_than_raw,
                      int,                                 \
                      int64_t,                             \
                      float,                               \
-                     phi::dtype::float16) {               \
+                     phi::dtype::float16,                 \
+                     bool) {                              \
     kernel->OutputAt(0).SetDataType(phi::DataType::BOOL); \
   }                                                       \
   PD_REGISTER_KERNEL(name##_raw,                          \
@@ -129,7 +130,8 @@ PD_REGISTER_KERNEL(less_than_raw,
                      int,                                 \
                      int64_t,                             \
                      float,                               \
-                     phi::dtype::float16) {               \
+                     phi::dtype::float16,                 \
+                     bool) {                              \
     kernel->OutputAt(0).SetDataType(phi::DataType::BOOL); \
   }
 
diff --git a/paddle/phi/kernels/xpu/interpolate_kernel.cc b/paddle/phi/kernels/xpu/interpolate_kernel.cc
index 43d7329301526..0bf7f6e1113fb 100644
--- a/paddle/phi/kernels/xpu/interpolate_kernel.cc
+++ b/paddle/phi/kernels/xpu/interpolate_kernel.cc
@@ -238,7 +238,8 @@ PD_REGISTER_KERNEL(nearest_interp,
                    ALL_LAYOUT,
                    phi::NearestInterpKernel,
                    phi::dtype::float16,
-                   float) {
+                   float,
+                   int64_t) {
   kernel->InputAt(1).SetBackend(phi::Backend::ALL_BACKEND);
   kernel->InputAt(2).SetBackend(phi::Backend::ALL_BACKEND);
   kernel->InputAt(3).SetBackend(phi::Backend::ALL_BACKEND);
diff --git a/paddle/phi/kernels/xpu/reduce_max_kernel.cc b/paddle/phi/kernels/xpu/reduce_max_kernel.cc
index c8d486d363688..cb9ff8f6bdb80 100644
--- a/paddle/phi/kernels/xpu/reduce_max_kernel.cc
+++ b/paddle/phi/kernels/xpu/reduce_max_kernel.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -49,4 +49,4 @@ void MaxRawKernel(const Context& dev_ctx,
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(max_raw, XPU, ALL_LAYOUT, phi::MaxRawKernel, float) {}
+PD_REGISTER_KERNEL(max_raw, XPU, ALL_LAYOUT, phi::MaxRawKernel, float, int) {}
diff --git a/paddle/phi/kernels/xpu/scatter_grad_kernel.cc b/paddle/phi/kernels/xpu/scatter_grad_kernel.cc
new file mode 100644
index 0000000000000..6e8bbd372ba4c
--- /dev/null
+++ b/paddle/phi/kernels/xpu/scatter_grad_kernel.cc
@@ -0,0 +1,98 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/scatter_grad_kernel.h"
+
+#include "paddle/phi/backends/xpu/enforce_xpu.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+#include "glog/logging.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void ScatterGradKernel(const Context &ctx,
+                       const DenseTensor &index,
+                       const DenseTensor &updates,
+                       const DenseTensor &out_grad,
+                       bool overwrite,
+                       DenseTensor *x_grad,
+                       DenseTensor *updates_grad) {
+  using XPUType = typename XPUTypeTrait<T>::Type;
+
+  const auto &index_type = index.dtype();
+  bool index_type_match =
+      index_type == phi::DataType::INT32 || index_type == phi::DataType::INT64;
+  PADDLE_ENFORCE_EQ(index_type_match,
+                    true,
+                    phi::errors::InvalidArgument(
+                        "scatter_op index holds the wrong type, it holds [%s],"
+                        "but desires to be [%s] or [%s]",
+                        index_type,
+                        phi::DataType::INT32,
+                        phi::DataType::INT64));
+
+  T *x_grad_data = nullptr;
+  T *updates_grad_data = nullptr;
+  if (x_grad != nullptr) {
+    ctx.template Alloc<T>(x_grad);
+    x_grad_data = x_grad->data<T>();
+  }
+  if (updates_grad != nullptr) {
+    ctx.template Alloc<T>(updates_grad);
+    updates_grad_data = updates_grad->data<T>();
+  }
+
+  std::vector<int64_t> x_grad_shape;
+  DDim out_dims = out_grad.dims();
+  for (int i = 0; i < out_dims.size(); i++) {
+    x_grad_shape.push_back(out_dims[i]);
+  }
+
+  int index_size = index.numel();
+
+  int r;
+  if (index_type == phi::DataType::INT32) {
+    auto index_data = const_cast<int *>(index.data<int>());
+    xpu::VectorParam<int> indices{nullptr, index_size, index_data};
+    r = xpu::scatter_grad<XPUType, int>(
+        ctx.x_context(),
+        reinterpret_cast<const XPUType *>(out_grad.data<T>()),
+        indices,
+        reinterpret_cast<XPUType *>(x_grad_data),
+        reinterpret_cast<XPUType *>(updates_grad_data),
+        x_grad_shape,
+        overwrite);
+  } else if (index_type == phi::DataType::INT64) {
+    auto index_data = const_cast<int64_t *>(index.data<int64_t>());
+    xpu::VectorParam<int64_t> indices{nullptr, index_size, index_data};
+    r = xpu::scatter_grad<XPUType, int64_t>(
+        ctx.x_context(),
+        reinterpret_cast<const XPUType *>(out_grad.data<T>()),
+        indices,
+        reinterpret_cast<XPUType *>(x_grad_data),
+        reinterpret_cast<XPUType *>(updates_grad_data),
+        x_grad_shape,
+        overwrite);
+  }
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "scatter grad");
+}
+}  // namespace phi
+
+PD_REGISTER_KERNEL(scatter_grad,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::ScatterGradKernel,
+                   float,
+                   phi::dtype::float16) {}
diff --git a/test/xpu/test_compare_op_xpu.py b/test/xpu/test_compare_op_xpu.py
index 4793122a81753..e798f174a9df6 100644
--- a/test/xpu/test_compare_op_xpu.py
+++ b/test/xpu/test_compare_op_xpu.py
@@ -1,4 +1,4 @@
-#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -415,6 +415,33 @@ def set_data(self):
             self.x_shape = [11, 17]
             self.y_shape = []
 
+    class EqualOpTestCaseBoolBase(TestCompareOpBase):
+        def config(self):
+            self.dtype = np.bool_
+            self.op_type = 'equal'
+            self.compute = np.equal
+            self.choices = [True, False]
+            self.set_shape()
+
+        def set_shape(self):
+            self.x_shape = [11, 17]
+            self.y_shape = [11, 17]
+
+        def set_case(self):
+            self.x = np.random.choice(self.choices, self.x_shape)
+            self.y = np.random.choice(self.choices, self.y_shape)
+            self.result = self.compute(self.x, self.y)
+
+    class EqualOpTestCaseBool1(EqualOpTestCaseBoolBase):
+        def set_shape(self):
+            self.x_shape = [11, 17]
+            self.y_shape = [1]
+
+    class EqualOpTestCaseBool2(EqualOpTestCaseBoolBase):
+        def set_shape(self):
+            self.x_shape = [1]
+            self.y_shape = [11, 17]
+
 
 support_types = get_xpu_op_support_types('equal')
 for stype in support_types:
diff --git a/test/xpu/test_reduce_max_op_xpu.py b/test/xpu/test_reduce_max_op_xpu.py
index 1b76f78d09ac7..8f27036a5eff2 100644
--- a/test/xpu/test_reduce_max_op_xpu.py
+++ b/test/xpu/test_reduce_max_op_xpu.py
@@ -1,4 +1,4 @@
-#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -45,7 +45,8 @@ def set_case(self):
                 'keep_dim': self.keep_dim,
                 'dim': self.axis,
             }
-            self.inputs = {'X': np.random.random(self.shape).astype("float32")}
+            self.dtype = self.in_type
+            self.inputs = {'X': np.random.random(self.shape).astype(self.dtype)}
             if self.attrs['reduce_all']:
                 self.outputs = {'Out': self.inputs['X'].max()}
             else:
@@ -74,6 +75,27 @@ def init_case(self):
             self.reduce_all = False
             self.keep_dim = True
 
+    class XPUTestReduceMaxCase2(XPUTestReduceMaxBase):
+        def init_case(self):
+            self.shape = (5, 6, 10)
+            self.axis = (1,)
+            self.reduce_all = False
+            self.keep_dim = True
+
+    class XPUTestReduceMaxCase3(XPUTestReduceMaxBase):
+        def init_case(self):
+            self.shape = (5, 6, 10)
+            self.axis = (1,)
+            self.reduce_all = False
+            self.keep_dim = False
+
+    class XPUTestReduceMaxCase4(XPUTestReduceMaxBase):
+        def init_case(self):
+            self.shape = (5, 6, 10)
+            self.axis = (1,)
+            self.reduce_all = True
+            self.keep_dim = False
+
 
 support_types = get_xpu_op_support_types('reduce_max')
 for stype in support_types:
diff --git a/test/xpu/test_scatter_op_xpu.py b/test/xpu/test_scatter_op_xpu.py
index 50c860bdd8673..c8b627fce82e5 100644
--- a/test/xpu/test_scatter_op_xpu.py
+++ b/test/xpu/test_scatter_op_xpu.py
@@ -1,4 +1,4 @@
-#  Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#  Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -139,11 +139,14 @@ def init_config(self):
             self.op_type = "scatter"
             self.place = paddle.XPUPlace(0)
             self.dtype = self.in_type
-            self.__class__.no_need_check_grad = True
+            self.__class__.no_need_check_grad = False
 
         def test_check_output(self):
             self.check_output_with_place(self.place)
 
+        def test_check_grad(self):
+            self.check_grad_with_place(self.place, ['X'], 'Out')
+
 
 support_types = get_xpu_op_support_types('scatter')
 for stype in support_types:
diff --git a/test/xpu/test_sparse_utils_op_xpu.py b/test/xpu/test_sparse_utils_op_xpu.py
new file mode 100644
index 0000000000000..37e0d39130ffa
--- /dev/null
+++ b/test/xpu/test_sparse_utils_op_xpu.py
@@ -0,0 +1,74 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+import paddle
+
+
+class TestSparseCreate(unittest.TestCase):
+    def test_create_coo_by_tensor(self):
+        indices = [[0, 0, 1, 2, 2], [1, 3, 2, 0, 1]]
+        values = [1, 2, 3, 4, 5]
+        dense_shape = [3, 4]
+        dense_indices = paddle.to_tensor(indices)
+        dense_elements = paddle.to_tensor(values, dtype='float32')
+        coo = paddle.sparse.sparse_coo_tensor(
+            dense_indices, dense_elements, dense_shape, stop_gradient=False
+        )
+        assert np.array_equal(indices, coo.indices().numpy())
+        assert np.array_equal(values, coo.values().numpy())
+
+    def test_create_coo_by_np(self):
+        indices = [[0, 1, 2], [1, 2, 0]]
+        values = [1.0, 2.0, 3.0]
+        dense_shape = [3, 3]
+        coo = paddle.sparse.sparse_coo_tensor(indices, values, dense_shape)
+        assert np.array_equal(3, coo.nnz())
+        assert np.array_equal(indices, coo.indices().numpy())
+        assert np.array_equal(values, coo.values().numpy())
+
+    def test_place(self):
+        indices = [[0, 1], [0, 1]]
+        values = [1.0, 2.0]
+        dense_shape = [2, 2]
+        coo = paddle.sparse.sparse_coo_tensor(indices, values, dense_shape)
+        assert coo.place.is_xpu_place()
+        assert coo.values().place.is_xpu_place()
+        assert coo.indices().place.is_xpu_place()
+
+    def test_dtype(self):
+        indices = [[0, 1], [0, 1]]
+        values = [1.0, 2.0]
+        dense_shape = [2, 2]
+        indices = paddle.to_tensor(indices, dtype='int32')
+        values = paddle.to_tensor(values, dtype='float32')
+        coo = paddle.sparse.sparse_coo_tensor(
+            indices, values, dense_shape, dtype='float64'
+        )
+        assert coo.dtype == paddle.float64
+
+    def test_create_coo_no_shape(self):
+        indices = [[0, 1], [0, 1]]
+        values = [1.0, 2.0]
+        indices = paddle.to_tensor(indices, dtype='int32')
+        values = paddle.to_tensor(values, dtype='float32')
+        coo = paddle.sparse.sparse_coo_tensor(indices, values)
+        assert [2, 2] == coo.shape
+
+
+if __name__ == "__main__":
+    unittest.main()

From a37ef769eb7148228658788296df927592ded65a Mon Sep 17 00:00:00 2001
From: Kim  Yann <baochen.yang@intel.com>
Date: Tue, 9 May 2023 17:27:09 +0800
Subject: [PATCH 238/405] rem tools/infer_prune_patches (#53596)

---
 .../analysis_predictor.cc.patch               |  31 ---
 tools/infer_prune_patches/analyzer.cc.patch   |  14 --
 .../device_context.cc.patch                   |  46 ----
 tools/infer_prune_patches/jitcode.h.patch     |  15 --
 tools/infer_prune_patches/op_registry.h.patch | 217 ------------------
 .../paddle_analysis_config.h.patch            |  21 --
 tools/infer_prune_patches/paddle_api.h.patch  |  12 -
 .../paddle_inference_api.h.patch              |  16 --
 tools/infer_prune_patches/phi_cmake.patch     |  13 --
 .../tensorrt_subgraph_pass.cc.patch           |  71 ------
 .../thread_local_allocator.cc.patch           |  96 --------
 .../thread_local_allocator.h.patch            |  30 ---
 12 files changed, 582 deletions(-)
 delete mode 100644 tools/infer_prune_patches/analysis_predictor.cc.patch
 delete mode 100644 tools/infer_prune_patches/analyzer.cc.patch
 delete mode 100644 tools/infer_prune_patches/device_context.cc.patch
 delete mode 100644 tools/infer_prune_patches/jitcode.h.patch
 delete mode 100644 tools/infer_prune_patches/op_registry.h.patch
 delete mode 100644 tools/infer_prune_patches/paddle_analysis_config.h.patch
 delete mode 100644 tools/infer_prune_patches/paddle_api.h.patch
 delete mode 100644 tools/infer_prune_patches/paddle_inference_api.h.patch
 delete mode 100644 tools/infer_prune_patches/phi_cmake.patch
 delete mode 100644 tools/infer_prune_patches/tensorrt_subgraph_pass.cc.patch
 delete mode 100644 tools/infer_prune_patches/thread_local_allocator.cc.patch
 delete mode 100644 tools/infer_prune_patches/thread_local_allocator.h.patch

diff --git a/tools/infer_prune_patches/analysis_predictor.cc.patch b/tools/infer_prune_patches/analysis_predictor.cc.patch
deleted file mode 100644
index 21fa24dd3d957..0000000000000
--- a/tools/infer_prune_patches/analysis_predictor.cc.patch
+++ /dev/null
@@ -1,31 +0,0 @@
-diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc
-index 0645af611b..6b05a7fffb 100644
---- a/paddle/fluid/inference/api/analysis_predictor.cc
-+++ b/paddle/fluid/inference/api/analysis_predictor.cc
-@@ -1923,7 +1923,7 @@ USE_TRT_CONVERTER(shuffle_channel);
- USE_TRT_CONVERTER(swish);
- USE_TRT_CONVERTER(group_norm);
- USE_TRT_CONVERTER(instance_norm);
--USE_TRT_CONVERTER(layer_norm);
-+//USE_TRT_CONVERTER(layer_norm);
- USE_TRT_CONVERTER(gelu);
- USE_TRT_CONVERTER(multihead_matmul);
- USE_TRT_CONVERTER(fused_embedding_eltwise_layernorm);
-@@ -1933,13 +1933,13 @@ USE_TRT_CONVERTER(scale);
- USE_TRT_CONVERTER(stack);
- USE_TRT_CONVERTER(clip);
- USE_TRT_CONVERTER(gather);
--USE_TRT_CONVERTER(anchor_generator);
-+//USE_TRT_CONVERTER(anchor_generator);
- USE_TRT_CONVERTER(yolo_box);
- USE_TRT_CONVERTER(yolo_box_head);
- USE_TRT_CONVERTER(arg_max);
--USE_TRT_CONVERTER(roi_align);
--USE_TRT_CONVERTER(affine_channel);
--USE_TRT_CONVERTER(multiclass_nms);
-+//USE_TRT_CONVERTER(roi_align);
-+//USE_TRT_CONVERTER(affine_channel);
-+//USE_TRT_CONVERTER(multiclass_nms);
- USE_TRT_CONVERTER(multiclass_nms3);
- USE_TRT_CONVERTER(nearest_interp);
- USE_TRT_CONVERTER(nearest_interp_v2);
diff --git a/tools/infer_prune_patches/analyzer.cc.patch b/tools/infer_prune_patches/analyzer.cc.patch
deleted file mode 100644
index 59a7b4d6b8c2e..0000000000000
--- a/tools/infer_prune_patches/analyzer.cc.patch
+++ /dev/null
@@ -1,14 +0,0 @@
-diff --git a/paddle/fluid/inference/analysis/analyzer.cc b/paddle/fluid/inference/analysis/analyzer.cc
-index be7d6ab868..498e09cb4d 100644
---- a/paddle/fluid/inference/analysis/analyzer.cc
-+++ b/paddle/fluid/inference/analysis/analyzer.cc
-@@ -32,6 +32,9 @@ void Analyzer::RunAnalysis(Argument *argument) {
-                         "analsis_passes is not valid in the argument."));
-   const bool disable_logs = argument->disable_logs();
-   for (auto &pass : argument->analysis_passes()) {
-+    if (pass == "ir_params_sync_among_devices_pass") {
-+      continue;
-+    }
-     if (!disable_logs) {
-       string::PrettyLogH1("--- Running analysis [%s]", pass);
-     }
diff --git a/tools/infer_prune_patches/device_context.cc.patch b/tools/infer_prune_patches/device_context.cc.patch
deleted file mode 100644
index 75be9a0d1d997..0000000000000
--- a/tools/infer_prune_patches/device_context.cc.patch
+++ /dev/null
@@ -1,46 +0,0 @@
-diff --git a/paddle/fluid/platform/device_context.cc b/paddle/fluid/platform/device_context.cc
-index 904e4854ba..4f8c955d8c 100644
---- a/paddle/fluid/platform/device_context.cc
-+++ b/paddle/fluid/platform/device_context.cc
-@@ -466,15 +466,15 @@ CUDAContext::CUDAContext(const CUDAPlace& place,
-   place_ = place;
-   CUDADeviceGuard guard(place_.device);
-   stream_.reset(new stream::CUDAStream(place, priority, flag));
--  InitEigenContext();
--  InitCuBlasContext();
--  InitCuDNNContext();
-+  //InitEigenContext();
-+  //InitCuBlasContext();
-+  //InitCuDNNContext();
- #ifndef PADDLE_WITH_HIP
- #if CUDA_VERSION >= 11060
--  InitCuBlasLtContext();
-+  //InitCuBlasLtContext();
- #endif
--  InitCuSparseContext();
--  InitCuSolverContext();
-+  //InitCuSparseContext();
-+  //InitCuSolverContext();
- #endif
- }
- 
-@@ -506,14 +506,14 @@ void CUDAContext::SetStream(gpuStream_t stream) {
- 
- CUDAContext::~CUDAContext() {
-   CUDADeviceGuard guard(place_.device);
--  DestoryCuDNNContext();
--  DestoryCuBlasContext();
-+  //DestoryCuDNNContext();
-+  //DestoryCuBlasContext();
- #ifndef PADDLE_WITH_HIP
- #if CUDA_VERSION >= 11060
--  InitCuBlasLtContext();
-+  //InitCuBlasLtContext();
- #endif
--  DestoryCuSparseContext();
--  DestoryCuSolverContext();
-+  //DestoryCuSparseContext();
-+  //DestoryCuSolverContext();
- #endif
- }
- 
diff --git a/tools/infer_prune_patches/jitcode.h.patch b/tools/infer_prune_patches/jitcode.h.patch
deleted file mode 100644
index a123939a4955b..0000000000000
--- a/tools/infer_prune_patches/jitcode.h.patch
+++ /dev/null
@@ -1,15 +0,0 @@
-diff --git a/paddle/fluid/operators/jit/gen/jitcode.h b/paddle/fluid/operators/jit/gen/jitcode.h
-index d71497275d..cb56e1d949 100644
---- a/paddle/fluid/operators/jit/gen/jitcode.h
-+++ b/paddle/fluid/operators/jit/gen/jitcode.h
-@@ -102,8 +102,8 @@ class JitCode : public GenBase, public Xbyak::CodeGenerator {
-     }
-     ret();
-   }
--  void L(const char* label) { Xbyak::CodeGenerator::L(label); }
--  void L(Xbyak::Label& label) { Xbyak::CodeGenerator::L(label); }  // NOLINT
-+  void L(const char* label) {  }
-+  void L(Xbyak::Label& label) {  }  // NOLINT
-   // Enhanced vector extension
-   Xbyak::Address EVEX_compress_addr(Xbyak::Reg64 base,
-                                     int offt,
diff --git a/tools/infer_prune_patches/op_registry.h.patch b/tools/infer_prune_patches/op_registry.h.patch
deleted file mode 100644
index fb41c98de50f4..0000000000000
--- a/tools/infer_prune_patches/op_registry.h.patch
+++ /dev/null
@@ -1,217 +0,0 @@
-diff --git a/paddle/fluid/framework/op_registry.h b/paddle/fluid/framework/op_registry.h
-index d38efbff31..f5bef776d6 100644
---- a/paddle/fluid/framework/op_registry.h
-+++ b/paddle/fluid/framework/op_registry.h
-@@ -186,9 +186,8 @@ struct OpKernelRegistrarFunctor<PlaceType, false, I, KernelTypes...> {
-         library_type,
-         customized_type_value,
- 
--        [op_type](const framework::ExecutionContext& ctx) {
-+        [](const framework::ExecutionContext& ctx) {
-           KERNEL_TYPE().Compute(ctx);
--          CheckKernelLaunch<PlaceType>(op_type);
-         });
-     constexpr auto size = std::tuple_size<std::tuple<KernelTypes...>>::value;
-     OpKernelRegistrarFunctor<PlaceType, I + 1 == size, I + 1, KernelTypes...>
-@@ -257,15 +256,8 @@ struct OpKernelRegistrarFunctorEx<PlaceType,
-   void operator()(const char* op_type,
-                   const char* library_type,
-                   int customized_type_value) const {
--    RegisterKernelClass<PlaceType, T>(
--        op_type,
--        library_type,
--        customized_type_value,
--
--        [op_type](const framework::ExecutionContext& ctx) {
--          Functor()(ctx);
--          CheckKernelLaunch<PlaceType>(op_type);
--        });
-+    RegisterKernelClass<PlaceType, T>(op_type, library_type,
-+                                      customized_type_value, Functor());
- 
-     constexpr auto size =
-         std::tuple_size<std::tuple<DataTypeAndKernelType...>>::value;
-@@ -296,7 +288,7 @@ struct OpKernelRegistrarFunctorEx<PlaceType,
-     VarTypeInference
-     InferShapeBase
- */
--#define REGISTER_OPERATOR(op_type, op_class, ...)                        \
-+#define REGISTER_OPERATOR__(op_type, op_class, ...)                        \
-   STATIC_ASSERT_GLOBAL_NAMESPACE(                                        \
-       __reg_op__##op_type,                                               \
-       "REGISTER_OPERATOR must be called in global namespace");           \
-@@ -307,15 +299,22 @@ struct OpKernelRegistrarFunctorEx<PlaceType,
-     return 0;                                                            \
-   }
- 
--#define REGISTER_OP_WITHOUT_GRADIENT(op_type, op_class, ...) \
-+#define REGISTER_OPERATOR(op_type, op_class, ...)
-+
-+#define REGISTER_OP_WITHOUT_GRADIENT__(op_type, op_class, ...) \
-   REGISTER_OPERATOR(op_type, op_class, __VA_ARGS__, \
-         paddle::framework::EmptyGradOpMaker<paddle::framework::OpDesc>,   \
-         paddle::framework::EmptyGradOpMaker<paddle::imperative::OpBase>)
- 
-+#define REGISTER_OP_WITHOUT_GRADIENT(op_type, op_class, ...)
- /**
-  * Macro to register OperatorKernel.
-  */
- #define REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE(op_type, library_type,             \
-+                                            place_class, customized_name,      \
-+                                            customized_type_value, ...)
-+
-+#define REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE__(op_type, library_type,             \
-                                             place_class, customized_name,      \
-                                             customized_type_value, ...)        \
-   STATIC_ASSERT_GLOBAL_NAMESPACE(                                              \
-@@ -332,18 +331,22 @@ struct OpKernelRegistrarFunctorEx<PlaceType,
-     return 0;                                                                  \
-   }
- 
--#define REGISTER_OP_KERNEL(op_type, library_type, place_class, ...)   \
--  REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE(                                \
-+#define REGISTER_OP_KERNEL(op_type, library_type, place_class, ...)
-+
-+#define REGISTER_OP_KERNEL__(op_type, library_type, place_class, ...)   \
-+  REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE__(                                \
-       op_type, library_type, place_class, DEFAULT_TYPE,               \
-       ::paddle::framework::OpKernelType::kDefaultCustomizedTypeValue, \
-       __VA_ARGS__)
- 
--#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-+#define REGISTER_OP_CUDA_KERNEL__(op_type, ...) \
-+  REGISTER_OP_KERNEL__(op_type, CUDA, ::paddle::platform::CUDAPlace, __VA_ARGS__)
-+
- #define REGISTER_OP_CUDA_KERNEL(op_type, ...) \
-   REGISTER_OP_KERNEL(op_type, CUDA, ::paddle::platform::CUDAPlace, __VA_ARGS__)
--#else
--#define REGISTER_OP_CUDA_KERNEL(op_type, ...)
--#endif
-+
-+#define REGISTER_OP_CPU_KERNEL__(op_type, ...) \
-+  REGISTER_OP_KERNEL__(op_type, CPU, ::paddle::platform::CPUPlace, __VA_ARGS__)
- 
- #define REGISTER_OP_CPU_KERNEL(op_type, ...) \
-   REGISTER_OP_KERNEL(op_type, CPU, ::paddle::platform::CPUPlace, __VA_ARGS__)
-@@ -361,6 +364,11 @@ struct OpKernelRegistrarFunctorEx<PlaceType,
-   REGISTER_OP_KERNEL(op_type, MLU, ::paddle::platform::MLUPlace, __VA_ARGS__)
- 
- #define REGISTER_OP_KERNEL_EX(op_type, library_type, place_class,  \
-+                              customized_name,                     \
-+                              customized_type_value,               \
-+			      ...)
-+
-+#define REGISTER_OP_KERNEL_EX__(op_type, library_type, place_class,  \
-                               customized_name,                     \
-                               customized_type_value,               \
-                               ...)                                 \
-@@ -378,8 +386,10 @@ struct OpKernelRegistrarFunctorEx<PlaceType,
-     return 0;                                                                  \
-   }
- 
--#define REGISTER_OP_CUDA_KERNEL_FUNCTOR(op_type, ...)                 \
--  REGISTER_OP_KERNEL_EX(                                              \
-+#define REGISTER_OP_CUDA_KERNEL_FUNCTOR(op_type, ...)
-+
-+#define REGISTER_OP_CUDA_KERNEL_FUNCTOR__(op_type, ...)                 \
-+  REGISTER_OP_KERNEL_EX__(                                              \
-       op_type, CUDA, ::paddle::platform::CUDAPlace, DEFAULT_TYPE,     \
-       ::paddle::framework::OpKernelType::kDefaultCustomizedTypeValue, \
-       __VA_ARGS__)
-@@ -396,12 +406,6 @@ struct OpKernelRegistrarFunctorEx<PlaceType,
-       ::paddle::framework::OpKernelType::kDefaultCustomizedTypeValue, \
-       __VA_ARGS__)
- 
--#define REGISTER_OP_NPU_KERNEL_FUNCTOR(op_type, ...)                  \
--  REGISTER_OP_KERNEL_EX(                                              \
--      op_type, NPU, ::paddle::platform::NPUPlace, DEFAULT_TYPE,       \
--      ::paddle::framework::OpKernelType::kDefaultCustomizedTypeValue, \
--      __VA_ARGS__)
--
- #define REGISTER_OP_MLU_KERNEL_FUNCTOR(op_type, ...)                  \
-   REGISTER_OP_KERNEL_EX(                                              \
-       op_type, MLU, ::paddle::platform::MLUPlace, DEFAULT_TYPE,       \
-@@ -413,7 +417,9 @@ struct OpKernelRegistrarFunctorEx<PlaceType,
-  * we will use and tell the compiler to
-  * link them into target.
-  */
--#define USE_OP_ITSELF(op_type)                             \
-+#define USE_OP_ITSELF(op_type)
-+
-+#define USE_OP_ITSELF__(op_type)                             \
-   STATIC_ASSERT_GLOBAL_NAMESPACE(                          \
-       __use_op_itself_##op_type,                           \
-       "USE_OP_ITSELF must be called in global namespace"); \
-@@ -421,6 +427,10 @@ struct OpKernelRegistrarFunctorEx<PlaceType,
-   UNUSED static int use_op_itself_##op_type##_ = TouchOpRegistrar_##op_type()
- 
- #define USE_OP_DEVICE_KERNEL_WITH_CUSTOM_TYPE(op_type,                     \
-+                                              LIBRARY_TYPE,                \
-+                                              customized_name)
-+
-+#define USE_OP_DEVICE_KERNEL_WITH_CUSTOM_TYPE__(op_type,                     \
-                                               LIBRARY_TYPE,                \
-                                               customized_name)             \
-   STATIC_ASSERT_GLOBAL_NAMESPACE(                                          \
-@@ -431,33 +441,58 @@ struct OpKernelRegistrarFunctorEx<PlaceType,
-   UNUSED static int use_op_kernel_##op_type##_##LIBRARY_TYPE##_##customized_name##_ = /* NOLINT */ \
-       TouchOpKernelRegistrar_##op_type##_##LIBRARY_TYPE##_##customized_name()
- 
--#define USE_OP_DEVICE_KERNEL(op_type, LIBRARY_TYPE) \
--  USE_OP_DEVICE_KERNEL_WITH_CUSTOM_TYPE(op_type, LIBRARY_TYPE, DEFAULT_TYPE)
-+#define USE_OP_DEVICE_KERNEL_WITH_CUSTOM_TYPE(op_type,                \
-+                                              LIBRARY_TYPE,           \
-+                                              customized_name)
-+
-+#define USE_OP_DEVICE_KERNEL__(op_type, LIBRARY_TYPE) \
-+  USE_OP_DEVICE_KERNEL_WITH_CUSTOM_TYPE__(op_type, LIBRARY_TYPE, DEFAULT_TYPE)
-+
-+#define USE_OP_DEVICE_KERNEL(op_type, LIBRARY_TYPE)
- 
- // TODO(fengjiayi): The following macros
- // seems ugly, do we have better method?
- 
--#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
-+#ifndef PADDLE_WITH_CUDA
- #define USE_OP_KERNEL(op_type) USE_OP_DEVICE_KERNEL(op_type, CPU)
-+#define USE_OP_KERNEL__(op_type) USE_OP_DEVICE_KERNEL__(op_type, CPU)
- #else
- #define USE_OP_KERNEL(op_type)        \
-   USE_OP_DEVICE_KERNEL(op_type, CPU); \
-   USE_OP_DEVICE_KERNEL(op_type, CUDA)
-+
-+#define USE_OP_KERNEL__(op_type)        \
-+  USE_OP_DEVICE_KERNEL__(op_type, CPU); \
-+  USE_OP_DEVICE_KERNEL__(op_type, CUDA)
- #endif
- 
- #define USE_NO_KERNEL_OP(op_type) USE_OP_ITSELF(op_type);
- 
-+#define USE_NO_KERNEL_OP__(op_type) USE_OP_ITSELF__(op_type);
-+
- #define USE_CPU_ONLY_OP(op_type) \
-   USE_OP_ITSELF(op_type);        \
-   USE_OP_DEVICE_KERNEL(op_type, CPU);
- 
-+#define USE_CPU_ONLY_OP__(op_type) \
-+  USE_OP_ITSELF__(op_type);        \
-+  USE_OP_DEVICE_KERNEL__(op_type, CPU);
-+
- #define USE_CUDA_ONLY_OP(op_type) \
-   USE_OP_ITSELF(op_type);         \
-   USE_OP_DEVICE_KERNEL(op_type, CUDA)
- 
-+#define USE_CUDA_ONLY_OP__(op_type) \
-+  USE_OP_ITSELF__(op_type);         \
-+  USE_OP_DEVICE_KERNEL__(op_type, CUDA)
-+
- #define USE_OP(op_type)   \
-   USE_OP_ITSELF(op_type); \
-   USE_OP_KERNEL(op_type)
-+
-+#define USE_OP__(op_type)   \
-+  USE_OP_ITSELF__(op_type); \
-+  USE_OP_KERNEL__(op_type)
- // clang-format on
- 
- }  // namespace framework
diff --git a/tools/infer_prune_patches/paddle_analysis_config.h.patch b/tools/infer_prune_patches/paddle_analysis_config.h.patch
deleted file mode 100644
index 292a1a7e10f00..0000000000000
--- a/tools/infer_prune_patches/paddle_analysis_config.h.patch
+++ /dev/null
@@ -1,21 +0,0 @@
-diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h
-index d6a0b643c2..511844b482 100644
---- a/paddle/fluid/inference/api/paddle_analysis_config.h
-+++ b/paddle/fluid/inference/api/paddle_analysis_config.h
-@@ -46,6 +46,7 @@
- namespace paddle {
- 
- class AnalysisPredictor;
-+class TensorRTPredictor;
- struct MkldnnQuantizerConfig;
- 
- struct LiteNNAdapterConfig {
-@@ -700,6 +701,8 @@ struct PD_INFER_DECL AnalysisConfig {
- 
-   friend class ::paddle::AnalysisPredictor;
- 
-+  friend class ::paddle::TensorRTPredictor;
-+
-   ///
-   /// \brief Get a pass builder for customize the passes in IR analysis phase.
-   /// NOTE: Just for developer, not an official API, easy to be broken.
diff --git a/tools/infer_prune_patches/paddle_api.h.patch b/tools/infer_prune_patches/paddle_api.h.patch
deleted file mode 100644
index 3d7baa7f21826..0000000000000
--- a/tools/infer_prune_patches/paddle_api.h.patch
+++ /dev/null
@@ -1,12 +0,0 @@
-diff --git a/paddle/fluid/inference/api/paddle_api.h b/paddle/fluid/inference/api/paddle_api.h
-index b28370fb82..aaf20a28b0 100644
---- a/paddle/fluid/inference/api/paddle_api.h
-+++ b/paddle/fluid/inference/api/paddle_api.h
-@@ -194,6 +194,7 @@ class PD_INFER_DECL ZeroCopyTensor : public paddle_infer::Tensor {
- 
-  private:
-   friend class AnalysisPredictor;
-+  friend class TensorRTPredictor;
-   friend class ONNXRuntimePredictor;
-   explicit ZeroCopyTensor(void* scope, const void* device_contexts)
-       : paddle_infer::Tensor{scope, device_contexts} {}
diff --git a/tools/infer_prune_patches/paddle_inference_api.h.patch b/tools/infer_prune_patches/paddle_inference_api.h.patch
deleted file mode 100644
index 96de7e189e744..0000000000000
--- a/tools/infer_prune_patches/paddle_inference_api.h.patch
+++ /dev/null
@@ -1,16 +0,0 @@
-diff --git a/paddle/fluid/inference/api/paddle_inference_api.h b/paddle/fluid/inference/api/paddle_inference_api.h
-index 35b90bfa54..ba8220d06a 100644
---- a/paddle/fluid/inference/api/paddle_inference_api.h
-+++ b/paddle/fluid/inference/api/paddle_inference_api.h
-@@ -41,6 +41,11 @@ limitations under the License. */
- /// \since 2.0.0-beta
- ///
- 
-+namespace paddle {
-+std::unique_ptr<PaddlePredictor> CreateTensorRTPredictor(
-+    const AnalysisConfig& config);
-+}
-+
- namespace paddle_infer {
- 
- using PrecisionType = paddle::AnalysisConfig::Precision;
diff --git a/tools/infer_prune_patches/phi_cmake.patch b/tools/infer_prune_patches/phi_cmake.patch
deleted file mode 100644
index 1d9c3f75343ff..0000000000000
--- a/tools/infer_prune_patches/phi_cmake.patch
+++ /dev/null
@@ -1,13 +0,0 @@
-diff --git a/paddle/phi/CMakeLists.txt b/paddle/phi/CMakeLists.txt
-index 9715fd7704..44109e1081 100644
---- a/paddle/phi/CMakeLists.txt
-+++ b/paddle/phi/CMakeLists.txt
-@@ -18,7 +18,7 @@ add_subdirectory(infermeta)
- # phi operator definitions
- add_subdirectory(ops)
- # phi tools
--add_subdirectory(tools)
-+#add_subdirectory(tools)
- # phi tests
- add_subdirectory(tests)
- # phi capi
diff --git a/tools/infer_prune_patches/tensorrt_subgraph_pass.cc.patch b/tools/infer_prune_patches/tensorrt_subgraph_pass.cc.patch
deleted file mode 100644
index f038a125f648f..0000000000000
--- a/tools/infer_prune_patches/tensorrt_subgraph_pass.cc.patch
+++ /dev/null
@@ -1,71 +0,0 @@
-diff --git a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc
-index 7a9c5b889d..c847a5d523 100644
---- a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc
-+++ b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc
-@@ -418,6 +418,7 @@ void TensorRtSubgraphPass::CreateTensorRTOp(
-       graph->Has(framework::ir::kEmbEltwiseLayernormPass) &&
-       graph->Has(framework::ir::kMultiheadMatmulPass));
- 
-+  std::unordered_set<std::string> param_set(params.begin(), params.end());
-   if (use_static_engine) {
-     trt_engine_serialized_data = GetTrtEngineSerializedData(
-         Get<std::string>("model_opt_cache_dir"), engine_key);
-@@ -427,6 +428,19 @@ void TensorRtSubgraphPass::CreateTensorRTOp(
-       LOG(INFO) << "Load TRT Optimized Info from "
-                 << GetTrtEngineSerializedPath(
-                        Get<std::string>("model_opt_cache_dir"), engine_key);
-+      const auto* root_scope{param_scope()};
-+      for (;root_scope->parent();) {
-+        root_scope = root_scope->parent();
-+      }
-+      for (const auto& name: param_set) {
-+        LOG(INFO) << " ===== Clear param: " << name;
-+        root_scope->FindLocalVar(name)->Clear();
-+      }
-+      for (int dev_id = 0; dev_id < paddle::platform::GetGPUDeviceCount();
-+          ++dev_id) {
-+        memory::Release(platform::CUDAPlace(dev_id));
-+      }
-+      memory::Release(platform::CPUPlace());
-       return;
-     }
-   }
-@@ -439,7 +453,6 @@ void TensorRtSubgraphPass::CreateTensorRTOp(
- 
-   auto *scope = param_scope();
-   framework::BlockDesc block_desc_temp(nullptr, block_desc.Proto());
--  std::unordered_set<std::string> param_set(params.begin(), params.end());
-   inference::Singleton<inference::tensorrt::OpConverter>::Global()
-       .ConvertBlockToTRTEngine(
-           &block_desc_temp,
-@@ -449,6 +462,21 @@ void TensorRtSubgraphPass::CreateTensorRTOp(
-           output_mapping,
-           trt_engine);
- 
-+  const auto* root_scope{scope};
-+  for (;root_scope->parent();) {
-+    root_scope = root_scope->parent();
-+  }
-+  VLOG(4) << "root_scope->LocalVarNames().size: " << root_scope->LocalVarNames().size();
-+  for (const auto& name: param_set) {
-+    VLOG(4) << "  ===== Clear param: " << name;
-+    root_scope->FindLocalVar(name)->Clear();
-+  }
-+  for (int dev_id = 0; dev_id < paddle::platform::GetGPUDeviceCount();
-+       ++dev_id) {
-+    memory::Release(platform::CUDAPlace(dev_id));
-+  }
-+  memory::Release(platform::CPUPlace());
-+
-   if (use_static_engine) {
-     nvinfer1::IHostMemory *serialized_engine_data = trt_engine->Serialize();
-     trt_engine_serialized_data =
-@@ -462,6 +490,8 @@ void TensorRtSubgraphPass::CreateTensorRTOp(
-               << GetTrtEngineSerializedPath(
-                      Get<std::string>("model_opt_cache_dir"), engine_key);
-   }
-+  trt_engine_serialized_data.clear();
-+  trt_engine_serialized_data.shrink_to_fit();
- }
- 
- }  // namespace analysis
diff --git a/tools/infer_prune_patches/thread_local_allocator.cc.patch b/tools/infer_prune_patches/thread_local_allocator.cc.patch
deleted file mode 100644
index b95480ad91dc7..0000000000000
--- a/tools/infer_prune_patches/thread_local_allocator.cc.patch
+++ /dev/null
@@ -1,96 +0,0 @@
-diff --git a/paddle/fluid/memory/allocation/thread_local_allocator.cc b/paddle/fluid/memory/allocation/thread_local_allocator.cc
-index 875e57cfd4..b111ada3ab 100644
---- a/paddle/fluid/memory/allocation/thread_local_allocator.cc
-+++ b/paddle/fluid/memory/allocation/thread_local_allocator.cc
-@@ -13,19 +13,62 @@
- // limitations under the License.
- 
- #include "paddle/fluid/memory/allocation/thread_local_allocator.h"
-+#include "paddle/fluid/platform/cuda_device_guard.h"
- 
- namespace paddle {
- namespace memory {
- namespace allocation {
- 
-+const int MALLOC_ALIGN = 64;
-+
-+#define CUDA_CALL(func)                                      \
-+  {                                                          \
-+    auto e = (func);                                         \
-+    CHECK(e == cudaSuccess || e == cudaErrorCudartUnloading) \
-+        << "CUDA: " << cudaGetErrorString(e);                \
-+  }
-+
-+void* DirectAllocator::Alloc(size_t unaligned_size) {
-+  if (platform::is_cpu_place(place_)) {
-+    size_t offset = sizeof(void*) + MALLOC_ALIGN - 1;
-+    char* p = static_cast<char*>(std::malloc(offset + unaligned_size));
-+    // Memory checking
-+    CHECK(p) << "Error occurred in malloc period: available space is not enough "
-+                "for mallocing "
-+            << unaligned_size << " bytes.";
-+    // Byte alignment
-+    void* r = reinterpret_cast<void*>(reinterpret_cast<size_t>(p + offset) &
-+                                      (~(MALLOC_ALIGN - 1)));
-+    static_cast<void**>(r)[-1] = p;
-+    return r;
-+  } else if (platform::is_gpu_place(place_)) {
-+    int dev_id = place_.GetDeviceId();
-+    platform::CUDADeviceGuard guard(dev_id);
-+    void* ptr{};
-+    CUDA_CALL(cudaMalloc(&ptr, unaligned_size));
-+    return ptr;
-+  }
-+  return nullptr;
-+}
-+
-+void DirectAllocator::Free(void* ptr) {
-+  if (platform::is_cpu_place(place_)) {
-+    if (ptr) {
-+      std::free(static_cast<void**>(ptr)[-1]);
-+    } 
-+  } else if (platform::is_gpu_place(place_)) {
-+    int dev_id = place_.GetDeviceId();
-+    platform::CUDADeviceGuard guard(dev_id);
-+    CUDA_CALL(cudaFree(ptr));
-+  }
-+}
-+
-+
-+
- ThreadLocalAllocatorImpl::ThreadLocalAllocatorImpl(const platform::Place& p)
-     : place_(p) {
-   if (platform::is_gpu_place(place_)) {
--    buddy_allocator_.reset(new memory::detail::BuddyAllocator(
--        std::unique_ptr<memory::detail::SystemAllocator>(
--            new memory::detail::GPUAllocator(place_.device)),
--        platform::GpuMinChunkSize(),
--        platform::GpuMaxChunkSize()));
-+    direct_allocator_.reset(new DirectAllocator{place_});
-   } else {
-     PADDLE_THROW(platform::errors::Unavailable(
-         "Thread local allocator only supports CUDAPlace now."));
-@@ -61,7 +104,7 @@ ThreadLocalCUDAAllocatorPool::ThreadLocalCUDAAllocatorPool()
- 
- ThreadLocalAllocation* ThreadLocalAllocatorImpl::AllocateImpl(size_t size) {
-   VLOG(10) << "ThreadLocalAllocatorImpl::AllocateImpl " << size;
--  void* ptr = buddy_allocator_->Alloc(size);
-+  void* ptr = direct_allocator_->Alloc(size);
-   auto* tl_allocation = new ThreadLocalAllocation(ptr, size, place_);
-   tl_allocation->SetThreadLocalAllocatorImpl(shared_from_this());
-   return tl_allocation;
-@@ -69,12 +112,12 @@ ThreadLocalAllocation* ThreadLocalAllocatorImpl::AllocateImpl(size_t size) {
- 
- void ThreadLocalAllocatorImpl::FreeImpl(ThreadLocalAllocation* allocation) {
-   VLOG(10) << "ThreadLocalAllocatorImpl::FreeImpl " << allocation;
--  buddy_allocator_->Free(allocation->ptr());
-+  direct_allocator_->Free(allocation->ptr());
-   delete allocation;
- }
- 
- uint64_t ThreadLocalAllocatorImpl::ReleaseImpl() {
--  return buddy_allocator_->Release();
-+  return direct_allocator_->Release();
- }
- 
- }  // namespace allocation
diff --git a/tools/infer_prune_patches/thread_local_allocator.h.patch b/tools/infer_prune_patches/thread_local_allocator.h.patch
deleted file mode 100644
index a3c24178d2093..0000000000000
--- a/tools/infer_prune_patches/thread_local_allocator.h.patch
+++ /dev/null
@@ -1,30 +0,0 @@
-diff --git a/paddle/fluid/memory/allocation/thread_local_allocator.h b/paddle/fluid/memory/allocation/thread_local_allocator.h
-index 654fb3fe7b..44c5dbf87f 100644
---- a/paddle/fluid/memory/allocation/thread_local_allocator.h
-+++ b/paddle/fluid/memory/allocation/thread_local_allocator.h
-@@ -26,6 +26,16 @@ namespace paddle {
- namespace memory {
- namespace allocation {
- 
-+class DirectAllocator {
-+public:
-+  DirectAllocator(const platform::Place& place) : place_{place} {}
-+  void* Alloc(size_t unaligned_size);
-+  void Free(void* ptr);
-+  uint64_t Release() { return 0;}
-+private:
-+  platform::Place place_;
-+};
-+
- class ThreadLocalAllocatorImpl;
- 
- class ThreadLocalAllocation : public Allocation {
-@@ -55,7 +65,7 @@ class ThreadLocalAllocatorImpl
-   uint64_t ReleaseImpl();
- 
-  private:
--  std::unique_ptr<memory::detail::BuddyAllocator> buddy_allocator_;
-+  std::unique_ptr<DirectAllocator> direct_allocator_;
-   platform::Place place_;
- };
- 

From e588f2d93fca03fde1f8542d4f5d027f028b165b Mon Sep 17 00:00:00 2001
From: zhwesky2010 <1183042833@qq.com>
Date: Tue, 9 May 2023 17:31:18 +0800
Subject: [PATCH 239/405] [Zero-Dim] add 0D Tensor UT case for XPU and expand
 kernel support 0D (#53555)

* [Zero-Dim] add 0D Tensor UT case for XPU

* fix comment

* remove some unnecessary UT
---
 paddle/phi/kernels/xpu/expand_grad_kernel.cc |    3 +-
 paddle/phi/kernels/xpu/expand_kernel.cc      |   17 +-
 test/xpu/test_zero_dim_tensor_xpu.py         | 1458 ++++++++++++++++--
 3 files changed, 1317 insertions(+), 161 deletions(-)

diff --git a/paddle/phi/kernels/xpu/expand_grad_kernel.cc b/paddle/phi/kernels/xpu/expand_grad_kernel.cc
index 52fc0fd38f1f8..1665b8e31926c 100644
--- a/paddle/phi/kernels/xpu/expand_grad_kernel.cc
+++ b/paddle/phi/kernels/xpu/expand_grad_kernel.cc
@@ -37,7 +37,8 @@ void ExpandGradKernel(const Context& ctx,
 
   // Two zero
   if (out_grad_dims.size() == 0 && in_grad_dims.size() == 0) {
-    return;
+    out_grad_dims = {1};
+    in_grad_dims = {1};
   }
 
   int r = xpu::expand_grad<XPUType>(
diff --git a/paddle/phi/kernels/xpu/expand_kernel.cc b/paddle/phi/kernels/xpu/expand_kernel.cc
index 10b8c18e9d33a..d8808d3c3aae3 100644
--- a/paddle/phi/kernels/xpu/expand_kernel.cc
+++ b/paddle/phi/kernels/xpu/expand_kernel.cc
@@ -94,26 +94,17 @@ void ExpandKernel(const Context& ctx,
           shape_size,
           rank));
 
-  if (shape_size == 0) {
-    phi::DDim out_dims = phi::make_ddim(final_expand_shape);
-    out->Resize(out_dims);
-    ctx.template Alloc<T>(out);
-
-    int r = xpu::copy<XPUType>(ctx.x_context(),
-                               reinterpret_cast<const XPUType*>(x.data<T>()),
-                               reinterpret_cast<XPUType*>(out->data<T>()),
-                               x.numel());
-    PADDLE_ENFORCE_XDNN_SUCCESS(r, "copy");
-    return;
-  }
   DDim out_dims = phi::make_ddim(final_expand_shape);
   out->Resize(out_dims);
   ctx.template Alloc<T>(out);
   auto& x_shape = vec_in_dims;
   auto out_shape = phi::vectorize<int>(out_dims);
+  if (shape_size == 0) {
+    x_shape = {1};
+    out_shape = {1};
+  }
 
   int r = XPU_SUCCESS;
-
   if (std::is_same<T, bool>::value) {
     auto x_data = reinterpret_cast<const int8_t*>(x.data<T>());
     auto out_data = reinterpret_cast<int8_t*>(out->data<T>());
diff --git a/test/xpu/test_zero_dim_tensor_xpu.py b/test/xpu/test_zero_dim_tensor_xpu.py
index 9369a9b0ed3ff..7591b3a402f5c 100644
--- a/test/xpu/test_zero_dim_tensor_xpu.py
+++ b/test/xpu/test_zero_dim_tensor_xpu.py
@@ -1,4 +1,4 @@
-#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#   Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,6 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Note:
+# 0D Tensor indicates that the tensor's dimension is 0
+# 0D Tensor's shape is always [], numel is 1
+# which can be created by paddle.rand([])
+
+import os
 import unittest
 
 import numpy as np
@@ -20,6 +26,7 @@
 import paddle.nn.functional as F
 
 paddle.set_device('xpu')
+paddle.disable_static()
 
 unary_api_list = [
     paddle.nn.functional.elu,
@@ -86,6 +93,8 @@
     paddle.bernoulli,
     paddle.nn.functional.softmax,
     paddle.nn.functional.log_softmax,
+    paddle.nn.functional.gumbel_softmax,
+    paddle.nn.functional.alpha_dropout,
 ]
 
 inplace_api_list = [
@@ -97,11 +106,11 @@
 # Use to test zero-dim in unary API.
 class TestUnaryAPI(unittest.TestCase):
     def test_dygraph_unary(self):
-        paddle.disable_static()
         for api in unary_api_list:
             x = paddle.rand([])
             x.stop_gradient = False
             out = api(x)
+
             out.retain_grads()
             out.backward()
 
@@ -117,8 +126,6 @@ def test_dygraph_unary(self):
             self.assertEqual(x.shape, [])
             self.assertEqual(out.shape, [])
 
-        paddle.enable_static()
-
 
 reduce_api_list = [
     paddle.sum,
@@ -139,7 +146,6 @@ def test_dygraph_unary(self):
 # Use to test zero-dim of reduce API
 class TestReduceAPI(unittest.TestCase):
     def test_dygraph_reduce(self):
-        paddle.disable_static()
         for api in reduce_api_list:
             # 1) x is 0D
             if api in [paddle.all, paddle.any]:
@@ -148,13 +154,18 @@ def test_dygraph_reduce(self):
                 x = paddle.rand([])
             x.stop_gradient = False
             out = api(x, None)
-            out.retain_grads()
 
+            out.retain_grads()
             out.backward()
 
             self.assertEqual(x.shape, [])
             self.assertEqual(out.shape, [])
             np.testing.assert_allclose(out.numpy(), x.numpy())
+
+            out_empty_list = api(x, [])
+            self.assertEqual(out_empty_list, out)
+            self.assertEqual(out_empty_list.shape, [])
+
             if x.grad is not None:
                 self.assertEqual(x.grad.shape, [])
                 self.assertEqual(out.grad.shape, [])
@@ -175,7 +186,35 @@ def test_dygraph_reduce(self):
                 self.assertEqual(x.grad.shape, [])
                 np.testing.assert_allclose(x.grad.numpy(), np.array(3.0))
 
-        paddle.enable_static()
+            # 2) x is ND, reduce to 0D
+            if api in [paddle.all, paddle.any]:
+                x = paddle.randint(0, 2, [3, 5]).astype('bool')
+            else:
+                x = paddle.rand([3, 5])
+            x.stop_gradient = False
+            out = api(x, None)
+            out.retain_grads()
+            out.backward()
+
+            self.assertEqual(out.shape, [])
+            if x.grad is not None:
+                self.assertEqual(out.grad.shape, [])
+                self.assertEqual(x.grad.shape, [3, 5])
+
+            # 3) x is 1D, axis=0, reduce to 0D
+            if api in [paddle.all, paddle.any]:
+                x = paddle.randint(0, 2, [5]).astype('bool')
+            else:
+                x = paddle.rand([5])
+            x.stop_gradient = False
+            out = api(x, 0)
+            out.retain_grads()
+            out.backward()
+
+            self.assertEqual(out.shape, [])
+            if x.grad is not None:
+                self.assertEqual(out.grad.shape, [])
+                self.assertEqual(x.grad.shape, [5])
 
 
 binary_api_list = [
@@ -198,33 +237,37 @@ def test_dygraph_reduce(self):
     paddle.logical_xor,
     paddle.maximum,
     paddle.minimum,
+    paddle.fmax,
+    paddle.fmin,
+    paddle.complex,
+    paddle.kron,
 ]
 
 binary_int_api_list = [
     paddle.bitwise_and,
     paddle.bitwise_or,
     paddle.bitwise_xor,
+    paddle.gcd,
+    paddle.lcm,
 ]
 
 
 # Use to test zero-dim of binary API
 class TestBinaryAPI(unittest.TestCase):
     def test_dygraph_binary(self):
-        paddle.disable_static()
         for api in binary_api_list:
             # 1) x is 0D, y is 0D
             x = paddle.rand([])
             y = paddle.rand([])
             x.stop_gradient = False
             y.stop_gradient = False
-            x.retain_grads()
-            y.retain_grads()
             if isinstance(api, dict):
                 out = api['func'](x, y)
                 out_cls = getattr(paddle.Tensor, api['cls_method'])(x, y)
                 np.testing.assert_array_equal(out_cls.numpy(), out.numpy())
             else:
                 out = api(x, y)
+
             out.retain_grads()
             out.backward()
 
@@ -247,6 +290,7 @@ def test_dygraph_binary(self):
                 np.testing.assert_array_equal(out_cls.numpy(), out.numpy())
             else:
                 out = api(x, y)
+
             out.retain_grads()
             out.backward()
 
@@ -263,14 +307,13 @@ def test_dygraph_binary(self):
             y = paddle.rand([2, 3, 4])
             x.stop_gradient = False
             y.stop_gradient = False
-            x.retain_grads()
-            y.retain_grads()
             if isinstance(api, dict):
                 out = api['func'](x, y)
                 out_cls = getattr(paddle.Tensor, api['cls_method'])(x, y)
                 np.testing.assert_array_equal(out_cls.numpy(), out.numpy())
             else:
                 out = api(x, y)
+
             out.retain_grads()
             out.backward()
 
@@ -288,6 +331,7 @@ def test_dygraph_binary(self):
             y = 0.5
             if isinstance(api, dict):
                 out = getattr(paddle.Tensor, api['cls_method'])(x, y)
+
                 out.retain_grads()
                 out.backward()
 
@@ -334,14 +378,11 @@ def test_dygraph_binary(self):
             self.assertEqual(out.shape, [3, 5])
             np.testing.assert_array_equal(out.numpy(), out_np)
 
-        paddle.enable_static()
-
 
 # Use to test zero-dim of Sundry API, which is unique and can not be classified
 # with others. It can be implemented here flexibly.
 class TestSundryAPI(unittest.TestCase):
     def setUp(self):
-        paddle.disable_static()
         self.x = paddle.rand([])
 
     def test_getitem(self):
@@ -610,6 +651,100 @@ def test_top_k(self):
         with self.assertRaises(ValueError):
             tmp = paddle.topk(x1, k=1, axis=2)
 
+    def test_broadcast_to(self):
+        x = paddle.full([], 1, 'float32')
+        x.stop_gradient = False
+        out = paddle.broadcast_to(x, shape=[1])
+        out.retain_grads()
+        out.backward()
+
+        self.assertEqual(out.shape, [1])
+        np.testing.assert_allclose(out, 1.0)
+        self.assertEqual(x.grad.shape, [])
+        np.testing.assert_allclose(x.grad, 1.0)
+        self.assertEqual(out.grad.shape, [1])
+        np.testing.assert_allclose(out.grad, 1.0)
+
+        # case2
+        x1 = paddle.full([], 1, 'float32')
+        x1.stop_gradient = False
+        out1 = paddle.broadcast_to(x1, shape=[])
+        out1.retain_grads()
+        out1.backward()
+
+        self.assertEqual(out1.shape, [])
+        np.testing.assert_allclose(out1, 1.0)
+        self.assertEqual(x1.grad.shape, [])
+        np.testing.assert_allclose(x1.grad, 1.0)
+        self.assertEqual(out1.grad.shape, [])
+        np.testing.assert_allclose(out1.grad, 1.0)
+
+        # case3
+        x2 = paddle.full([], 1, 'float32')
+        x2.stop_gradient = False
+        out2 = paddle.broadcast_to(x2, shape=[1, 1])
+        out2.retain_grads()
+        out2.backward()
+
+        self.assertEqual(out2.shape, [1, 1])
+        np.testing.assert_allclose(out2, 1.0)
+        self.assertEqual(x2.grad.shape, [])
+        np.testing.assert_allclose(x2.grad, 1.0)
+        self.assertEqual(out2.grad.shape, [1, 1])
+        np.testing.assert_allclose(out2.grad, 1.0)
+
+        # case4
+        x3 = paddle.full([], 1, 'float32')
+        x3.stop_gradient = False
+        out3 = paddle.broadcast_to(x3, shape=[3, 3])
+        out3.retain_grads()
+        out3.backward()
+
+        self.assertEqual(out3.shape, [3, 3])
+        np.testing.assert_allclose(out3, 1.0)
+        self.assertEqual(x3.grad.shape, [])
+        np.testing.assert_allclose(x3.grad, 9.0)
+        self.assertEqual(out3.grad.shape, [3, 3])
+        np.testing.assert_allclose(out3.grad, 1.0)
+
+    def test_broadcast_tensors(self):
+        # 1) x is 0D, y is 0D
+        x1 = paddle.full([], 2.0)
+        x1.stop_gradient = False
+        x2 = paddle.full([], 2.0)
+        x2.stop_gradient = False
+        out1, out2 = paddle.broadcast_tensors([x1, x2])
+        # backward has bug now
+        # out1.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out2.shape, [])
+        # self.assertEqual(x1.grad.shape, [])
+
+        # 2) x is ND , y is 0D
+        x1 = paddle.full([2, 3], 2.0)
+        x1.stop_gradient = False
+        x2 = paddle.full([], 2.0)
+        x2.stop_gradient = False
+        out1, out2 = paddle.broadcast_tensors([x1, x2])
+        # out1.backward()
+
+        self.assertEqual(out1.shape, [2, 3])
+        self.assertEqual(out2.shape, [2, 3])
+        # self.assertEqual(x1.grad.shape, [2, 3])
+
+        # 3) x is 0D , y is ND
+        x1 = paddle.full([], 2.0)
+        x1.stop_gradient = False
+        x2 = paddle.full([2, 3], 2.0)
+        x2.stop_gradient = False
+        out1, out2 = paddle.broadcast_tensors([x1, x2])
+        # out1.backward()
+
+        self.assertEqual(out1.shape, [2, 3])
+        self.assertEqual(out2.shape, [2, 3])
+        # self.assertEqual(x1.grad.shape, [2, 3])
+
     def test_argmin(self):
         # 1) x is 0D
         x = paddle.rand([])
@@ -679,6 +814,7 @@ def test_argmax(self):
         self.assertEqual(out.shape, [1, 1])
 
     def test_median(self):
+        # 1) x is 0D
         x = paddle.rand([])
         x.stop_gradient = False
         out1 = paddle.median(x, 0)
@@ -701,149 +837,505 @@ def test_median(self):
         self.assertEqual(x.grad.shape, [])
         np.testing.assert_allclose(x.grad, 3.0)
 
-    def test_linear(self):
-        x = paddle.randn([3, 2])
-        w = paddle.full(shape=[2, 4], fill_value=0.5)
-        b = paddle.zeros([])
-
-        np.testing.assert_array_equal(
-            F.linear(x, w, b).numpy(), F.linear(x, w).numpy()
-        )
+        # 2) x is 1D
+        x = paddle.rand([5])
+        x.stop_gradient = False
+        out = paddle.median(x, 0)
+        out.backward()
+        self.assertEqual(out.shape, [])
+        self.assertEqual(x.grad.shape, [5])
 
-    def test_is_floating_point(self):
-        self.assertTrue(paddle.is_floating_point(self.x))
+        # 3) x is ND
+        x = paddle.rand([3, 5])
+        x.stop_gradient = False
+        out = paddle.median(x, None)
+        out.backward()
+        self.assertEqual(out.shape, [])
+        self.assertEqual(x.grad.shape, [3, 5])
 
-    def test_is_integer(self):
-        x = paddle.randint(0, 10, [])
-        self.assertTrue(paddle.is_integer(x))
+        # 4) x is ND, keepdim=True
+        x = paddle.rand([3, 5])
+        x.stop_gradient = False
+        out = paddle.median(x, keepdim=True)
+        out.backward()
+        self.assertEqual(out.shape, [1, 1])
+        self.assertEqual(x.grad.shape, [3, 5])
 
-    def test_is_tensor(self):
-        self.assertTrue(paddle.is_tensor(self.x))
+    def test_kthvalue(self):
+        # 1) x is 0D
+        x = paddle.randn([])
+        x.stop_gradient = False
+        out, index = paddle.kthvalue(x, 1)
+        out.backward()
 
-    def test_is_empty(self):
-        x = paddle.rand([3, 0, 5])
-        self.assertTrue(paddle.is_empty(x))
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out, x)
+        self.assertEqual(index.shape, [])
+        self.assertEqual(index, 0)
 
-    def test_isfinite(self):
-        out = paddle.isfinite(self.x)
-        np.testing.assert_array_equal(out.numpy(), np.array(True))
+        self.assertEqual(x.grad.shape, [])
+        self.assertEqual(x.grad, 1.0)
 
-    def test_isinf(self):
-        x = paddle.to_tensor(np.array(float('-inf')))
-        out = paddle.isinf(x)
-        np.testing.assert_array_equal(out.numpy(), np.array(True))
+        # 2) x is 1D
+        x1 = paddle.randn([5])
+        x1.stop_gradient = False
+        out1, index1 = paddle.kthvalue(x1, 1)
+        out1.backward()
 
-    def test_isnan(self):
-        x = paddle.to_tensor(np.array(float('nan')))
-        out = paddle.isnan(x)
-        np.testing.assert_array_equal(out.numpy(), np.array(True))
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(index1.shape, [])
+        self.assertEqual(x1.grad.shape, [5])
 
-    def test_isclose(self):
-        out = paddle.isclose(self.x, self.x)
-        np.testing.assert_array_equal(out.numpy(), np.array(True))
+    def test_mode(self):
+        # 1) x is 0D
+        x = paddle.randn([])
+        x.stop_gradient = False
+        out, index = paddle.mode(x)
+        out.backward()
 
-    def test_clone(self):
-        out = paddle.clone(self.x)
-        np.testing.assert_array_equal(out.numpy(), self.x.numpy())
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out, x)
+        self.assertEqual(index.shape, [])
+        self.assertEqual(index, 0)
 
-    def test_assign(self):
-        out = paddle.assign(self.x)
-        np.testing.assert_array_equal(out.numpy(), self.x.numpy())
+        self.assertEqual(x.grad.shape, [])
+        self.assertEqual(x.grad, 1.0)
 
-    def test_item(self):
-        x = paddle.full([], 0.5)
-        self.assertEqual(x.item(), 0.5)
+        # 2) x is 1D
+        x1 = paddle.randn([5])
+        x1.stop_gradient = False
+        out1, index1 = paddle.mode(x1)
+        out1.backward()
 
-    def test_tolist(self):
-        x = paddle.full([], 0.5)
-        self.assertEqual(x.tolist(), 0.5)
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(index1.shape, [])
 
-    def test_numpy(self):
-        x = paddle.full([], 0.5)
-        np.testing.assert_array_equal(x.numpy(), np.array(0.5))
+        self.assertEqual(x1.grad.shape, [5])
 
-    def test_numel(self):
+    def test_is_empty(self):
         # 1) x is 0D
-        out = paddle.numel(self.x)
+        x = paddle.rand([])
+        out = paddle.is_empty(x)
+        self.assertFalse(out)
         self.assertEqual(out.shape, [])
-        np.testing.assert_array_equal(out.numpy(), np.array(1))
 
-        # 2) x is ND
-        x = paddle.full([3, 5], 0.5)
-        out = paddle.numel(x)
+        # 2) x is 1D
+        x = paddle.rand([5])
+        out = paddle.is_empty(x)
+        self.assertFalse(out)
         self.assertEqual(out.shape, [])
-        np.testing.assert_array_equal(out.numpy(), np.array(15))
 
-    def test_rank(self):
-        # 1) x is 0D
-        out = paddle.rank(self.x)
+        # 3) x is ND
+        x = paddle.rand([3, 5])
+        out = paddle.is_empty(x)
+        self.assertFalse(out)
         self.assertEqual(out.shape, [])
-        np.testing.assert_array_equal(out.numpy(), np.array(0))
 
-        # 1) x is ND
-        x = paddle.full([3, 5], 0.5)
-        out = paddle.rank(x)
+        x = paddle.rand([3, 0, 5])
+        out = paddle.is_empty(x)
+        self.assertTrue(out)
         self.assertEqual(out.shape, [])
-        np.testing.assert_array_equal(out.numpy(), np.array(2))
-
-    def test_shape(self):
-        out = paddle.shape(self.x)
-        self.assertEqual(out.shape, [0])
-        np.testing.assert_array_equal(out.numpy(), np.array([]))
 
-    def test_pow_factor(self):
+    def test_squeeze_(self):
+        # 1) x is 0D
         x = paddle.rand([])
+        x.squeeze_(0)
+        self.assertEqual(x.shape, [])
+
+        # 2) x is 1D
+        x = paddle.rand([1])
+        x.squeeze_(0)
+        self.assertEqual(x.shape, [])
+
+        # 3）x is ND
+        x = paddle.rand([2, 1])
+        x.squeeze_(1)
+        self.assertEqual(x.shape, [2])
+
+    def test_as_complex(self):
+        x = paddle.rand([2])
         x.stop_gradient = False
-        x.retain_grads()
-        out = paddle.pow(x, 2.0)
+        out = paddle.as_complex(x)
         out.retain_grads()
         out.backward()
 
+        self.assertEqual(x.shape, [2])
         self.assertEqual(out.shape, [])
+        self.assertEqual(x.grad.shape, [2])
         self.assertEqual(out.grad.shape, [])
-        self.assertEqual(x.grad.shape, [])
 
-    def test_cast(self):
-        x = paddle.full([], 1.0, 'float32')
+    def test_dot(self):
+        # 1) x is 1D
+        x = paddle.rand([2])
         x.stop_gradient = False
-        x.retain_grads()
-        out = paddle.cast(x, 'int32')
+        y = paddle.rand([2])
+        y.stop_gradient = False
+        out = paddle.dot(x, y)
         out.retain_grads()
         out.backward()
 
+        self.assertEqual(x.grad.shape, [2])
         self.assertEqual(out.shape, [])
         self.assertEqual(out.grad.shape, [])
-        self.assertEqual(x.grad.shape, [])
 
-    def test_clip(self):
-        x = paddle.uniform([], None, -10, 10)
+        # 2) x is 2D
+        x1 = paddle.rand([2, 2])
+        x1.stop_gradient = False
+        y1 = paddle.rand([2, 2])
+        y1.stop_gradient = False
+        out1 = paddle.dot(x1, y1)
+        out1.retain_grads()
+        out1.backward()
+
+        self.assertEqual(x1.grad.shape, [2, 2])
+        self.assertEqual(out1.shape, [2])
+        self.assertEqual(out1.grad.shape, [2])
+
+    def test_inner(self):
+        # 0) input is 0D
+        x = paddle.rand([])
         x.stop_gradient = False
-        x.retain_grads()
-        out = paddle.clip(x, -5, 5)
+        y = paddle.rand([])
+        y.stop_gradient = False
+        out = paddle.inner(x, y)
         out.retain_grads()
         out.backward()
 
+        self.assertEqual(x.grad.shape, [])
         self.assertEqual(out.shape, [])
         self.assertEqual(out.grad.shape, [])
-        self.assertEqual(x.grad.shape, [])
 
-    def test_increment(self):
-        x = paddle.rand([])
+        # 1) input is 1D
+        x = paddle.rand([2])
         x.stop_gradient = False
-        out = paddle.increment(x, 1.0)
+        y = paddle.rand([2])
+        y.stop_gradient = False
+        out = paddle.inner(x, y)
+        out.retain_grads()
         out.backward()
 
+        self.assertEqual(x.grad.shape, [2])
         self.assertEqual(out.shape, [])
         self.assertEqual(out.grad.shape, [])
-        self.assertEqual(x.grad.shape, [])
 
-    def test_bitwise_not(self):
-        x = paddle.randint(-1, 1, [])
-        out1 = ~x
-        out2 = paddle.bitwise_not(x)
+        # 2) input is 2D
+        x = paddle.rand([2, 3])
+        x.stop_gradient = False
+        y = paddle.rand([3, 3])
+        y.stop_gradient = False
+        out = paddle.inner(x, y)
+        out.retain_grads()
+        out.backward()
 
-        self.assertEqual(out1.shape, [])
-        self.assertEqual(out2.shape, [])
+        self.assertEqual(x.grad.shape, [2, 3])
+        self.assertEqual(out.shape, [2, 3])
+        self.assertEqual(out.grad.shape, [2, 3])
+
+    def test_tensordot(self):
+        # 1) input is 1D
+        x = paddle.arange(10, dtype='float64')
+        x.stop_gradient = False
+        y = paddle.arange(10, dtype='float64')
+        y.stop_gradient = False
+        out = paddle.tensordot(x, y, axes=1)
+        out.retain_grads()
+        out.backward()
+
+        self.assertEqual(x.grad.shape, [10])
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out.grad.shape, [])
+
+        # 2) input is 2D
+        x = paddle.arange(6, dtype='float64').reshape([2, 3])
+        y = paddle.arange(6, dtype='float64').reshape([2, 3])
+        x.stop_gradient = False
+        out = paddle.tensordot(x, y, axes=2)
+        out.retain_grads()
+        out.backward()
+
+        self.assertEqual(x.grad.shape, [2, 3])
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out.grad.shape, [])
+
+    def test_metric_accuracy(self):
+        x = paddle.full(shape=[2, 4], fill_value=0.25)
+        y = paddle.full(shape=[2, 1], fill_value=1, dtype="int64")
+        out = paddle.metric.accuracy(input=x, label=y, k=1)
+        self.assertEqual(out.shape, [])
+
+    def test_std(self):
+        # 1) x is 0D
+        x = paddle.rand([])
+        x.stop_gradient = False
+        out1 = paddle.std(x)
+        out2 = paddle.std(x, [])
+        out1.backward()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out2.shape, [])
+        self.assertEqual(out1, 0)
+        self.assertEqual(out2, 0)
+
+        self.assertEqual(x.grad.shape, [])
+
+        # 2) x is ND
+        x = paddle.rand([3, 5])
+        x.stop_gradient = False
+        out = paddle.std(x)
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        self.assertEqual(x.grad.shape, [3, 5])
+
+    def test_var(self):
+        # 1) x is 0D
+        x = paddle.rand([])
+        x.stop_gradient = False
+        out1 = paddle.var(x)
+        out2 = paddle.var(x, [])
+        out1.backward()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out2.shape, [])
+        self.assertEqual(out1, 0)
+        self.assertEqual(out2, 0)
+
+        self.assertEqual(x.grad.shape, [])
+        np.testing.assert_allclose(x.grad, 0)
+
+        # 2) x is ND
+        x = paddle.rand([3, 5])
+        x.stop_gradient = False
+        out = paddle.std(x)
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        self.assertEqual(x.grad.shape, [3, 5])
+
+    def test_quantile(self):
+        # 1) x is 0D
+        x = paddle.rand([])
+        x.stop_gradient = False
+        out = paddle.quantile(x, 0.5, axis=None)
+
+        out.retain_grads()
+        out.backward()
+
+        out_empty_list = paddle.quantile(x, 0.5, axis=[])
+        self.assertEqual(out_empty_list, out)
+
+        self.assertEqual(x.shape, [])
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out, x)
+
+        self.assertEqual(x.grad.shape, [])
+        self.assertEqual(x.grad, 1.0)
+        self.assertEqual(out.grad.shape, [])
+        self.assertEqual(out.grad, 1.0)
+
+        # 2) x is ND
+        x = paddle.rand([2, 3])
+        x.stop_gradient = False
+        out = paddle.quantile(x, 0.5, axis=None)
+
+        out.retain_grads()
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out.grad.shape, [])
+        self.assertEqual(out.grad, 1.0)
+        self.assertEqual(x.grad.shape, [2, 3])
+
+    def test_flip(self):
+        x = paddle.rand([])
+        x.stop_gradient = False
+        out = paddle.flip(x, axis=[])
+        out.retain_grads()
+        out.backward()
+        self.assertEqual(x.shape, [])
+        self.assertEqual(out.shape, [])
+        self.assertEqual(x.grad.shape, [])
+        self.assertEqual(out.grad.shape, [])
+
+    def test_linear(self):
+        x = paddle.randn([3, 2])
+        w = paddle.full(shape=[2, 4], fill_value=0.5)
+        b = paddle.zeros([])
+
+        np.testing.assert_array_equal(
+            F.linear(x, w, b).numpy(), F.linear(x, w).numpy()
+        )
+
+    def test_is_floating_point(self):
+        self.assertTrue(paddle.is_floating_point(self.x))
+
+    def test_is_integer(self):
+        x = paddle.randint(0, 10, [])
+        self.assertTrue(paddle.is_integer(x))
+
+    def test_is_tensor(self):
+        self.assertTrue(paddle.is_tensor(self.x))
+
+    def test_isfinite(self):
+        out = paddle.isfinite(self.x)
+        np.testing.assert_array_equal(out.numpy(), np.array(True))
+
+    def test_isinf(self):
+        x = paddle.to_tensor(np.array(float('-inf')))
+        out = paddle.isinf(x)
+        np.testing.assert_array_equal(out.numpy(), np.array(True))
+
+    def test_isnan(self):
+        x = paddle.to_tensor(np.array(float('nan')))
+        out = paddle.isnan(x)
+        np.testing.assert_array_equal(out.numpy(), np.array(True))
+
+    def test_isclose(self):
+        out = paddle.isclose(self.x, self.x)
+        np.testing.assert_array_equal(out.numpy(), np.array(True))
+
+    def test_clone(self):
+        out = paddle.clone(self.x)
+        np.testing.assert_array_equal(out.numpy(), self.x.numpy())
+
+    def test_assign(self):
+        out = paddle.assign(self.x)
+        np.testing.assert_array_equal(out.numpy(), self.x.numpy())
+
+    def test_item(self):
+        x = paddle.full([], 0.5)
+        self.assertEqual(x.item(), 0.5)
+
+    def test_tolist(self):
+        x = paddle.full([], 0.5)
+        self.assertEqual(x.tolist(), 0.5)
+
+    def test_numpy(self):
+        x = paddle.full([], 0.5)
+        x_np = x.numpy()
+        np.testing.assert_array_equal(x_np.shape, ())
+        np.testing.assert_array_equal(x_np, np.array(0.5))
+
+        x_np = x.numpy(False)
+        np.testing.assert_array_equal(x_np.shape, ())
+        np.testing.assert_array_equal(x_np, np.array(0.5))
+
+    def test_numel(self):
+        # 1) x is 0D
+        out = paddle.numel(self.x)
+        self.assertEqual(out.shape, [])
+        np.testing.assert_array_equal(out.numpy(), np.array(1))
+
+        # 2) x is ND
+        x = paddle.full([3, 5], 0.5)
+        out = paddle.numel(x)
+        self.assertEqual(out.shape, [])
+        np.testing.assert_array_equal(out.numpy(), np.array(15))
+
+    def test_rank(self):
+        # 1) x is 0D
+        x = paddle.rand([])
+        out = paddle.rank(x)
+        self.assertEqual(out.shape, [])
+        np.testing.assert_array_equal(out.numpy(), np.array(0))
+
+        # 1) x is ND
+        x = paddle.full([3, 5], 0.5)
+        out = paddle.rank(x)
+        self.assertEqual(out.shape, [])
+        np.testing.assert_array_equal(out.numpy(), np.array(2))
+
+    def test_shape(self):
+        out = paddle.shape(self.x)
+        np.testing.assert_array_equal(out.numpy(), np.array([]))
+        self.assertEqual(out.shape, [0])
+
+    def test_equal_scalar(self):
+        x = paddle.rand([])
+        out = paddle.equal(x, 2.0)
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out, False)
+
+        x1 = paddle.full([], 2.0)
+        out1 = paddle.equal(x1, 2.0)
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out1, True)
+
+    def test_pow_scalar(self):
+        x = paddle.rand([])
+        x.stop_gradient = False
+        out = paddle.pow(x, 2.0)
+        out.retain_grads()
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out.grad.shape, [])
+        self.assertEqual(x.grad.shape, [])
+
+    def test_cast(self):
+        x = paddle.full([], 1.0, 'float32')
+        x.stop_gradient = False
+        out = paddle.cast(x, 'int32')
+        out.retain_grads()
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out.grad.shape, [])
+        self.assertEqual(x.grad.shape, [])
+
+    def test_cumprod(self):
+        x = paddle.full([], 1.0, 'float32')
+        x.stop_gradient = False
+        out = paddle.cumprod(x, 0)
+        out.retain_grads()
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out.grad.shape, [])
+        self.assertEqual(x.grad.shape, [])
+
+        with self.assertRaises(ValueError):
+            tmp = paddle.cumprod(x, 2)
+
+    def test_clip(self):
+        x = paddle.uniform([], None, -10, 10)
+        x.stop_gradient = False
+        out = paddle.clip(x, -5, 5)
+        out.retain_grads()
+        out.backward()
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out.grad.shape, [])
+        self.assertEqual(x.grad.shape, [])
+
+        x1 = paddle.uniform([], None, -10, 10)
+        x1.stop_gradient = False
+        out1 = paddle.clip(x1, paddle.full([], 5.0), paddle.full([], 5.0))
+        out1.retain_grads()
+        out1.backward()
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out1.grad.shape, [])
+        self.assertEqual(x1.grad.shape, [])
+
+    def test_increment(self):
+        x = paddle.rand([])
+        x.stop_gradient = False
+        out = paddle.increment(x, 1.0)
+        out.retain_grads()
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out.grad.shape, [])
+        self.assertEqual(x.grad.shape, [])
+
+    def test_bitwise_not(self):
+        x = paddle.randint(-1, 1, [])
+        out1 = ~x
+        out2 = paddle.bitwise_not(x)
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out2.shape, [])
 
     def test_logical_not(self):
         x = paddle.randint(0, 1, [])
@@ -852,10 +1344,10 @@ def test_logical_not(self):
         self.assertEqual(out.shape, [])
 
     def test_searchsorted(self):
+        # have no backward
         x = paddle.to_tensor([1, 3, 5, 7, 9])
         y = paddle.rand([])
 
-        # only has forward kernel
         out = paddle.searchsorted(x, y)
 
         self.assertEqual(out.shape, [])
@@ -925,11 +1417,85 @@ def test_gather_xD_axis_1(self):
         )
         index = paddle.full([], 1, 'int64')
         out = paddle.gather(x, index, axis=1)
+        out.retain_grads()
+        out.backward()
 
         self.assertEqual(out.shape, [2])
         np.testing.assert_array_equal(out.numpy(), [2.0, 5.0])
+        self.assertEqual(x.grad.shape, [2, 3])
+        self.assertEqual(out.grad.shape, [2])
+
+    def test_gather_nd(self):
+        x1 = paddle.to_tensor([1.0, 3.0, 5.0, 7.0, 9.0], stop_gradient=False)
+        x2 = paddle.to_tensor(
+            [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], stop_gradient=False
+        )
+
+        index1 = paddle.full([1], 1, 'int64')
+        index2 = paddle.full([2], 1, 'int64')
+
+        out1 = paddle.gather_nd(x1, index1)
+        out2 = paddle.gather_nd(x2, index2)
+
+        out1.retain_grads()
+        out2.retain_grads()
+
+        out1.backward()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out2.shape, [])
+        np.testing.assert_array_equal(out1, np.array(3.0))
+        np.testing.assert_array_equal(out2, np.array(5.0))
+        self.assertEqual(x1.grad.shape, [5])
+        self.assertEqual(x2.grad.shape, [2, 3])
+        self.assertEqual(out1.grad.shape, [])
+        self.assertEqual(out2.grad.shape, [])
+
+    def test_einsum(self):
+        os.environ['FLAGS_new_einsum'] = "0"
+        x = paddle.rand([5])
+        # sum
+        out1 = paddle.einsum('i->', x)
+        expect1 = np.einsum('i->', x)
+        # dot
+        out2 = paddle.einsum('i,i->', x, x)
+        expect2 = np.einsum('i,i->', x, x)
+
+        out1.retain_grads()
+        out2.retain_grads()
+
+        out1.backward()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out2.shape, [])
+        np.testing.assert_allclose(out1, expect1, rtol=1e-03)
+        np.testing.assert_allclose(out2, expect2, rtol=1e-03)
+
+    def test_einsum_V2(self):
+        os.environ['FLAGS_new_einsum'] = "1"
+        x = paddle.rand([5])
+        # sum
+        out1 = paddle.einsum('i->', x)
+        expect1 = np.einsum('i->', x)
+        # dot
+        out2 = paddle.einsum('i,i->', x, x)
+        expect2 = np.einsum('i,i->', x, x)
+
+        out1.retain_grads()
+        out2.retain_grads()
+
+        out1.backward()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out2.shape, [])
+        np.testing.assert_allclose(out1, expect1, rtol=1e-03)
+        np.testing.assert_allclose(out2, expect2, rtol=1e-03)
 
     def test_scatter_1D(self):
+        # have no backward now
         x = paddle.to_tensor([1.0, 3.0, 5.0, 7.0, 9.0], stop_gradient=False)
         index = paddle.full([], 2, 'int64')
         updates = paddle.full([], 4.0)
@@ -939,6 +1505,7 @@ def test_scatter_1D(self):
         self.assertEqual(out.numpy()[2], 4)
 
     def test_scatter_XD(self):
+        # have no backward now
         x = paddle.to_tensor(
             [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], stop_gradient=False
         )
@@ -1000,24 +1567,39 @@ def test_scatter__XD(self):
         out = paddle.scatter_(x, index, updates)
         np.testing.assert_array_equal(out.numpy()[1], [1.0, 2.0, 3.0])
 
+    def test_scatter_nd(self):
+        index = paddle.to_tensor([3], dtype="int64")
+        updates = paddle.full([], 2, dtype='float32')
+
+        out = paddle.scatter_nd(index, updates, [5])
+        self.assertEqual(out.shape, [5])
+        self.assertEqual(out.numpy()[3], 2)
+
     def test_flatten(self):
-        x = paddle.full([], 1, 'float32')
+        x = paddle.rand([])
         x.stop_gradient = False
 
         start_axis = 0
         stop_axis = -1
 
         out = paddle.flatten(x, start_axis=start_axis, stop_axis=stop_axis)
+        out.retain_grads()
         out.backward()
 
         self.assertEqual(out.shape, [1])
+        self.assertEqual(out.grad.shape, [1])
         self.assertEqual(x.grad.shape, [])
 
+    def test_histogram(self):
+        x = paddle.rand([])
+        out = paddle.histogram(x, bins=5, min=1, max=5)
+        self.assertEqual(out.shape, [5])
+
     def test_scale(self):
         x = paddle.rand([])
         x.stop_gradient = False
-        x.retain_grads()
         out = paddle.scale(x, scale=2.0, bias=1.0)
+
         out.retain_grads()
         out.backward()
 
@@ -1025,6 +1607,11 @@ def test_scale(self):
         self.assertEqual(out.grad.shape, [])
         self.assertEqual(x.grad.shape, [])
 
+    def test_scale_(self):
+        x = paddle.rand([])
+        out = x.scale_(scale=2.0, bias=1.0)
+        self.assertEqual(out.shape, [])
+
     def test_floor_divide(self):
         # 1-d // 0-d
         x = paddle.to_tensor([1, -2, 3], dtype="int64")
@@ -1066,12 +1653,17 @@ def test_cumsum(self):
         out2.backward()
         out3.backward()
 
+        self.assertEqual(x1.grad.shape, [])
+        self.assertTrue(x1.grad.numpy() == 3)
         self.assertEqual(out1.shape, [1])
         self.assertEqual(out1.grad.shape, [1])
+        self.assertTrue(out1.grad.numpy() == 1)
         self.assertEqual(out2.shape, [])
         self.assertEqual(out2.grad.shape, [])
+        self.assertTrue(out2.grad.numpy() == 1)
         self.assertEqual(out3.shape, [])
         self.assertEqual(out3.grad.shape, [])
+        self.assertTrue(out3.grad.numpy() == 1)
 
     def test_add_n(self):
         x1 = paddle.rand([])
@@ -1090,6 +1682,12 @@ def test_add_n(self):
         out1.backward()
         out2.backward()
 
+        self.assertEqual(x1.grad.shape, [])
+        self.assertTrue(x1.grad.numpy() == 1)
+        self.assertEqual(x2.grad.shape, [])
+        self.assertTrue(x2.grad.numpy() == 1)
+        self.assertEqual(x3.grad.shape, [])
+        self.assertTrue(x3.grad.numpy() == 1)
         self.assertEqual(out1.shape, [])
         self.assertEqual(out1.grad.shape, [])
         self.assertEqual(out2.shape, [])
@@ -1098,8 +1696,8 @@ def test_add_n(self):
     def test_reshape_list(self):
         x = paddle.rand([])
         x.stop_gradient = False
-
         out = paddle.reshape(x, [])
+
         out.retain_grads()
         out.backward()
         self.assertEqual(x.grad.shape, [])
@@ -1130,8 +1728,8 @@ def test_reshape_list(self):
     def test_reshape_tensor(self):
         x = paddle.rand([1, 1])
         x.stop_gradient = False
-
         out = paddle.reshape(x, [])
+
         out.retain_grads()
         out.backward()
         self.assertEqual(x.grad.shape, [1, 1])
@@ -1193,15 +1791,23 @@ def test_reshape__tensor(self):
         out = paddle.reshape_(x, new_shape)
         self.assertEqual(out.shape, [1, 1])
 
+    def test_reverse(self):
+        x = paddle.rand([])
+        x.stop_gradient = False
+        out = paddle.reverse(x, axis=[])
+        out.retain_grads()
+        out.backward()
+        self.assertEqual(x.shape, [])
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out.grad.shape, [])
+
     def test_sort(self):
         x1 = paddle.rand([])
         x2 = paddle.rand([])
         x1.stop_gradient = False
         x2.stop_gradient = False
-
         x1.retain_grads()
         x2.retain_grads()
-
         out1 = paddle.sort(x1, axis=-1)
         out2 = paddle.sort(x2, axis=0)
 
@@ -1250,36 +1856,153 @@ def test_argsort(self):
         self.assertEqual(x1.grad.numpy(), 0)
         self.assertEqual(x2.grad.numpy(), 0)
 
-    def test_sigmoid_focal_loss(self):
-        logit = paddle.to_tensor(
-            [[0.97, 0.91, 0.03], [0.55, 0.43, 0.71]],
-            dtype='float32',
-            stop_gradient=False,
-        )
-        label = paddle.to_tensor(
-            [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]], dtype='float32'
-        )
-        fg_num_0 = paddle.full([], 2.0)
-        fg_num_1 = paddle.full([1], 2.0)
+    def test_lerp(self):
+        # 0D + 0D, weight is float scalar
+        x = paddle.rand([])
+        y = paddle.rand([])
+        x.stop_gradient = False
+        y.stop_gradient = False
+        out = paddle.lerp(x, y, 0.5)
+        out.backward()
 
-        out0 = F.sigmoid_focal_loss(logit, label, normalizer=fg_num_0)
-        out1 = F.sigmoid_focal_loss(logit, label, normalizer=fg_num_1)
+        self.assertEqual(out.shape, [])
+        self.assertEqual(x.grad.shape, [])
+        self.assertEqual(y.grad.shape, [])
 
-        np.testing.assert_array_equal(
-            out0.numpy(),
-            out1.numpy(),
-        )
-        self.assertEqual(out0.shape, [])
+        # 0D + 0D, weigh is 0D
+        x0 = paddle.rand([])
+        y0 = paddle.rand([])
+        w0 = paddle.rand([])
+        x0.stop_gradient = False
+        y0.stop_gradient = False
+        y0.retain_grads()
 
-        out0.retain_grads()
+        out0 = paddle.lerp(x0, y0, w0)
         out0.backward()
-        self.assertEqual(out0.grad.shape, [])
-        self.assertEqual(logit.grad.shape, [2, 3])
+
+        self.assertEqual(out0.shape, [])
+        self.assertEqual(x0.grad.shape, [])
+        self.assertEqual(y0.grad.shape, [])
+
+        # 0D + ND
+        x1 = paddle.rand([])
+        y1 = paddle.rand([64, 64])
+        w1 = paddle.rand([])
+        x1.stop_gradient = False
+        y1.stop_gradient = False
+        x1.retain_grads()
+        y1.retain_grads()
+
+        out1 = paddle.lerp(x1, y1, w1)
+        out1.backward()
+
+        self.assertEqual(out1.shape, [64, 64])
+        self.assertEqual(x1.grad.shape, [])
+        self.assertEqual(y1.grad.shape, [64, 64])
+
+        # ND + 0D
+        x2 = paddle.rand([64, 64])
+        y2 = paddle.rand([])
+        w2 = paddle.rand([])
+        x2.stop_gradient = False
+        y2.stop_gradient = False
+        x2.retain_grads()
+        y2.retain_grads()
+
+        out2 = paddle.lerp(x2, y2, w2)
+        out2.backward()
+
+        self.assertEqual(out2.shape, [64, 64])
+        self.assertEqual(x2.grad.shape, [64, 64])
+        self.assertEqual(y2.grad.shape, [])
+
+    def test_repeat_interleave(self):
+        x = paddle.randn(())
+        x.stop_gradient = False
+
+        out = paddle.repeat_interleave(x, 2, None)
+        out.backward()
+
+        # check shape of output
+        self.assertEqual(out.shape, [2])
+
+        # check grad shape
+        self.assertEqual(x.grad.shape, [])
+
+        repeats = paddle.to_tensor([3], dtype='int32')
+        out = paddle.repeat_interleave(x, repeats, None)
+
+        # check shape of output with 1D repeats
+        self.assertEqual(out.shape, [3])
+
+        # check grad shape with 1D repeats
+        self.assertEqual(x.grad.shape, [])
 
     def test_allclose(self):
+        # 1) x is 0D
+        x = paddle.full([], 0.5)
+        y = paddle.full([], 0.6)
+        out = paddle.allclose(x, y)
+        self.assertEqual(out.shape, [])
+        self.assertFalse(out)
+
+        # 2) x is ND
+        x = paddle.full([2, 3], 0.5)
+        y = paddle.full([2, 3], 0.6)
+        out = paddle.allclose(x, y)
+        self.assertEqual(out.shape, [])
+        self.assertFalse(out)
+
+    def test_equal_all(self):
+        # 1) x is 0D
         x = paddle.full([], 0.5)
         y = paddle.full([], 0.6)
-        self.assertFalse(paddle.allclose(x, y))
+        out = paddle.equal_all(x, y)
+        self.assertEqual(out.shape, [])
+        self.assertFalse(out)
+
+        # 2) x is ND
+        x = paddle.full([2, 3], 0.5)
+        y = paddle.full([2, 3], 0.6)
+        out = paddle.equal_all(x, y)
+        self.assertEqual(out.shape, [])
+        self.assertFalse(out)
+
+    def test_where(self):
+        x1 = paddle.full([], 1)
+        x2 = paddle.full([], 2)
+        x1.stop_gradient = False
+        x2.stop_gradient = False
+        x1.retain_grads()
+        x2.retain_grads()
+        out = paddle.where(x1 > x2, x1, x2)
+        out.retain_grads()
+        out.backward()
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out.numpy(), 2)
+        self.assertEqual(out.grad.shape, [])
+        self.assertEqual(x1.grad.shape, [])
+        self.assertEqual(x2.grad.shape, [])
+        self.assertEqual(x1.grad.numpy(), 0)
+        self.assertEqual(x2.grad.numpy(), 1)
+
+    def test_atan2(self):
+        x1 = paddle.full([], 0)
+        x2 = paddle.full([], 2)
+        x1.retain_grads()
+        x2.retain_grads()
+        x1.stop_gradient = False
+        x2.stop_gradient = False
+        out = paddle.atan2(x1, x2)
+        out.retain_grads()
+        out.backward()
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out.numpy(), 0)
+        self.assertEqual(out.grad.shape, [])
+        self.assertEqual(x1.grad.shape, [])
+        self.assertEqual(x2.grad.shape, [])
+        self.assertEqual(x1.grad.numpy(), 0.5)
+        self.assertEqual(x2.grad.numpy(), 0)
 
     def test_interpolate(self):
         from paddle.nn.functional import interpolate
@@ -1336,12 +2059,73 @@ def test_interpolate(self):
             origin_result.numpy(), out3.numpy(), rtol=1e-05
         )
 
-    def test_equalall(self):
-        x = paddle.full([], 0.5)
-        y = paddle.full([], 0.6)
-        out = paddle.equal_all(x, y)
-        self.assertEqual(out.shape, [])
-        self.assertFalse(out)
+    def test_upsample(self):
+        from paddle.nn.functional import upsample
+
+        input_x = paddle.rand([2, 3, 6, 6])
+        input_x.stop_gradient = False
+
+        output_size = [
+            paddle.full([], 12, dtype="int32"),
+            paddle.full([], 12, dtype="int32"),
+        ]
+        out1 = upsample(
+            x=input_x, size=output_size, mode="bilinear", align_corners=False
+        )
+        out1.backward()
+
+        self.assertEqual(out1.shape, [2, 3, 12, 12])
+        self.assertEqual(input_x.grad.shape, [2, 3, 6, 6])
+
+    def test_unstack(self):
+        x1 = paddle.full([1], 0)
+        x2 = paddle.full([2], 2)
+        x1.retain_grads()
+        x2.retain_grads()
+        x1.stop_gradient = False
+        x2.stop_gradient = False
+
+        [out1] = paddle.unstack(x1, 0)
+        out1.retain_grads()
+        out1.backward()
+        [out2_1, out2_2] = paddle.unstack(x2, 0)
+        out2 = paddle.add_n([out2_1, out2_2])
+        out2.retain_grads()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out1.numpy(), 0)
+
+        self.assertEqual(out2_1.shape, [])
+        self.assertEqual(out2_1.numpy(), 2)
+        self.assertEqual(out2_2.shape, [])
+        self.assertEqual(out2_2.numpy(), 2)
+        self.assertEqual(x2.grad.shape, [2])
+
+    def test_unbind(self):
+        x1 = paddle.full([1], 0)
+        x2 = paddle.full([2], 2)
+        x1.retain_grads()
+        x2.retain_grads()
+        x1.stop_gradient = False
+        x2.stop_gradient = False
+
+        [out1] = paddle.unbind(x1, 0)
+        out1.retain_grads()
+        out1.backward()
+        [out2_1, out2_2] = paddle.unbind(x2, 0)
+        out2 = paddle.add_n([out2_1, out2_2])
+        out2.retain_grads()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out1.numpy(), 0)
+
+        self.assertEqual(out2_1.shape, [])
+        self.assertEqual(out2_1.numpy(), 2)
+        self.assertEqual(out2_2.shape, [])
+        self.assertEqual(out2_2.numpy(), 2)
+        self.assertEqual(x2.grad.shape, [2])
 
     def test_maseked_select(self):
         x = paddle.rand([])
@@ -1357,18 +2141,53 @@ def test_maseked_select(self):
         self.assertEqual(x.grad.shape, [])
         self.assertEqual(x.grad.numpy(), 1)
 
+    def test_squeeze(self):
+        x1 = paddle.full([], 2)
+        x1.stop_gradient = False
+        x1.retain_grads()
+        out1 = paddle.squeeze(x1, axis=0)
+        out1.retain_grads()
+        out1.backward()
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(x1.grad.shape, [])
+
+        x2 = paddle.full([], 3)
+        x3 = paddle.full([1], 0, dtype='int32')
+        x2.stop_gradient = False
+        x2.retain_grads()
+        out2 = paddle.squeeze(x2, axis=x3)
+        out2.retain_grads()
+        out2.backward()
+        self.assertEqual(out2.shape, [])
+        self.assertEqual(x2.grad.shape, [])
+
     def test_unsqueeze(self):
         x1 = paddle.full([], 2)
         x1.stop_gradient = False
+        x1.retain_grads()
         out1 = paddle.unsqueeze(x1, axis=0)
+        out1.retain_grads()
         out1.backward()
         self.assertEqual(out1.shape, [1])
         self.assertEqual(x1.grad.shape, [])
 
         x2 = paddle.full([], 0, dtype='int32')
         out2 = paddle.unsqueeze(x1, axis=x2)
+        out2.retain_grads()
         out2.backward()
         self.assertEqual(out2.shape, [1])
+        self.assertEqual(x1.grad.shape, [])
+
+    def test_t(self):
+        x = paddle.full([], 2.0)
+        x.stop_gradient = False
+        x.retain_grads()
+        out = paddle.t(x)
+        out.retain_grads()
+        out.backward()
+        self.assertEqual(out.shape, [])
+        self.assertEqual(out.grad.shape, [])
+        self.assertEqual(x.grad.shape, [])
 
     def test_prelu(self):
         x1 = paddle.full([], 1.0, 'float32')
@@ -1397,11 +2216,298 @@ def test_prelu(self):
         self.assertEqual(x2.grad.shape, [])
         self.assertEqual(x2.grad.numpy(), 0.25)
 
+    def test_while_loop(self):
+        def cond(i, x):
+            return paddle.less_than(i, eleven)
+
+        def body(i, x):
+            x = x + i
+            i = i + 1
+            return [i, x]
+
+        i = paddle.full([], 1.0, dtype='float32')
+        i.stop_gradient = False
+        eleven = paddle.full([], 11, dtype='float32')
+        x = paddle.full([], 0.0, dtype='float32')
+        x.stop_gradient = False
+        out_i, out_x = paddle.static.nn.while_loop(cond, body, [i, x])
+        out_x.backward()
+
+        self.assertEqual(out_i.shape, [])
+        np.testing.assert_allclose(out_i, np.array(11))
+        self.assertEqual(out_x.shape, [])
+        np.testing.assert_allclose(out_x, np.array(55))
+        self.assertEqual(i.grad.shape, [])
+        np.testing.assert_allclose(i.grad, np.array(10))
+        self.assertEqual(x.grad.shape, [])
+        np.testing.assert_allclose(x.grad, np.array(1.0))
+
+    def test_to_tensor(self):
+        out1 = paddle.to_tensor(1)
+        out2 = paddle.to_tensor(2.5)
+
+        out1.retain_grads()
+        out1.backward()
+        out2.retain_grads()
+        out2.backward()
+
+        self.assertEqual(out1.shape, [])
+        self.assertEqual(out1, 1)
+        self.assertEqual(out2.shape, [])
+        self.assertEqual(out2, 2.5)
+
+    def test_linalg_slogdet(self):
+        # 2-D input
+        x = paddle.randn([3, 3])
+        x.stop_gradient = False
+        out = paddle.linalg.slogdet(x)
+        out.retain_grads()
+        out.backward()
+
+        self.assertTrue(out.shape, [2])
+        self.assertTrue(x.grad.shape, [3, 3])
+
+        # 3-D input
+        x1 = paddle.randn([3, 3, 3])
+        x1.stop_gradient = False
+        out1 = paddle.linalg.slogdet(x1)
+        out1.retain_grads()
+        out1.backward()
+
+        self.assertTrue(out1.shape, [2, 3])
+        self.assertTrue(x1.grad.shape, [3, 3, 3])
+
+    def test_multi_dot(self):
+        a = paddle.randn([4])
+        a.stop_gradient = False
+        b = paddle.randn([4, 5])
+        b.stop_gradient = False
+        c = paddle.randn([5])
+        c.stop_gradient = False
+
+        out = paddle.linalg.multi_dot([a, b, c])
+        out.retain_grads()
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        self.assertEqual(a.grad.shape, [4])
+        self.assertEqual(b.grad.shape, [4, 5])
+        self.assertEqual(c.grad.shape, [5])
+
+    def test_cov(self):
+        xt = paddle.randn((3, 4))
+        xt.stop_gradient = False
+        xt_1 = paddle.randn((12,))
+        xt_1.stop_gradient = False
+
+        xt_out = paddle.linalg.cov(xt)
+        xt_out.retain_grads()
+        xt_out.backward()
+        self.assertEqual(xt_out.shape, [3, 3])
+        self.assertEqual(xt.grad.shape, [3, 4])
+
+        xt_1_out = paddle.linalg.cov(xt_1)
+        xt_1.retain_grads()
+        xt_1_out.backward()
+        self.assertEqual(xt_1_out.shape, [])
+        self.assertEqual(xt_1.grad.shape, [12])
+
+    def test_det(self):
+        xt = paddle.randn([3, 3, 3])
+        xt.stop_gradient = False
+        xt_1 = paddle.randn([3, 3])
+        xt_1.stop_gradient = False
+
+        xt_out = paddle.linalg.det(xt)
+        xt.retain_grads()
+        xt_out.backward()
+        self.assertEqual(xt_out.shape, [3])
+        self.assertEqual(xt.grad.shape, [3, 3, 3])
+
+        xt_1_out = paddle.linalg.det(xt_1)
+        xt_1.retain_grads()
+        xt_1_out.backward()
+        self.assertEqual(xt_1_out.shape, [])
+        self.assertEqual(xt_1.grad.shape, [3, 3])
+
+    def test_dist(self):
+        x = paddle.to_tensor([[3, 3], [3, 3]], dtype="float32")
+        y = paddle.to_tensor([[3, 3], [3, 1]], dtype="float32")
+        x.stop_gradient = False
+        y.stop_gradient = False
+        out = paddle.dist(x, y, 0)
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        np.testing.assert_allclose(out, np.array(1))
+        self.assertEqual(x.grad.shape, [2, 2])
+        self.assertEqual(y.grad.shape, [2, 2])
+
+    def test_linalg_norm(self):
+        # 1D input, p = fro ,axis = None, using reduceInferMeta
+        x_1 = paddle.arange(24, dtype="float32") - 12
+        x_1.stop_gradient = False
+        out_1 = paddle.linalg.norm(x_1)
+        out_1.retain_grads()
+        out_1.backward()
+
+        self.assertEqual(out_1.shape, [])
+        self.assertTrue(x_1.grad.shape, [24])
+
+        # 1D input, p = 1 ,axis = None,
+        # using p_nrom, as_vector = True
+        x_2 = paddle.arange(24, dtype="float32") - 12
+        x_2.stop_gradient = False
+        out_2 = paddle.linalg.norm(x_2, p=1)
+        out_2.retain_grads()
+        out_2.backward()
+
+        self.assertEqual(out_2.shape, [])
+        self.assertEqual(x_2.grad.shape, [24])
+
+        # 1D input, p = 1 ,axis = 0,
+        # using p_nrom, as_vector = False
+        x_2_p = paddle.arange(24, dtype="float32") - 12
+        x_2_p.stop_gradient = False
+        out_2_p = paddle.linalg.norm(x_2_p, p=1, axis=0)
+        out_2_p.retain_grads()
+        out_2_p.backward()
+
+        self.assertEqual(out_2_p.shape, [])
+        self.assertEqual(x_2_p.grad.shape, [24])
+
+        # 1D input, p = fro ,axis = 0,
+        # using p_nrom, as_vector = False
+        x_2_fro = paddle.arange(24, dtype="float32") - 12
+        x_2_fro.stop_gradient = False
+        out_2_fro = paddle.linalg.norm(x_2_fro, p="fro", axis=0)
+        out_2_fro.retain_grads()
+        out_2_fro.backward()
+
+        self.assertEqual(out_2_fro.shape, [])
+        self.assertEqual(x_2_fro.grad.shape, [24])
+
+        # 2D input, p = 1, axis = [0, 1]
+        # using p_matrix_norm ,depends on paddle.sum
+        x_3 = paddle.arange(24, dtype="float32").reshape([4, 6])
+        x_3.stop_gradient = False
+        out_3 = paddle.linalg.norm(x_3, p=1, axis=[0, 1])
+        out_3.retain_grads()
+        out_3.backward()
+        self.assertEqual(out_3.shape, [])
+        self.assertEqual(x_3.grad.shape, [4, 6])
+
+        # 2D input, p = 1, axis = None
+        # using p_matrix_norm, depends on paddle.sum
+        x_4 = paddle.arange(24, dtype="float32").reshape([4, 6])
+        x_4.stop_gradient = False
+        out_4 = paddle.linalg.norm(x_4)
+        out_4.retain_grads()
+        out_4.backward()
+        self.assertEqual(out_4.shape, [])
+        self.assertEqual(x_4.grad.shape, [4, 6])
+
+        # 2D input, p = inf, axis = [0, 1]
+        # using p_matrix_norm, depends on paddle.sum
+        x_5 = paddle.arange(24, dtype="float32").reshape([4, 6])
+        x_5.stop_gradient = False
+        out_5 = paddle.linalg.norm(x_5, p=2, axis=[0, 1])
+        out_5.retain_grads()
+        out_5.backward()
+
+        self.assertEqual(out_5.shape, [])
+        self.assertEqual(x_5.grad.shape, [4, 6])
+
+        # 2D input, p = -inf, axis = [0, 1]
+        x_6 = paddle.arange(24, dtype="float32").reshape([4, 6])
+        x_6.stop_gradient = False
+        out_6 = paddle.linalg.norm(x_6, p=-float("inf"), axis=[0, 1])
+        out_6.retain_grads()
+        out_6.backward()
+
+        self.assertEqual(out_6.shape, [])
+        self.assertEqual(x_6.grad.shape, [4, 6])
+
+    def test_linalg_cond(self):
+        def assert_shape(out):
+            self.assertEqual(out.shape, [])
+
+        x1 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x1.stop_gradient = False
+        # p = 2 : use paddle.sum
+        out = paddle.linalg.cond(x1)
+        out.backward()
+        assert_shape(out)
+        self.assertEqual(x1.grad.shape, [3, 3])
+
+        # p = fro : use paddle.sum
+        x2 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x2.stop_gradient = False
+        out_fro = paddle.linalg.cond(x2, p='fro')
+        out_fro.backward()
+        assert_shape(out_fro)
+        self.assertEqual(x2.grad.shape, [3, 3])
+
+        # p = nuc : use paddle.sum
+        x3 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x3.stop_gradient = False
+        out_nuc = paddle.linalg.cond(x3, p='nuc')
+        out_nuc.backward()
+        assert_shape(out_nuc)
+        self.assertEqual(x3.grad.shape, [3, 3])
+
+        # p in (-1, 1) : use paddle.sum
+        x4 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x4.stop_gradient = False
+        out_1 = paddle.linalg.cond(x4, p=1)
+        out_1.backward()
+        assert_shape(out_1)
+        self.assertEqual(x4.grad.shape, [3, 3])
+
+        x5 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x5.stop_gradient = False
+        out_minus_1 = paddle.linalg.cond(x5, p=-1)
+        out_minus_1.backward()
+        assert_shape(out_minus_1)
+        self.assertEqual(x5.grad.shape, [3, 3])
+
+        # p in (-2, 2)  depends on paddle.sum
+        x6 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x6.stop_gradient = False
+        out_2 = paddle.linalg.cond(x6, p=2)
+        out_2.backward()
+        assert_shape(out_2)
+        self.assertEqual(x6.grad.shape, [3, 3])
+
+        # p in (-inf, inf):use paddle.sum
+        x8 = paddle.to_tensor([[1.0, 0, -1], [0, 1, 0], [1, 0, 1]])
+        x8.stop_gradient = False
+        out_inf = paddle.linalg.cond(x8, p=float("inf"))
+        out_inf.backward()
+        assert_shape(out_inf)
+        self.assertEqual(x8.grad.shape, [3, 3])
+
+        a = paddle.randn([2, 4, 4])
+        a.stop_gradient = False
+        a_cond_fro = paddle.linalg.cond(a, p='fro')
+        a_cond_fro.backward()
+        self.assertEqual(len(a_cond_fro.shape), 1)
+        self.assertEqual(a.grad.shape, [2, 4, 4])
+
+    def test_trace(self):
+        x = paddle.to_tensor([[3, 2], [1, 9]], dtype="float32")
+        x.stop_gradient = False
+        out = paddle.trace(x)
+        out.backward()
+
+        self.assertEqual(out.shape, [])
+        np.testing.assert_allclose(out, np.array(12))
+        self.assertEqual(x.grad.shape, [2, 2])
+
 
 # Use to test API whose zero-dim input tensors don't have grad and not need to test backward in OpTest.
 class TestNoBackwardAPI(unittest.TestCase):
     def setUp(self):
-        paddle.disable_static()
         self.shape = [
             paddle.full([], 2, 'int32'),
             paddle.full([], 3, 'int32'),
@@ -1545,12 +2651,70 @@ def test_one_hot_label(self):
         self.assertEqual(one_hot_label.shape, [4])
         self.assertEqual(one_hot_label.numpy()[2], 1)
 
-    def test_where(self):
-        x1 = paddle.full([], 1)
-        x2 = paddle.full([], 2)
-        out = paddle.where(x1 > x2, x1, x2)
+    def test_unique_consecutive(self):
+        x = paddle.rand([])
+        y, inverse, counts = paddle.unique_consecutive(
+            x,
+            return_inverse=True,
+            return_counts=True,
+        )
+
+        self.assertEqual(y, x)
+        self.assertEqual(inverse, 0)
+        self.assertEqual(counts, 1)
+        self.assertEqual(y.shape, [1])
+        self.assertEqual(inverse.shape, [1])
+        self.assertEqual(counts.shape, [1])
+
+    def test_unique(self):
+        x = paddle.rand([])
+        y, index, inverse, counts = paddle.unique(
+            x,
+            return_index=True,
+            return_inverse=True,
+            return_counts=True,
+        )
+
+        self.assertEqual(y, x)
+        self.assertEqual(index, 0)
+        self.assertEqual(inverse, 0)
+        self.assertEqual(counts, 1)
+        self.assertEqual(y.shape, [1])
+        self.assertEqual(index.shape, [1])
+        self.assertEqual(inverse.shape, [1])
+        self.assertEqual(counts.shape, [1])
+
+    def test_matrix_rank(self):
+        x = paddle.eye(10)
+        x.stop_gradient = False
+        out = paddle.linalg.matrix_rank(x)
+
         self.assertEqual(out.shape, [])
-        self.assertEqual(out.numpy(), 2)
+        np.testing.assert_equal(out, np.array(10))
+
+        c = paddle.ones(shape=[3, 4, 5])
+        c.stop_gradient = False
+        out_c = paddle.linalg.matrix_rank(c)
+        self.assertEqual(out_c.shape, [3])
+        np.testing.assert_equal(out_c, np.array([1, 1, 1]))
+
+        # 2D, tol->float : OUTPUT 0D
+        x_tol = paddle.eye(10)
+        x_tol.stop_gradient = False
+        out_tol = paddle.linalg.matrix_rank(x_tol, tol=0.1)
+        self.assertEqual(out_tol.shape, [])
+
+        # 3D, tol->float : OUTPUT 1D
+        c_tol = paddle.ones(shape=[3, 4, 5])
+        c_tol.stop_gradient = False
+        out_c_tol = paddle.linalg.matrix_rank(c_tol, tol=0.1)
+        self.assertEqual(out_c_tol.shape, [3])
+
+        tol_2 = paddle.randn([2])
+        # 2D, tol->Tensor[1,2] : OUTPUT 1D
+        d = paddle.eye(10)
+        out_d = paddle.linalg.matrix_rank(d, tol=tol_2)
+        self.assertEqual(out_d.shape, [2])
 
 
 if __name__ == "__main__":

From bafc346900904733bf23007339b90ddb3c837558 Mon Sep 17 00:00:00 2001
From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com>
Date: Tue, 9 May 2023 17:46:55 +0800
Subject: [PATCH 240/405] remove some [-Wunused-parameter]warning (#53617)

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop
---
 .../collective/process_group_custom.cc        |  2 +-
 .../distributed/ps/service/brpc_ps_server.h   |  6 ++++--
 .../ps/service/communicator/communicator.h    | 20 +++++++++----------
 .../ps/service/coordinator_client.h           |  4 ++--
 paddle/fluid/framework/ir/generate_pass.cc    |  2 +-
 paddle/fluid/framework/trainer.h              |  2 +-
 .../fused/mkldnn/fusion_rnn_mkldnn.h          | 10 +++++-----
 paddle/fluid/pybind/process_group_utils.h     |  4 ++--
 paddle/phi/backends/onednn/onednn_helper.h    |  3 ++-
 paddle/phi/backends/onednn/onednn_reuse.h     |  2 +-
 paddle/phi/kernels/autotune/cache_base.h      |  2 +-
 .../phi/kernels/cpu/viterbi_decode_kernel.cc  |  2 +-
 .../kernels/impl/logsumexp_grad_kernel_impl.h |  2 +-
 paddle/phi/kernels/impl/matmul_kernel_impl.h  |  2 +-
 .../kernels/impl/searchsorted_kernel_impl.h   |  8 ++++----
 paddle/phi/kernels/impl/split_kernel_impl.h   |  2 +-
 paddle/phi/kernels/sparse/cpu/full_kernel.cc  |  2 +-
 .../phi/kernels/sparse/sparse_utils_kernel.h  |  2 +-
 18 files changed, 40 insertions(+), 37 deletions(-)

diff --git a/paddle/fluid/distributed/collective/process_group_custom.cc b/paddle/fluid/distributed/collective/process_group_custom.cc
index f0222b007c53e..b6c7063fd6fb7 100644
--- a/paddle/fluid/distributed/collective/process_group_custom.cc
+++ b/paddle/fluid/distributed/collective/process_group_custom.cc
@@ -189,7 +189,7 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupCustom::Collective(
     std::vector<phi::DenseTensor>& outputs,
     Fn fn,
     CommType op_type,
-    bool sync_op,
+    bool sync_op UNUSED,
     bool use_calc_stream) {
   const auto places = GetPlaceList(inputs);
   const auto key = GetKeyFromPlaces(places);
diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_server.h b/paddle/fluid/distributed/ps/service/brpc_ps_server.h
index 0343b3f8c58a7..321adf156c4d8 100644
--- a/paddle/fluid/distributed/ps/service/brpc_ps_server.h
+++ b/paddle/fluid/distributed/ps/service/brpc_ps_server.h
@@ -227,8 +227,10 @@ class DownpourPServerBrpcClosure : public PServerClosure {
   PsRequestMessage *request(size_t i) { return &_requests[i]; }
   PsResponseMessage *response(size_t i) { return &_responses[i]; }
   brpc::Controller *cntl(size_t i) { return _cntls[i].get(); }
-  int check_response(size_t request_idx, int cmd_id) { return 1; }
-  int check_save_response(size_t request_idx, int cmd_id) { return 1; }
+  int check_response(size_t request_idx UNUSED, int cmd_id UNUSED) { return 1; }
+  int check_save_response(size_t request_idx UNUSED, int cmd_id UNUSED) {
+    return 1;
+  }
 
  private:
   std::atomic<int32_t> _waiting_num;
diff --git a/paddle/fluid/distributed/ps/service/communicator/communicator.h b/paddle/fluid/distributed/ps/service/communicator/communicator.h
index f3aa23a77826d..643c91b5b05e9 100644
--- a/paddle/fluid/distributed/ps/service/communicator/communicator.h
+++ b/paddle/fluid/distributed/ps/service/communicator/communicator.h
@@ -287,10 +287,10 @@ class Communicator {
     return {};
   }
   virtual void SaveFLStrategy(
-      const std::unordered_map<uint32_t, std::string> &fl_strategy) {}
+      const std::unordered_map<uint32_t, std::string> &fl_strategy UNUSED) {}
   virtual void StartCoordinator(
-      const std::string &self_endpoint,
-      const std::vector<std::string> &trainer_endpoints) {}
+      const std::string &self_endpoint UNUSED,
+      const std::vector<std::string> &trainer_endpoints UNUSED) {}
 
   virtual ~Communicator() {}
   virtual void RpcProfilerControl();
@@ -337,13 +337,13 @@ class Communicator {
 
   virtual void BarrierTriggerDecrement() {}
 
-  virtual void BarrierTriggerReset(int init_counter) {}
+  virtual void BarrierTriggerReset(int init_counter UNUSED) {}
 
   virtual void InitEnvs() = 0;
 
-  virtual void InitImpl(const RpcCtxMap &send_varname_to_ctx,
-                        const RecvCtxMap &recv_varname_to_ctx,
-                        Scope *recv_scope) {}
+  virtual void InitImpl(const RpcCtxMap &send_varname_to_ctx UNUSED,
+                        const RecvCtxMap &recv_varname_to_ctx UNUSED,
+                        Scope *recv_scope UNUSED) {}
 
   static Communicator *GetInstance() { return communicator_.get(); }
 
@@ -682,9 +682,9 @@ class FLCommunicator : public GeoCommunicator {
   virtual void InitBrpcClient(const std::string &dist_desc,
                               const std::vector<std::string> &host_sign_list);
 
-  void InitImpl(const RpcCtxMap &send_varname_to_ctx,
-                const RecvCtxMap &recv_varname_to_ctx,
-                Scope *recv_scope) {}
+  void InitImpl(const RpcCtxMap &send_varname_to_ctx UNUSED,
+                const RecvCtxMap &recv_varname_to_ctx UNUSED,
+                Scope *recv_scope UNUSED) {}
 
   void StartCoordinatorClient(
       const std::vector<std::string> &trainer_endpoints);
diff --git a/paddle/fluid/distributed/ps/service/coordinator_client.h b/paddle/fluid/distributed/ps/service/coordinator_client.h
index 883799fe50038..bd1f0f7754d8c 100644
--- a/paddle/fluid/distributed/ps/service/coordinator_client.h
+++ b/paddle/fluid/distributed/ps/service/coordinator_client.h
@@ -151,8 +151,8 @@ class CoordinatorService : public PsService {
                          ::google::protobuf::Closure* done);
 
   int32_t SaveFLClientInfo(const CoordinatorReqMessage& request,
-                           CoordinatorResMessage* response,
-                           brpc::Controller* cntl) {
+                           CoordinatorResMessage* response UNUSED,
+                           brpc::Controller* cntl UNUSED) {
     _coordinator_service_handle->SaveFLClientInfo(request);
     return 0;
   }
diff --git a/paddle/fluid/framework/ir/generate_pass.cc b/paddle/fluid/framework/ir/generate_pass.cc
index dd58b090764cb..61c6ce5757aa1 100644
--- a/paddle/fluid/framework/ir/generate_pass.cc
+++ b/paddle/fluid/framework/ir/generate_pass.cc
@@ -26,7 +26,7 @@ class element_visitor {
   explicit element_visitor(int index) : index_(index) {}
 
   template <typename T>
-  Attribute operator()(const T& attr) const {
+  Attribute operator()(const T& attr UNUSED) const {
     PADDLE_THROW(platform::errors::Unimplemented("Unimplemented operand."));
   }
 
diff --git a/paddle/fluid/framework/trainer.h b/paddle/fluid/framework/trainer.h
index 455487541abb9..bf69bed9d4851 100644
--- a/paddle/fluid/framework/trainer.h
+++ b/paddle/fluid/framework/trainer.h
@@ -70,7 +70,7 @@ class TrainerBase {
   virtual Scope* GetWorkerScope(int thread_id) = 0;
   virtual void InitDumpEnv() = 0;
   virtual void DumpWork(int tid);
-  virtual void ResetDataset(Dataset* dataset_ptr) {}
+  virtual void ResetDataset(Dataset* dataset_ptr UNUSED) {}
 
  protected:
   virtual std::string GetDumpPath(int tid) = 0;
diff --git a/paddle/fluid/operators/fused/mkldnn/fusion_rnn_mkldnn.h b/paddle/fluid/operators/fused/mkldnn/fusion_rnn_mkldnn.h
index f8854d3d7b489..db3967ceddfd2 100644
--- a/paddle/fluid/operators/fused/mkldnn/fusion_rnn_mkldnn.h
+++ b/paddle/fluid/operators/fused/mkldnn/fusion_rnn_mkldnn.h
@@ -30,12 +30,12 @@ class RNNMKLDNNHandler : public phi::funcs::OneDNNHandlerT<T, T_alg> {
  public:
   RNNMKLDNNHandler(const paddle::framework::ExecutionContext& ctx,
                    const phi::OneDNNContext& dev_ctx,
-                   const dnnl::engine onednn_engine,
+                   const dnnl::engine onednn_engine UNUSED,
                    platform::Place cpu_place,
-                   const phi::DenseTensor* input,
-                   const phi::DenseTensor* weight_h,
-                   const phi::DenseTensor* h0,
-                   const bool is_reverse,
+                   const phi::DenseTensor* input UNUSED,
+                   const phi::DenseTensor* weight_h UNUSED,
+                   const phi::DenseTensor* h0 UNUSED,
+                   const bool is_reverse UNUSED,
                    const int64_t N,
                    const int64_t Ti,
                    const int64_t IC,
diff --git a/paddle/fluid/pybind/process_group_utils.h b/paddle/fluid/pybind/process_group_utils.h
index a35962ce84133..1a6b640b3a3cf 100644
--- a/paddle/fluid/pybind/process_group_utils.h
+++ b/paddle/fluid/pybind/process_group_utils.h
@@ -56,7 +56,7 @@ struct ConcatDenseTensor<platform::CustomDeviceContext, T> {
   void operator()(const platform::CustomDeviceContext &context,
                   const std::vector<phi::DenseTensor> &in,
                   phi::DenseTensor *out,
-                  int axis = 0) {
+                  int axis UNUSED = 0) {
     auto *out_data = out->data<T>();
     auto *device = phi::DeviceManager::GetDeviceWithPlace(context.GetPlace());
     size_t offset = 0;
@@ -80,7 +80,7 @@ struct SplitDenseTensor<platform::CustomDeviceContext, T> {
   void operator()(const platform::CustomDeviceContext &context,
                   const phi::DenseTensor &in,
                   std::vector<phi::DenseTensor *> *out,
-                  int axis = 0) {
+                  int axis UNUSED = 0) {
     auto *in_data = in.data<T>();
     auto *device = phi::DeviceManager::GetDeviceWithPlace(context.GetPlace());
     size_t offset = 0;
diff --git a/paddle/phi/backends/onednn/onednn_helper.h b/paddle/phi/backends/onednn/onednn_helper.h
index 9b5aa167a6276..84e36a26ca487 100644
--- a/paddle/phi/backends/onednn/onednn_helper.h
+++ b/paddle/phi/backends/onednn/onednn_helper.h
@@ -191,7 +191,8 @@ inline void AppendKey(std::string* key, const std::vector<T>& dims) {
 }
 
 template <typename... ArgTypes>
-inline std::string CreateKey(const OneDNNContext& dev_ctx, ArgTypes&&... args) {
+inline std::string CreateKey(const OneDNNContext& dev_ctx UNUSED,
+                             ArgTypes&&... args) {
   std::string key;
   key.reserve(64);
   using expand_type = int[];
diff --git a/paddle/phi/backends/onednn/onednn_reuse.h b/paddle/phi/backends/onednn/onednn_reuse.h
index 463c55a7c5317..330f0318a6e09 100644
--- a/paddle/phi/backends/onednn/onednn_reuse.h
+++ b/paddle/phi/backends/onednn/onednn_reuse.h
@@ -676,7 +676,7 @@ class OneDNNHandlerNoCachingT {
       const dnnl::memory::desc& user_md,
       const dnnl::memory::desc& target_md,
       void* ptr,
-      bool is_persistent = false,
+      bool is_persistent UNUSED = false,
       std::function<std::shared_ptr<F>(const F*)> custom_reorder_func = {}) {
     std::shared_ptr<dnnl::memory> target_memory_p;
     if (custom_reorder_func) {
diff --git a/paddle/phi/kernels/autotune/cache_base.h b/paddle/phi/kernels/autotune/cache_base.h
index 267c8ef3f6859..798898f4dd7d1 100644
--- a/paddle/phi/kernels/autotune/cache_base.h
+++ b/paddle/phi/kernels/autotune/cache_base.h
@@ -24,7 +24,7 @@
 
 DECLARE_int32(search_cache_max_number);
 
-inline void HashCombine(std::size_t* seed) {}
+inline void HashCombine(std::size_t* seed UNUSED) {}
 
 // combine hash value
 // https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x
diff --git a/paddle/phi/kernels/cpu/viterbi_decode_kernel.cc b/paddle/phi/kernels/cpu/viterbi_decode_kernel.cc
index 6342bfee4dfe9..42fdd78c6205f 100644
--- a/paddle/phi/kernels/cpu/viterbi_decode_kernel.cc
+++ b/paddle/phi/kernels/cpu/viterbi_decode_kernel.cc
@@ -127,7 +127,7 @@ template <typename Context,
           class BinaryFunctor,
           typename T>
 struct BinaryOperation {
-  void operator()(const Context& dev_ctx,
+  void operator()(const Context& dev_ctx UNUSED,
                   const DenseTensor& lhs,
                   const DenseTensor& rhs,
                   DenseTensor* output) {
diff --git a/paddle/phi/kernels/impl/logsumexp_grad_kernel_impl.h b/paddle/phi/kernels/impl/logsumexp_grad_kernel_impl.h
index 0db6c12d4a07c..ab65769ee6931 100644
--- a/paddle/phi/kernels/impl/logsumexp_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/logsumexp_grad_kernel_impl.h
@@ -38,7 +38,7 @@ struct LogsumexpGradFunctor {
                   DX* dx,
                   DY* dy,
                   const Dim& dim,
-                  int size) {
+                  int size UNUSED) {
     using MT = typename phi::dtype::MPTypeTrait<T>::Type;
     auto x_mt = (*x).template cast<MT>();
     auto y_mt = (*y).template cast<MT>();
diff --git a/paddle/phi/kernels/impl/matmul_kernel_impl.h b/paddle/phi/kernels/impl/matmul_kernel_impl.h
index acc7affc00e26..14f50786d0c86 100644
--- a/paddle/phi/kernels/impl/matmul_kernel_impl.h
+++ b/paddle/phi/kernels/impl/matmul_kernel_impl.h
@@ -103,7 +103,7 @@ void MatMulFunctionImplWithBlas(
     bool trans_x,
     bool trans_y,
     bool flag = false,
-    phi::funcs::MatmulPlanner* matmul_planner = nullptr) {
+    phi::funcs::MatmulPlanner* matmul_planner UNUSED = nullptr) {
   const int x_ndim = x_dims.size();
   const int y_ndim = y_dims.size();
 
diff --git a/paddle/phi/kernels/impl/searchsorted_kernel_impl.h b/paddle/phi/kernels/impl/searchsorted_kernel_impl.h
index 6c0891e59bcb9..2b28e3fed49a2 100644
--- a/paddle/phi/kernels/impl/searchsorted_kernel_impl.h
+++ b/paddle/phi/kernels/impl/searchsorted_kernel_impl.h
@@ -39,8 +39,8 @@ class GpuAndCpuSearchSortedCompute {
     return std::isnan(x);
 #endif
   }
-  static HOSTDEVICE bool IsNan(int x) { return false; }
-  static HOSTDEVICE bool IsNan(int64_t x) { return false; }
+  static HOSTDEVICE bool IsNan(int x UNUSED) { return false; }
+  static HOSTDEVICE bool IsNan(int64_t x UNUSED) { return false; }
 
   static HOSTDEVICE bool IsInf(float x) {
 #ifdef __NVCC__
@@ -56,8 +56,8 @@ class GpuAndCpuSearchSortedCompute {
     return std::isinf(x);
 #endif
   }
-  static HOSTDEVICE bool IsInf(int x) { return false; }
-  static HOSTDEVICE bool IsInf(int64_t x) { return false; }
+  static HOSTDEVICE bool IsInf(int x UNUSED) { return false; }
+  static HOSTDEVICE bool IsInf(int64_t x UNUSED) { return false; }
 
   HOSTDEVICE GpuAndCpuSearchSortedCompute(const T1* sequence_data,
                                           const T2* value_data,
diff --git a/paddle/phi/kernels/impl/split_kernel_impl.h b/paddle/phi/kernels/impl/split_kernel_impl.h
index 83968d913feb4..2df379fcdb3b8 100644
--- a/paddle/phi/kernels/impl/split_kernel_impl.h
+++ b/paddle/phi/kernels/impl/split_kernel_impl.h
@@ -25,7 +25,7 @@ namespace phi {
 template <typename T, typename Context>
 void SplitKernel(const Context& dev_ctx,
                  const DenseTensor& x,
-                 const IntArray& sections,
+                 const IntArray& sections UNUSED,
                  const Scalar& axis_scalar,
                  std::vector<DenseTensor*> outs) {
   std::vector<const DenseTensor*> shape_refer;
diff --git a/paddle/phi/kernels/sparse/cpu/full_kernel.cc b/paddle/phi/kernels/sparse/cpu/full_kernel.cc
index 5659bcf2159da..d9209544ec7b9 100644
--- a/paddle/phi/kernels/sparse/cpu/full_kernel.cc
+++ b/paddle/phi/kernels/sparse/cpu/full_kernel.cc
@@ -54,7 +54,7 @@ template <typename T, typename Context>
 void FullLikeCsrKernel(const Context& dev_ctx,
                        const SparseCsrTensor& x,
                        const Scalar& val,
-                       DataType dtype,
+                       DataType dtype UNUSED,
                        SparseCsrTensor* out) {
   phi::Copy<Context>(dev_ctx,
                      x.non_zero_crows(),
diff --git a/paddle/phi/kernels/sparse/sparse_utils_kernel.h b/paddle/phi/kernels/sparse/sparse_utils_kernel.h
index 241f3d8b0670a..e06391d9404ef 100644
--- a/paddle/phi/kernels/sparse/sparse_utils_kernel.h
+++ b/paddle/phi/kernels/sparse/sparse_utils_kernel.h
@@ -158,7 +158,7 @@ void ValuesCsrKernel(const Context& dev_ctx UNUSED,
 }
 
 template <typename T, typename Context>
-void IndicesCooKernel(const Context& dev_ctx,
+void IndicesCooKernel(const Context& dev_ctx UNUSED,
                       const SparseCooTensor& x,
                       DenseTensor* out) {
   *out = x.indices();

From 9cd0a5bc74a9c241dbed7535ce73ee099d793a01 Mon Sep 17 00:00:00 2001
From: Ainavo <57820731+Ainavo@users.noreply.github.com>
Date: Tue, 9 May 2023 17:54:12 +0800
Subject: [PATCH 241/405] [Hackthon 4th No.7] add  paddle.unflatten API
 (#53055)

---
 python/paddle/__init__.py                     |   2 +
 .../fluid/tests/unittests/test_unflatten.py   | 311 ++++++++++++++++++
 python/paddle/nn/__init__.py                  |   2 +
 python/paddle/nn/layer/__init__.py            |   4 +-
 python/paddle/nn/layer/common.py              |  50 +++
 python/paddle/tensor/__init__.py              |   2 +
 python/paddle/tensor/manipulation.py          |  69 ++++
 7 files changed, 439 insertions(+), 1 deletion(-)
 create mode 100644 python/paddle/fluid/tests/unittests/test_unflatten.py

diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py
index e9a4d5e35be44..673a855c79540 100644
--- a/python/paddle/__init__.py
+++ b/python/paddle/__init__.py
@@ -198,6 +198,7 @@
 from .tensor.manipulation import repeat_interleave  # noqa: F401
 from .tensor.manipulation import index_add  # noqa: F401
 from .tensor.manipulation import index_add_  # noqa: F401
+from .tensor.manipulation import unflatten  # noqa: F401
 from .tensor.math import abs  # noqa: F401
 from .tensor.math import acos  # noqa: F401
 from .tensor.math import asin  # noqa: F401
@@ -691,5 +692,6 @@
     'cumulative_trapezoid',
     'polar',
     'vander',
+    'unflatten',
     'nextafter',
 ]
diff --git a/python/paddle/fluid/tests/unittests/test_unflatten.py b/python/paddle/fluid/tests/unittests/test_unflatten.py
new file mode 100644
index 0000000000000..6764ba3e2a404
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_unflatten.py
@@ -0,0 +1,311 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+import paddle
+
+
+def numpy_unflatten(x, axis, shape):
+    if isinstance(shape, (list, tuple)):
+        if len(shape) == 0:
+            raise ValueError("The input for shape cannot be empty.")
+        if isinstance(shape, list) or isinstance(shape, tuple):
+            if np.min(shape) < -1:
+                raise ValueError(f"invalid shape dimension {np.min(shape)}.")
+            if shape.count(-1) > 1:
+                raise ValueError("The shape can contain only one -1.")
+            elif shape.count(-1) == 1:
+                list(shape)[shape.index(-1)] = x.shape[axis] / abs(
+                    np.prod(shape)
+                )
+            else:
+                sizes = np.prod(shape)
+                if sizes != x.shape[axis]:
+                    raise ValueError(
+                        "The product of the elements in shape{} is not equal to {}.".format(
+                            shape, x.shape[axis]
+                        )
+                    )
+    else:
+        raise TypeError(
+            "The data type of x should be one of ['List', 'Tuple', 'Tensor'], but got {}".format(
+                type(shape)
+            )
+        )
+    length = len(x.shape)
+    if axis < 0:
+        axis = axis + length
+    new_shape = x.shape[:axis] + tuple(shape) + x.shape[axis + 1 :]
+    x = x.reshape(new_shape)
+    return x
+
+
+class TestUnflattenAPI(unittest.TestCase):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16)
+        self.axis = 0
+        self.shape = (2, 2)
+        self.shape_is_tensor = False
+
+    def get_output(self):
+        self.output = self.ref_api(self.x, self.axis, self.shape)
+
+    def set_api(self):
+        self.ref_api = numpy_unflatten
+        self.paddle_api = paddle.unflatten
+
+    def setUp(self):
+        self.set_api()
+        self.set_args()
+        self.get_output()
+        self.places = [paddle.CPUPlace()]
+        if paddle.device.is_compiled_with_cuda():
+            self.places.append(paddle.CUDAPlace(0))
+
+    def func_dygraph(self):
+        for place in self.places:
+            paddle.disable_static()
+            x = paddle.to_tensor(self.x, place=place)
+            if self.shape_is_tensor:
+                shape = paddle.to_tensor(self.shape)
+            else:
+                shape = self.shape
+            out = self.paddle_api(x=x, axis=self.axis, shape=shape)
+            np.testing.assert_allclose(out, self.output, rtol=1e-05)
+
+    def test_dygraph(self):
+        self.setUp()
+        self.func_dygraph()
+
+    def test_static(self):
+        paddle.enable_static()
+        places = [paddle.CPUPlace()]
+        if paddle.device.is_compiled_with_cuda():
+            places.append(paddle.CUDAPlace(0))
+        for place in places:
+            with paddle.static.program_guard(
+                paddle.static.Program(), paddle.static.Program()
+            ):
+                x = paddle.static.data(
+                    name="x", shape=self.x.shape, dtype=self.x.dtype
+                )
+                if self.shape_is_tensor:
+                    shape = np.array(self.shape)
+                    shape = paddle.static.data(
+                        name='shape', shape=shape.shape, dtype=shape.dtype
+                    )
+                else:
+                    shape = self.shape
+                exe = paddle.static.Executor(place)
+                out = self.paddle_api(x=x, axis=self.axis, shape=shape)
+                fetches = exe.run(
+                    paddle.static.default_main_program(),
+                    feed={
+                        "x": self.x,
+                        "axis": self.axis,
+                        "shape": self.shape,
+                    },
+                    fetch_list=[out],
+                )
+
+                np.testing.assert_allclose(fetches[0], self.output, rtol=1e-05)
+
+
+# check the data type of the input x
+class TestUnflattenInputInt16(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('int16')
+        self.axis = 0
+        self.shape = (2, 2)
+        self.shape_is_tensor = False
+
+
+class TestUnflattenInputInt32(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('int32')
+        self.axis = 0
+        self.shape = (2, 2)
+        self.shape_is_tensor = False
+
+
+class TestUnflattenInputInt64(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('int64')
+        self.axis = 0
+        self.shape = (2, 2)
+        self.shape_is_tensor = False
+
+
+class TestUnflattenInputFloat16(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('float16')
+        self.axis = 0
+        self.shape = (2, 2)
+        self.shape_is_tensor = False
+
+
+class TestUnflattenInputFloat32(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('float32')
+        self.axis = 0
+        self.shape = (2, 2)
+        self.shape_is_tensor = False
+
+
+class TestUnflattenInputFloat64(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('float64')
+        self.axis = 0
+        self.shape = (2, 2)
+        self.shape_is_tensor = False
+
+
+class TestUnflattenInputbool(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('bool')
+        self.axis = 0
+        self.shape = (2, 2)
+        self.shape_is_tensor = False
+
+
+# check the data type and edge cases of shape
+class TestUnflattenShapeList1(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('float32')
+        self.axis = 0
+        self.shape = [2, 2]
+        self.shape_is_tensor = False
+
+
+class TestUnflattenShapeList2(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('float32')
+        self.axis = -1
+        self.shape = [-1, 2]
+        self.shape_is_tensor = False
+
+
+class TestUnflattenShapeList3(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('float32')
+        self.axis = 0
+        self.shape = [-1]
+        self.shape_is_tensor = False
+
+
+class TestUnflattenTupleShape1(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('float32')
+        self.axis = 0
+        self.shape = (2, 2)
+        self.shape_is_tensor = False
+
+
+class TestUnflattenTupleShape2(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('float32')
+        self.axis = 0
+        self.shape = (-1, 2)
+        self.shape_is_tensor = False
+
+
+class TestUnflattenTupleShape3(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('float32')
+        self.axis = 0
+        self.shape = (-1,)
+        self.shape_is_tensor = False
+
+
+class TestUnflattenShapeTensorInt32(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('float32')
+        self.axis = 0
+        self.shape = tuple(np.array((-1, 4)).astype('int32'))
+        self.shape_is_tensor = True
+
+
+# check the value of axis
+class TestUnflattenAxis1(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('float32')
+        self.axis = 1
+        self.shape = (2, 3)
+        self.shape_is_tensor = False
+
+
+class TestUnflattenAxis2(TestUnflattenAPI):
+    def set_args(self):
+        self.x = np.random.rand(4, 6, 16).astype('float32')
+        self.axis = -1
+        self.shape = (2, 8)
+        self.shape_is_tensor = False
+
+
+class TestLayer(unittest.TestCase):
+    def set_args(self):
+        self.x = np.random.randn(3, 4, 4, 5).astype('float32')
+        self.axis = 1
+        self.shape = [2, 2]
+
+    def setUp(self):
+        self.set_args()
+        self.places = [paddle.CPUPlace()]
+        if paddle.device.is_compiled_with_cuda():
+            self.places.append(paddle.CUDAPlace(0))
+
+    def test_layer(self):
+        paddle.enable_static()
+        places = [paddle.CPUPlace()]
+        if paddle.device.is_compiled_with_cuda():
+            places.append(paddle.CUDAPlace(0))
+        for place in places:
+            with paddle.static.program_guard(
+                paddle.static.Program(), paddle.static.Program()
+            ):
+                x = paddle.static.data(
+                    name="x", dtype=self.x.dtype, shape=self.x.shape
+                )
+                exe = paddle.static.Executor(place)
+                unflatten = paddle.nn.Unflatten(self.axis, self.shape)
+                out = unflatten(x)
+                static_ret = exe.run(
+                    paddle.static.default_main_program(),
+                    feed={"x": self.x, "axis": self.axis, "shape": self.shape},
+                    fetch_list=[out],
+                )[0]
+        for place in self.places:
+            paddle.disable_static()
+            x = paddle.to_tensor(self.x, dtype='float32', place=place)
+            unflatten = paddle.nn.Unflatten(self.axis, self.shape)
+            dy_ret_value = unflatten(self.x)
+        np.testing.assert_array_equal(static_ret, dy_ret_value)
+
+
+class TestLayerName(unittest.TestCase):
+    def test_name(self):
+        self.x = np.random.randn(3, 4, 4, 5).astype('float32')
+        self.axis = 1
+        self.shape = [2, 2]
+        self.name = 'unflatten'
+        unflatten = paddle.nn.Unflatten(self.axis, self.shape, self.name)
+        _name = unflatten.extra_repr()
+
+
+if __name__ == '__main__':
+    paddle.enable_static()
+    unittest.main()
diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py
index 41d86c5298003..d682255a343ab 100644
--- a/python/paddle/nn/__init__.py
+++ b/python/paddle/nn/__init__.py
@@ -70,6 +70,7 @@
 from .layer.common import AlphaDropout  # noqa: F401
 from .layer.common import Unfold  # noqa: F401
 from .layer.common import Fold  # noqa: F401
+from .layer.common import Unflatten  # noqa: F401
 
 from .layer.pooling import AvgPool1D  # noqa: F401
 from .layer.pooling import AvgPool2D  # noqa: F401
@@ -338,4 +339,5 @@ def weight_norm(*args):
     'TripletMarginLoss',
     'SoftMarginLoss',
     'GaussianNLLLoss',
+    'Unflatten',
 ]
diff --git a/python/paddle/nn/layer/__init__.py b/python/paddle/nn/layer/__init__.py
index 7bf1ab6b62ce4..f83b8454456ff 100644
--- a/python/paddle/nn/layer/__init__.py
+++ b/python/paddle/nn/layer/__init__.py
@@ -45,7 +45,9 @@
 from .common import AlphaDropout  # noqa: F401
 from .common import UpsamplingBilinear2D  # noqa: F401
 from .common import UpsamplingNearest2D  # noqa: F401
-from .common import Fold
+from .common import Fold  # noqa: F401
+from .common import Unflatten  # noqa: F401
+
 from .pooling import AvgPool1D  # noqa: F401
 from .pooling import AvgPool2D  # noqa: F401
 from .pooling import AvgPool3D  # noqa: F401
diff --git a/python/paddle/nn/layer/common.py b/python/paddle/nn/layer/common.py
index 1e7d9d7c46da9..938c4b5a1ab48 100644
--- a/python/paddle/nn/layer/common.py
+++ b/python/paddle/nn/layer/common.py
@@ -1741,3 +1741,53 @@ def forward(self, input):
             input, start_axis=self.start_axis, stop_axis=self.stop_axis
         )
         return out
+
+
+class Unflatten(Layer):
+    """
+    This interface is used to construct a callable object of the ``Unflatten`` class.
+    For more details, refer to code examples.
+    It a certain dimension of the input x Tensor into a desired shape.
+
+    Parameters:
+        axis (int): :attr:`axis` to be unflattened, specified as an index into `x.shape`.
+        shape (list|tuple|Tensor): Unflatten :attr:`shape` on the specified :attr:`axis`. At most one dimension of the target :attr:`shape` can be -1.
+            If the input :attr:`shape` does not contain -1 , the product of all elements in ``shape`` should be equal to ``x.shape[axis]``.
+            The data type is `int` . If :attr:`shape` is a list or tuple, the elements of it should be integers or Tensors with shape [].
+            If :attr:`shape` is an Tensor, it should be an 1-D Tensor.
+        name(str, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
+
+    Returns:
+        None
+
+    Examples:
+
+        .. code-block:: python
+
+            import paddle
+
+            x = paddle.randn(shape=[4, 6, 8])
+            shape = [2, 3]
+            axis = 1
+            unflatten = paddle.nn.Unflatten(axis, shape)
+            res = unflatten(x)
+            print(res.shape)
+            # [4, 2, 3, 8]
+
+    """
+
+    def __init__(self, axis, shape, name=None):
+        super().__init__()
+        self.axis = axis
+        self.shape = shape
+        self.name = name
+
+    def forward(self, input):
+        out = paddle.unflatten(
+            input, axis=self.axis, shape=self.shape, name=self.name
+        )
+        return out
+
+    def extra_repr(self):
+        name_str = f', name={self.name}' if self.name else ''
+        return f'axis={self.axis}, shape={self.shape}{name_str}'
diff --git a/python/paddle/tensor/__init__.py b/python/paddle/tensor/__init__.py
index 4cd1614a1d60b..f8d75c9651d28 100644
--- a/python/paddle/tensor/__init__.py
+++ b/python/paddle/tensor/__init__.py
@@ -135,6 +135,7 @@
 from .manipulation import repeat_interleave  # noqa: F401
 from .manipulation import index_add  # noqa: F401
 from .manipulation import index_add_  # noqa: F401
+from .manipulation import unflatten  # noqa: F401
 from .math import abs  # noqa: F401
 from .math import acos  # noqa: F401
 from .math import asin  # noqa: F401
@@ -544,6 +545,7 @@
     'sigmoid_',
     'vander',
     'nextafter',
+    'unflatten',
 ]
 
 # this list used in math_op_patch.py for magic_method bind
diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py
index d0427886be2a6..cc48d8a83dcbe 100644
--- a/python/paddle/tensor/manipulation.py
+++ b/python/paddle/tensor/manipulation.py
@@ -4795,6 +4795,75 @@ def index_add_(x, index, axis, value, name=None):
     return _C_ops.index_add_(x, index, value, axis)
 
 
+def unflatten(x, axis, shape, name=None):
+    """
+    Expand a certain dimension of the input x Tensor into a desired shape.
+
+    Args:
+        x (Tensor) : An N-D Tensor. The data type is float16, float32, float64, int16, int32, int64, bool, uint16.
+        axis (int): :attr:`axis` to be unflattened, specified as an index into `x.shape`.
+        shape (list|tuple|Tensor): Unflatten :attr:`shape` on the specified :attr:`axis`. At most one dimension of the target :attr:`shape` can be -1.
+            If the input :attr:`shape` does not contain -1 , the product of all elements in ``shape`` should be equal to ``x.shape[axis]``.
+            The data type is `int` . If :attr:`shape` is a list or tuple, the elements of it should be integers or Tensors with shape [].
+            If :attr:`shape` is an Tensor, it should be an 1-D Tensor.
+        name(str, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
+
+    Returns:
+        Tensor, return the unflatten tensor of :attr:`x`.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+
+            x = paddle.randn(shape=[4, 6, 8])
+            shape = [2, 3]
+            axis = 1
+            res = paddle.unflatten(x, axis, shape)
+            print(res.shape)
+            # [4, 2, 3, 8]
+
+            x = paddle.randn(shape=[4, 6, 8])
+            shape = (-1, 2)
+            axis = -1
+            res = paddle.unflatten(x, axis, shape)
+            print(res.shape)
+            # [4, 6, 4, 2]
+
+            x = paddle.randn(shape=[4, 6, 8])
+            shape = paddle.to_tensor([2, 2])
+            axis = 0
+            res = paddle.unflatten(x, axis, shape)
+            print(res.shape)
+            # [2, 2, 6, 8]
+    """
+
+    # determine whether the input axis is valid.
+    axis = non_negative_axis(x, axis)
+
+    if isinstance(shape, (list, tuple)):
+        new_shape = (
+            list(x.shape[:axis]) + list(shape) + list(x.shape[axis + 1 :])
+        )
+    elif isinstance(shape, Variable):
+        # The data type returned by `paddle.shape` is only 'int32'.
+        new_shape = paddle.concat(
+            [
+                paddle.shape(x)[:axis],
+                paddle.cast(shape, 'int32'),
+                paddle.shape(x)[axis + 1 :],
+            ]
+        )
+    else:
+        raise TypeError(
+            "The data type of x should be one of ['List', 'Tuple', 'Tensor'], but got {}".format(
+                type(shape)
+            )
+        )
+    x = x.reshape(new_shape)
+    return x
+
+
 # TODO(dev): We need avoid implementing it by this way.
 __METHODS = {
     'fill_': fill_,

From 9244ceb6899c79be6ffd3f114c08c5a92f6e9864 Mon Sep 17 00:00:00 2001
From: Wilber <jiweibo@baidu.com>
Date: Tue, 9 May 2023 18:42:14 +0800
Subject: [PATCH 242/405] update openblas version (#52983)

---
 cmake/external/openblas.cmake | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake
index 01b9371da1509..180214970be99 100644
--- a/cmake/external/openblas.cmake
+++ b/cmake/external/openblas.cmake
@@ -18,6 +18,15 @@ set(CBLAS_PREFIX_DIR ${THIRD_PARTY_PATH}/openblas)
 set(CBLAS_INSTALL_DIR ${THIRD_PARTY_PATH}/install/openblas)
 set(CBLAS_REPOSITORY ${GIT_URL}/xianyi/OpenBLAS.git)
 set(CBLAS_TAG v0.3.7)
+
+# Please refer to
+# https://github.com/PaddlePaddle/Paddle/pull/52983
+if(UNIX
+   AND NOT APPLE
+   AND NOT WITH_ROCM)
+  set(CBLAS_TAG v0.3.18)
+endif()
+
 if(APPLE AND WITH_ARM)
   set(CBLAS_TAG v0.3.13)
 endif()

From eaed16810343000f05131c4b4446237bb8463f28 Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Tue, 9 May 2023 19:26:40 +0800
Subject: [PATCH 243/405] [static op generation] coalesce_tensor (#53570)

* [phi][api] add autogen code coalesce_tensor

* [phi][api]fix args

* [phi][api] supplement attrs
---
 paddle/fluid/operators/coalesce_tensor_op.cc | 519 -------------------
 paddle/phi/api/yaml/legacy_ops.yaml          |   9 -
 paddle/phi/api/yaml/op_compat.yaml           |   8 +
 paddle/phi/api/yaml/op_version.yaml          |  18 +
 paddle/phi/api/yaml/ops.yaml                 |   9 +
 paddle/phi/kernels/coalesce_tensor_kernel.cc |   1 +
 paddle/phi/ops/compat/coalesce_tensor_sig.cc |  38 --
 7 files changed, 36 insertions(+), 566 deletions(-)
 delete mode 100644 paddle/fluid/operators/coalesce_tensor_op.cc
 delete mode 100644 paddle/phi/ops/compat/coalesce_tensor_sig.cc

diff --git a/paddle/fluid/operators/coalesce_tensor_op.cc b/paddle/fluid/operators/coalesce_tensor_op.cc
deleted file mode 100644
index 2739cdd76ed02..0000000000000
--- a/paddle/fluid/operators/coalesce_tensor_op.cc
+++ /dev/null
@@ -1,519 +0,0 @@
-// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <sstream>
-#include <vector>
-
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/framework/op_version_registry.h"
-#include "paddle/fluid/framework/operator.h"
-#include "paddle/fluid/framework/var_type.h"
-#include "paddle/phi/backends/device_memory_aligment.h"
-#include "paddle/phi/kernels/funcs/math_function.h"
-
-#include "paddle/fluid/framework/convert_utils.h"
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/phi/infermeta/multiary.h"
-
-namespace paddle {
-namespace operators {
-
-template <typename DeviceContext>
-struct FillConstantVisitor {
-  FillConstantVisitor(const DeviceContext &dev_ctx,
-                      phi::DenseTensor *tensor,
-                      const float value,
-                      framework::proto::VarType::Type dtype,
-                      const framework::ExecutionContext &context)
-      : dev_ctx_(dev_ctx),
-        tensor_(tensor),
-        value_(value),
-        dtype_(dtype),
-        context_(context) {}
-
-  template <typename T>
-  void apply(typename std::enable_if<std::is_same<T, int8_t>::value ||
-                                     std::is_same<T, int16_t>::value>::type * =
-                 nullptr) const {
-    PADDLE_THROW(platform::errors::InvalidArgument(
-        "Not support data type for set_constant attr"));
-  }
-
-  template <typename T>
-  void apply(typename std::enable_if<!(std::is_same<T, int8_t>::value ||
-                                       std::is_same<T, int16_t>::value)>::type
-                 * = nullptr) const {
-    phi::funcs::SetConstant<DeviceContext, T> set_constant;
-    set_constant(dev_ctx_, tensor_, static_cast<T>(value_));
-  }
-
-  const DeviceContext &dev_ctx_;
-  phi::DenseTensor *tensor_;
-  float value_;
-  framework::proto::VarType::Type dtype_;
-  const framework::ExecutionContext &context_;
-};
-
-template <typename DeviceContext, typename T>
-class CoalesceTensorOpKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext &context) const override {
-    auto in_var_names = context.InputNames("Input");
-    auto out_var_names = context.OutputNames("Output");
-    const auto &in_tensors = context.MultiInput<phi::DenseTensor>("Input");
-    auto out_tensors = context.MultiOutput<phi::DenseTensor>("Output");
-
-    PADDLE_ENFORCE_GT(in_var_names.size(),
-                      static_cast<size_t>(0),
-                      platform::errors::InvalidArgument(
-                          "The CoalesceTensor operator has no input."));
-    PADDLE_ENFORCE_EQ(in_var_names.size(),
-                      out_var_names.size(),
-                      platform::errors::InvalidArgument(
-                          "The number of CoalesceTensor operator's input and "
-                          "output is not match, "
-                          "input number is %u, output number is %u.",
-                          in_var_names.size(),
-                          out_var_names.size()));
-
-    // Input & Output check: only support phi::DenseTensor
-    bool has_not_init_in_vars = false;
-    for (size_t i = 0; i < in_tensors.size(); ++i) {
-      PADDLE_ENFORCE_NOT_NULL(
-          in_tensors[i],
-          platform::errors::InvalidArgument(
-              "The %d-th input tensor cannot be nullptr.", i));
-      PADDLE_ENFORCE_NOT_NULL(
-          out_tensors[i],
-          platform::errors::InvalidArgument(
-              "The %d-th output tensor cannot be nullptr.", i));
-      if (!in_tensors[i]->IsInitialized()) {
-        has_not_init_in_vars = true;
-      }
-    }
-
-    if (has_not_init_in_vars) {
-      const auto &concated_shapes =
-          context.Attr<std::vector<int64_t>>("concated_shapes");
-      const auto &concated_ranks =
-          context.Attr<std::vector<int64_t>>("concated_ranks");
-      PADDLE_ENFORCE_EQ(concated_ranks.size(),
-                        out_tensors.size(),
-                        platform::errors::InvalidArgument(
-                            "The attribute(concated_ranks) length must be "
-                            "equal to the output tensor number."));
-      int64_t accumulated_ranks = 0;
-      for (size_t i = 0; i < in_tensors.size(); ++i) {
-        framework::DDim dims(concated_shapes.data() + accumulated_ranks,
-                             concated_ranks[i]);
-        if (!in_tensors[i]->IsInitialized()) {
-          PADDLE_ENFORCE_EQ(
-              in_tensors[i],
-              out_tensors[i],
-              platform::errors::InvalidArgument(
-                  "The %d-th output tensor and %d-th input tensor when the "
-                  "%d-th input tensor is not initialized.",
-                  i,
-                  i,
-                  i));
-          out_tensors[i]->Resize(dims);
-        } else {
-          PADDLE_ENFORCE_EQ(
-              in_tensors[i]->dims(),
-              dims,
-              platform::errors::InvalidArgument(
-                  "The %d-th input tensor shape does not match the "
-                  "attribute(concated_shapes) and "
-                  "attribute(concated_ranks).",
-                  i));
-        }
-        accumulated_ranks += concated_ranks[i];
-        PADDLE_ENFORCE_LE(accumulated_ranks,
-                          concated_shapes.size(),
-                          platform::errors::InvalidArgument(
-                              "The attribute(concated_shapes) and "
-                              "attribute(concated_ranks) do not match."));
-      }
-      PADDLE_ENFORCE_EQ(accumulated_ranks,
-                        concated_shapes.size(),
-                        platform::errors::InvalidArgument(
-                            "The attribute(concated_shapes) and "
-                            "attribute(concated_ranks) do not match."));
-    }
-
-    bool use_align = context.Attr<bool>("use_align");
-    auto align_size = context.Attr<int>("align_size");
-    auto size_of_dtype = context.Attr<int>("user_defined_size_of_dtype");
-
-    if (context.Attr<bool>("check_name")) {
-      for (size_t i = 0; i < in_var_names.size(); ++i) {
-        PADDLE_ENFORCE_EQ(
-            in_var_names[i],
-            out_var_names[i],
-            platform::errors::InvalidArgument(
-                "The input and output variable of CoalesceTensor operator is "
-                "different, %dth input is %s, %dth output is %s.",
-                i,
-                in_var_names[i],
-                i,
-                out_var_names[i]));
-      }
-    } else {
-      // Init the output as input
-      for (size_t i = 0; i < in_tensors.size(); ++i) {
-        out_tensors[i]->Resize(in_tensors[i]->dims());
-      }
-    }
-
-    auto &dev_ctx = context.template device_context<DeviceContext>();
-
-    // Get numel and dtype
-    size_t numel = 0;
-    auto dtype = static_cast<framework::proto::VarType::Type>(
-        context.Attr<int>("dtype"));
-    if (size_of_dtype == -1) {
-      size_of_dtype = framework::SizeOfType(dtype);
-    }
-    GetMemSizeAndDtype(in_tensors,
-                       in_var_names,
-                       &numel,
-                       size_of_dtype,
-                       context.GetPlace(),
-                       use_align,
-                       align_size);
-
-    // Alloc the continuous space
-    auto fused_tensor = context.Output<phi::DenseTensor>("FusedOutput");
-    void *fused_tensor_ptr =
-        fused_tensor->Resize(phi::make_ddim({static_cast<int64_t>(numel)}))
-            .mutable_data(context.GetPlace(),
-                          framework::TransToPhiDataType(dtype));
-    VLOG(10) << "Fused tensor addr " << fused_tensor_ptr;
-
-    // Init the continuous space
-    size_t offset = 0;
-    if (context.Attr<bool>("copy_data")) {
-      for (size_t i = 0; i < in_var_names.size(); ++i) {
-        size_t len = static_cast<size_t>(in_tensors[i]->numel());
-        auto sub_tensor = fused_tensor->Slice(
-            static_cast<int64_t>(offset), static_cast<int64_t>(offset + len));
-        framework::TensorCopy(
-            *in_tensors[i], context.GetPlace(), dev_ctx, &sub_tensor);
-
-        offset += use_align ? phi::Alignment(len * size_of_dtype,
-                                             context.GetPlace(),
-                                             align_size) /
-                                  size_of_dtype
-                            : len;
-      }
-    } else if (context.Attr<bool>("set_constant")) {
-      framework::VisitDataType(
-          dtype,
-          FillConstantVisitor<DeviceContext>(dev_ctx,
-                                             fused_tensor,
-                                             context.Attr<float>("constant"),
-                                             dtype,
-                                             context));
-    } else if (context.Attr<bool>("persist_output")) {
-      for (size_t i = 0; i < out_var_names.size(); ++i) {
-        size_t len = static_cast<size_t>(out_tensors[i]->numel());
-        auto sub_tensor = fused_tensor->Slice(
-            static_cast<int64_t>(offset), static_cast<int64_t>(offset + len));
-        // some var may not persistable, or persistable var may not init
-        if (out_tensors[i]->IsInitialized()) {
-          framework::TensorCopy(
-              *out_tensors[i], context.GetPlace(), dev_ctx, &sub_tensor);
-        }
-        offset += use_align ? phi::Alignment(len * size_of_dtype,
-                                             context.GetPlace(),
-                                             align_size) /
-                                  size_of_dtype
-                            : len;
-      }
-    }
-
-    // Make the outputs point to the continuous space.
-    offset = 0;
-    std::stringstream ss;
-    ss << "alloc_space_for_vars: ";
-
-    for (size_t i = 0; i < out_tensors.size(); ++i) {
-      size_t len = static_cast<size_t>(out_tensors[i]->numel());
-      auto dim = out_tensors[i]->dims();
-      VLOG(4) << len << " " << dim << " " << offset;
-      out_tensors[i]
-          ->ShareDataWith(fused_tensor->Slice(
-              static_cast<int64_t>(offset), static_cast<int64_t>(offset + len)))
-          .Resize(dim);
-      len = use_align
-                ? phi::Alignment(
-                      len * size_of_dtype, context.GetPlace(), align_size) /
-                      size_of_dtype
-                : len;
-      ss << "output(" << out_var_names[i] << ")  dim:(" << dim << ")"
-         << " address: " << out_tensors[i]->data() << " len: " << len << ", ";
-      offset += len;
-    }
-    PADDLE_ENFORCE_EQ(
-        (int64_t)offset,
-        fused_tensor->numel(),
-        platform::errors::InvalidArgument(
-            "The alloc_space_for_vars's offset: %s is unequal with "
-            "fused_tensor's numel: %s.",
-            offset,
-            fused_tensor->numel()));
-    VLOG(10) << ss.str();
-  }
-
- private:
-  void GetMemSizeAndDtype(
-      const std::vector<const phi::DenseTensor *> &lod_tensors,
-      const std::vector<std::string> var_names,
-      size_t *numel,
-      const size_t &size_of_dtype,
-      const platform::Place &place,
-      const bool use_align = true,
-      const int align_size = -1) const {
-    PADDLE_ENFORCE_EQ(
-        lod_tensors.size(),
-        var_names.size(),
-        platform::errors::InvalidArgument(
-            "The number of input tensor and variable does not match, the "
-            "number of input tensor is %u, the number of input variable is %u.",
-            lod_tensors.size(),
-            var_names.size()));
-    *numel = 0;
-    std::stringstream ss;
-    ss << "alloc_space_for_vars: ";
-    for (size_t i = 0; i < var_names.size(); ++i) {
-      auto size = lod_tensors[i]->numel();
-      PADDLE_ENFORCE_GT(
-          size,
-          0,
-          platform::errors::InvalidArgument(
-              "The number of tensor `%s`'s elements is 0.", var_names[i]));
-      auto len = use_align
-                     ? phi::Alignment(static_cast<size_t>(size) * size_of_dtype,
-                                      place,
-                                      align_size) /
-                           size_of_dtype
-                     : static_cast<size_t>(size);
-      const void *ptr =
-          lod_tensors[i]->IsInitialized() ? lod_tensors[i]->data() : nullptr;
-      VLOG(4) << size << " " << len;
-      ss << "input(" << var_names[i] << ") dim:(" << lod_tensors[i]->dims()
-         << ") "
-         << " addres:" << ptr << " len: " << len << ", ";
-      *numel += len;
-    }
-    VLOG(10) << ss.str();
-  }
-};
-
-class CoalesceTensorOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
-  void InferShape(framework::InferShapeContext *ctx) const override {
-    auto use_align = ctx->Attrs().Get<bool>("use_align");
-    auto align_size = ctx->Attrs().Get<int>("align_size");
-    auto size_of_dtype = ctx->Attrs().Get<int>("user_defined_size_of_dtype");
-
-    auto dtype = static_cast<framework::proto::VarType::Type>(
-        ctx->Attrs().Get<int>("dtype"));
-    if (size_of_dtype == -1) {
-      size_of_dtype = framework::SizeOfType(dtype);
-    }
-    if (ctx->IsRuntime()) {
-#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-      int64_t numel = 0;
-      auto dims = ctx->GetInputsDim("Input");
-      for (const auto &dim : dims) {
-        auto size = phi::product(dim);
-        auto len = use_align ? phi::Alignment(
-                                   static_cast<size_t>(size) * size_of_dtype,
-                                   phi::GPUPlace(),
-                                   align_size) /
-                                   size_of_dtype
-                             : static_cast<size_t>(size);
-        numel += len;
-      }
-      ctx->SetOutputDim("FusedOutput", phi::make_ddim({numel}));
-      VLOG(4) << "FusedOutput size:" << phi::make_ddim({numel});
-#else
-      return;
-#endif
-    } else {
-      auto alignment = [](size_t size, size_t align_size) {
-        size_t remaining = size % align_size;
-        auto aligned_size =
-            remaining == 0 ? size : size + (align_size - remaining);
-        VLOG(4) << remaining << " " << size << " " << align_size << " "
-                << aligned_size;
-        return aligned_size;
-      };
-      VLOG(4) << "align_size: " << align_size;
-      if (use_align && align_size > 0) {
-        int64_t numel = 0;
-        auto dims = ctx->GetInputsDim("Input");
-        for (const auto &dim : dims) {
-          auto size = phi::product(dim);
-          auto len = use_align
-                         ? alignment(static_cast<size_t>(size) * size_of_dtype,
-                                     align_size) /
-                               size_of_dtype
-                         : static_cast<size_t>(size);
-          numel += len;
-        }
-        ctx->SetOutputDim("FusedOutput", phi::make_ddim({numel}));
-        VLOG(4) << "FusedOutput size:" << phi::make_ddim({numel});
-      }
-    }
-  }
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext &context) const override {
-    auto dtype = static_cast<framework::proto::VarType::Type>(
-        context.Attr<int>("dtype"));
-    return phi::KernelKey(dtype, context.GetPlace());
-  }
-
-  phi::KernelKey GetKernelTypeForVar(
-      const std::string &var_name,
-      const phi::DenseTensor &tensor,
-      const phi::KernelKey &expected_kernel_type) const override {
-    return phi::KernelKey(phi::Backend::ALL_BACKEND,
-                          tensor.layout(),
-                          expected_kernel_type.dtype());
-  }
-};
-
-class CoalesceTensorOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput("Input",
-             "(vector<phi::DenseTensor>) The input tensors of"
-             " coalesce_tensor operator.")
-        .AsDuplicable();
-    AddOutput("Output",
-              "(vector<phi::DenseTensor>) The output "
-              "tensors of coalesce_tensor operator. And the address "
-              "of output tensors are continuous, they are sliced from the "
-              "tensor of FusedOutput.")
-        .AsDuplicable();
-    AddOutput("FusedOutput",
-              "(phi::DenseTensor) The output tensor "
-              "of coalesce_tensor operator. And the tensors of"
-              " Output is sliced from the tensor of FusedOutput.");
-    AddAttr<int>("dtype", "The output data type.");
-    AddAttr<bool>("copy_data", "Whether to copy the Input value to Output.")
-        .SetDefault(false);
-    AddAttr<bool>("set_constant",
-                  "Whether to set the Output with a constant value.")
-        .SetDefault(false);
-    AddAttr<bool>("persist_output",
-                  "Whether to persist the original Output value.")
-        .SetDefault(false);
-    AddAttr<float>("constant",
-                   "If set_constant is true, the constant value will be used "
-                   "to set the Output.")
-        .SetDefault(0.0);
-    AddAttr<bool>("check_name",
-                  "Whether to check the name of Input and Output to ensure "
-                  "they are the same separately.")
-        .SetDefault(false);
-    AddAttr<bool>("use_align",
-                  "Whether to consider memory chunk and take alignment into "
-                  "account for inputs and outputs.")
-        .SetDefault(true);
-    AddAttr<int>("align_size", "The alignment size when use_align is True")
-        .SetDefault(-1);
-    AddAttr<int>("user_defined_size_of_dtype",
-                 "The user defined size of dtype. This is used to coalesce "
-                 "grad vars and merged_grad vars at the same time. For some "
-                 "strategy, the dtype of fused_grad_vars and the dtype of "
-                 "fused_grad_merged_vars are not identical, which will cause "
-                 "the shape of these two coalesced vars are different. To "
-                 "make sure the shape of these two vars are identical with "
-                 "each other, this attr is added.")
-        .SetDefault(-1);
-    AddAttr<std::vector<int64_t>>(
-        "concated_shapes",
-        "The concated shapes of each shape of the input tensors. "
-        "If any of the input tensors are not inited, this is used to "
-        "init the output tensor shape, together with "
-        "attribute(concated_ranks).")
-        .SetDefault({});
-    AddAttr<std::vector<int64_t>>(
-        "concated_ranks",
-        "The concated ranks of each rank of the input tensors. "
-        "If any of the input tensors are not inited, this is used to "
-        "init the output tensor shape, together with "
-        "attribute(concated_shapes).")
-        .SetDefault({});
-    AddComment(R"DOC(
-CoalesceTensor Operator.
-
-coalesce_tensor is used to make the address of Output
-continuous according to the Input. This Op will alloc a big tensor
-according to the tensors of Input, the dtype is the same with those input tensors,
-the size is the sum of those input tensors' numel, and the dim of the big
-tensor is {sum(numel)}. And the big tensor is stored in FusedOutput.
-The tensors of Output are sliced from the tensor of FusedOutput.
-Note that, the dtype of Input should be the same, and the dim of Input
-and Output should equal.
-The tensors of Input and Output could be the same or different. And
-coalesce_tensor allows copying the value of Input to Output, or
-setting the Output with a constant value, or persist the original Output
-value.
-
-)DOC");
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-DECLARE_INFER_SHAPE_FUNCTOR(coalesce_tensor,
-                            CoalesceTensorInferShapeFunctor,
-                            PD_INFER_META(phi::CoalesceTensorInferMeta));
-
-REGISTER_OPERATOR(coalesce_tensor,
-                  paddle::operators::CoalesceTensorOp,
-                  paddle::operators::CoalesceTensorOpMaker,
-                  CoalesceTensorInferShapeFunctor);
-namespace ops = paddle::operators;
-namespace plat = paddle::platform;
-
-REGISTER_OP_VERSION(coalesce_tensor)
-    .AddCheckpoint(
-        R"ROC(
-              Upgrade coalesce_tensor: add a new attribute [use_align].)ROC",
-        paddle::framework::compatible::OpVersionDesc().NewAttr(
-            "use_align",
-            "In order to optionally take memory alignment into account when "
-            "coalescing tensors. The default value is true to be compatible "
-            "with before.",
-            true))
-    .AddCheckpoint(
-        R"ROC(
-                Upgrade coalesce_tensor: add a new attribute [align_size].)ROC",
-        paddle::framework::compatible::OpVersionDesc().NewAttr(
-            "align_size",
-            "In order to optionally take memory alignment into account when "
-            "coalescing tensors. The default value is -1 and use the default "
-            "align_size "
-            "of each place to be compatible with before.",
-            -1));
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index 266e981254f36..e7b0a5ca4f330 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -171,15 +171,6 @@
     data_type : x
   inplace : (x -> out), (input_found_infinite -> output_found_infinite)
 
-- op : coalesce_tensor
-  args : (Tensor[] input, DataType dtype, bool copy_data = false, bool set_constant = false, bool persist_output = false, float constant = 0.0, bool use_align = true, int align_size = -1, int size_of_dtype = -1, int64_t[] concated_shapes = {}, int64_t[] concated_ranks = {})
-  output : Tensor[](output){input.size()}, Tensor(fused_output)
-  infer_meta :
-    func : CoalesceTensorInferMeta
-  kernel :
-    func : coalesce_tensor
-    data_type : dtype
-
 - op : concat
   args : (Tensor[] x, Scalar(int64_t) axis)
   output : Tensor
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index 97cca3ced8078..928797a3c3941 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -419,6 +419,14 @@
   outputs :
     out : Out
 
+- op : coalesce_tensor
+  inputs :
+    {input : Input}
+  outputs :
+    {output : Output, fused_output : FusedOutput}
+  attrs :
+    {size_of_dtype : user_defined_size_of_dtype}
+
 - op : complex
   backward : complex_grad
   inputs :
diff --git a/paddle/phi/api/yaml/op_version.yaml b/paddle/phi/api/yaml/op_version.yaml
index 35ee674cd14da..b13b432c460f7 100644
--- a/paddle/phi/api/yaml/op_version.yaml
+++ b/paddle/phi/api/yaml/op_version.yaml
@@ -72,6 +72,24 @@
         - add_input : Max
           comment : Pass the mix, min value as input, not attribute. Max is dispensable.
 
+- op : coalesce_tensor
+  version :
+    - checkpoint : "Upgrade coalesce_tensor: add a new attribute [use_align]."
+      action :
+        - add_attr : use_align
+          comment : In order to optionally take memory alignment into account when
+            coalescing tensors. The default value is true to be compatible
+            with before.
+          default : "true"
+    - checkpoint : "Upgrade coalesce_tensor: add a new attribute [align_size]."
+      action :
+        - add_attr : align_size
+          comment : In order to optionally take memory alignment into account when
+            coalescing tensors. The default value is -1 and use the default
+            align_size
+            of each place to be compatible with before.
+          default : -1
+
 - op : embedding
   version :
     - checkpoint : Upgrade flip, add new attr [axis] and delete attr [dims]
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index 2f0960e9d30c5..eaaf9f61fa7ac 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -416,6 +416,15 @@
     func : clip_by_norm {dense -> dense}
            clip_by_norm_sr {selected_rows -> selected_rows}
 
+- op : coalesce_tensor
+  args : (Tensor[] input, DataType dtype, bool copy_data = false, bool set_constant = false, bool persist_output = false, float constant = 0.0, bool use_align = true, int align_size = -1, int size_of_dtype = -1, int64_t[] concated_shapes = {}, int64_t[] concated_ranks = {})
+  output : Tensor[](output){input.size()}, Tensor(fused_output)
+  infer_meta :
+    func : CoalesceTensorInferMeta
+  kernel :
+    func : coalesce_tensor
+    data_type : dtype
+
 - op : complex
   args : (Tensor real, Tensor imag)
   output : Tensor
diff --git a/paddle/phi/kernels/coalesce_tensor_kernel.cc b/paddle/phi/kernels/coalesce_tensor_kernel.cc
index 559c88eebbc48..7b91b328acdf5 100644
--- a/paddle/phi/kernels/coalesce_tensor_kernel.cc
+++ b/paddle/phi/kernels/coalesce_tensor_kernel.cc
@@ -273,6 +273,7 @@ PD_REGISTER_KERNEL(coalesce_tensor,
                    int,
                    float,
                    double) {
+  kernel->InputAt(0).SetBackend(phi::Backend::ALL_BACKEND);
   kernel->OutputAt(1).SetDataType(phi::DataType::UNDEFINED);
 }
 
diff --git a/paddle/phi/ops/compat/coalesce_tensor_sig.cc b/paddle/phi/ops/compat/coalesce_tensor_sig.cc
deleted file mode 100644
index a2219850ea6d5..0000000000000
--- a/paddle/phi/ops/compat/coalesce_tensor_sig.cc
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/core/compat/op_utils.h"
-
-namespace phi {
-KernelSignature CoalesceTensorOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature("coalesce_tensor",
-                         {"Input"},
-                         {"dtype",
-                          "copy_data",
-                          "set_constant",
-                          "persist_output",
-                          "constant",
-                          "use_align",
-                          "align_size",
-                          "user_defined_size_of_dtype",
-                          "concated_shapes",
-                          "concated_ranks"},
-                         {"Output", "FusedOutput"});
-}
-
-}  // namespace phi
-
-PD_REGISTER_ARG_MAPPING_FN(coalesce_tensor,
-                           phi::CoalesceTensorOpArgumentMapping);

From 45ce0ad536cd7e189bed2e2a82d3c6077009371d Mon Sep 17 00:00:00 2001
From: Aurelius84 <zhangliujie@baidu.com>
Date: Tue, 9 May 2023 21:26:29 +0800
Subject: [PATCH 244/405] [CINN]Adjust Bert unittest loss ground truth (#53628)

[CINN]Adjust Bert unittest loss ground truth, see: PaddlePaddle/CINN#1357
---
 test/prim/model/test_bert_prim_cinn.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/test/prim/model/test_bert_prim_cinn.py b/test/prim/model/test_bert_prim_cinn.py
index c6438536673f6..aed75a9ca5dc3 100644
--- a/test/prim/model/test_bert_prim_cinn.py
+++ b/test/prim/model/test_bert_prim_cinn.py
@@ -59,16 +59,16 @@
 ]
 
 DY2ST_PRIM_CINN_GT = [
-    10.976988792419434,
-    10.345282554626465,
-    10.33430004119873,
-    10.275235176086426,
-    10.22808837890625,
-    10.200264930725098,
-    10.161019325256348,
-    10.10872745513916,
-    10.121539115905762,
-    9.990568161010742,
+    10.861940383911133,
+    10.350666046142578,
+    10.331628799438477,
+    10.267815589904785,
+    10.22376537322998,
+    10.198311805725098,
+    10.16231632232666,
+    10.10612678527832,
+    10.122684478759766,
+    10.0067138671875,
 ]
 
 

From 49074859740b87255e55ecde173858c9586735c9 Mon Sep 17 00:00:00 2001
From: zhangkaihuo <zhangkaihuo@baidu.com>
Date: Tue, 9 May 2023 22:12:13 +0800
Subject: [PATCH 245/405] Add compare accuracy api (#53430)

---
 python/paddle/amp/__init__.py         |   1 +
 python/paddle/amp/accuracy_compare.py | 739 ++++++++++++++++++++++++++
 python/paddle/amp/debugging.py        |  62 +++
 python/unittest_py/requirements.txt   |   1 +
 test/amp/test_compare_accuracy_api.py |  71 +++
 5 files changed, 874 insertions(+)
 create mode 100644 python/paddle/amp/accuracy_compare.py
 create mode 100644 test/amp/test_compare_accuracy_api.py

diff --git a/python/paddle/amp/__init__.py b/python/paddle/amp/__init__.py
index 5fa8055ba233b..8758291107702 100644
--- a/python/paddle/amp/__init__.py
+++ b/python/paddle/amp/__init__.py
@@ -25,6 +25,7 @@
 from .grad_scaler import OptimizerState  # noqa: F401
 
 from . import debugging  # noqa: F401
+from . import accuracy_compare  # noqa: F401
 
 from paddle.fluid import core
 from paddle.fluid.framework import (
diff --git a/python/paddle/amp/accuracy_compare.py b/python/paddle/amp/accuracy_compare.py
new file mode 100644
index 0000000000000..85f8f78ac0da5
--- /dev/null
+++ b/python/paddle/amp/accuracy_compare.py
@@ -0,0 +1,739 @@
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import numpy as np
+
+
+# Judge whether the value is within the range indicated by fp16
+def is_infinite(value, dtype=np.float16):
+    # return value > np.finfo(np.float16).max or value < np.finfo(np.float16).min
+    array = np.array([value]).astype(dtype)
+    return np.isinf(array) or np.isnan(array)
+
+
+# Judge whether the value of fp32 is equal to that of fp16
+def is_allclose(actual, expected, atol=1e-2, rtol=1e-2):
+    return np.allclose(
+        np.array([actual]), np.array([expected]), atol=atol, rtol=rtol
+    )
+
+
+class TensorInfo:
+    def __init__(self):
+        self.op_type = None
+        self.tensor_name = None
+        self.dtype = None
+        self.numel = None
+        self.max_value = None
+        self.min_value = None
+        self.mean_value = None
+        self.has_inf = None
+        self.has_nan = None
+        self.num_zero = None
+
+    def __str__(self):
+        return "[TensorInfo] op_type={}, tensor_name={}, dtype={}, numel={}, has_inf={}, has_nan={}, num_zero={}, max_value={:.6f}, min_value={:.6f}, mean_value={:.6f}".format(
+            self.op_type,
+            self.tensor_name,
+            self.dtype,
+            self.numel,
+            self.has_inf,
+            self.has_nan,
+            self.num_zero,
+            self.max_value,
+            self.min_value,
+            self.mean_value,
+        )
+
+    def key(
+        self,
+    ):
+        return self.op_type + "/" + self.tensor_name
+
+    def init_from_string(self, line):
+        try:
+            line_frags = line.strip().split(" ")
+            for frag in line_frags:
+                word_str = (
+                    frag.replace("[", "").replace("]", "").replace(",", "")
+                )
+                words = word_str.split("=")
+                if words[0] == "op":
+                    self.op_type = words[1]
+                elif words[0] == "tensor":
+                    self.tensor_name = words[1]
+                elif words[0] == "dtype":
+                    self.dtype = words[1]
+                elif words[0] == "numel":
+                    self.numel = np.int64(words[1])
+                elif words[0] == "max":
+                    self.max_value = np.float32(words[1])
+                elif words[0] == "min":
+                    self.min_value = np.float32(words[1])
+                elif words[0] == "mean":
+                    self.mean_value = np.float32(words[1])
+                elif words[0] == "num_inf":
+                    self.has_inf = int(words[1])
+                elif words[0] == "num_nan":
+                    self.has_nan = int(words[1])
+                elif words[0] == "num_zero":
+                    self.num_zero = np.int64(words[1])
+        except Exception as e:
+            print(f"!! Error parsing {line}")
+        return self
+
+
+class MixedPrecisionTensorInfo:
+    def __init__(
+        self, fp32_tensor_info, fp16_tensor_info, fp32_idx=0, grad_scale=1.0
+    ):
+        self.is_normal = True
+        self.fp32_idx = fp32_idx
+
+        self.fp32_tensor_name = None
+        self.fp32_dtype = None
+        self.fp32_max_value = None
+        self.fp32_min_value = None
+        self.fp32_mean_value = None
+        self.fp32_num_zero = None
+        self.scaled_fp32_max_value = None
+        self.scaled_fp32_min_value = None
+
+        self.fp16_tensor_name = None
+        self.fp16_dtype = None
+        self.fp16_max_value = None
+        self.fp16_min_value = None
+        self.fp16_mean_value = None
+        self.fp16_num_zero = None
+        self.fp16_has_inf = None
+        self.fp16_has_nan = None
+
+        self.fp32_div_fp16_max_value = None
+        self.fp32_div_fp16_min_value = None
+        self.fp32_div_fp16_mean_value = None
+
+        if fp32_tensor_info is not None:
+            self.op_type = fp32_tensor_info.op_type
+            self.numel = fp32_tensor_info.numel
+            self.fp32_num_zero = fp32_tensor_info.num_zero
+            self.fp32_tensor_name = fp32_tensor_info.tensor_name
+            self.fp32_dtype = fp32_tensor_info.dtype
+            self.fp32_max_value = fp32_tensor_info.max_value
+            self.fp32_min_value = fp32_tensor_info.min_value
+            self.fp32_mean_value = fp32_tensor_info.mean_value
+            if "GRAD" in self.fp32_tensor_name:
+                self.scaled_fp32_max_value = (
+                    grad_scale * fp32_tensor_info.max_value
+                )
+                self.scaled_fp32_min_value = (
+                    grad_scale * fp32_tensor_info.min_value
+                )
+
+        if fp16_tensor_info is not None:
+            self.op_type = fp16_tensor_info.op_type
+            self.numel = fp16_tensor_info.numel
+            self.fp16_num_zero = fp16_tensor_info.num_zero
+            self.fp16_tensor_name = fp16_tensor_info.tensor_name
+            self.fp16_dtype = fp16_tensor_info.dtype
+            self.fp16_max_value = fp16_tensor_info.max_value
+            self.fp16_min_value = fp16_tensor_info.min_value
+            self.fp16_mean_value = fp16_tensor_info.mean_value
+            self.fp16_has_inf = fp16_tensor_info.has_inf
+            self.fp16_has_nan = fp16_tensor_info.has_nan
+
+        if fp32_tensor_info is not None and fp16_tensor_info is not None:
+            # Check whether the op name and data are equal
+            assert fp32_tensor_info.op_type == fp16_tensor_info.op_type
+            assert (
+                fp32_tensor_info.numel == fp16_tensor_info.numel
+            ), "Error:\n\tFP32 Tensor Info:{}\n\tFP16 Tensor Info:{}".format(
+                fp32_tensor_info, fp16_tensor_info
+            )
+            # Fp16 divided by fp32
+            self.fp32_div_fp16_max_value = self._div(
+                self.fp16_max_value, self.fp32_max_value
+            )
+            self.fp32_div_fp16_min_value = self._div(
+                self.fp16_min_value, self.fp32_min_value
+            )
+            self.fp32_div_fp16_mean_value = self._div(
+                self.fp16_mean_value, self.fp32_mean_value
+            )
+
+        self._check_normal()
+
+    def __str__(self):
+        def _float_str(value):
+            return f"{value:.6f}" if value is not None else value
+
+        debug_str = "[MixedPrecisionTensorInfo] op_type={}, numel={}".format(
+            self.op_type, self.numel
+        )
+        debug_str += "\n  FP32: tensor_name={}, dtype={}, max_value={}, min_value={}, mean_value={}".format(
+            self.fp32_tensor_name,
+            self.fp32_dtype,
+            _float_str(self.fp32_max_value),
+            _float_str(self.fp32_min_value),
+            _float_str(self.fp32_mean_value),
+        )
+        debug_str += "\n  FP16: tensor_name={}, dtype={}, max_value={}, min_value={}, mean_value={}, has_inf={}, has_nan={}".format(
+            self.fp16_tensor_name,
+            self.fp16_dtype,
+            _float_str(self.fp16_max_value),
+            _float_str(self.fp16_min_value),
+            _float_str(self.fp16_mean_value),
+            self.fp16_has_inf,
+            self.fp16_has_nan,
+        )
+        return debug_str
+
+    def _div(self, a, b):
+        if a is not None and b is not None:
+            return a / b if b != 0 else 1
+        return None
+
+    def get_tensor_name(self):
+        if self.fp32_tensor_name is None:
+            return self.fp16_tensor_name  # + "#" + str(self.idx)
+        elif self.fp16_tensor_name is None:
+            return self.fp32_tensor_name + "#" + str(self.fp32_idx)
+        else:
+            return (
+                self.fp16_tensor_name.replace(".cast_fp16", "/.cast_fp16/")
+                + "#"
+                + str(self.fp32_idx)
+            )
+
+    def _check_normal(self):
+        # When the OP meets the following conditions, it is abnormal data, and use --skip_normal_tensors to retain the data in Excel:
+        # 1. The number of OP outputs exceeds the indication range of int32
+        # 2. The output data exceeds the representation range of fp16
+        # 3. Nan or inf appears in fp16 output data
+        # 4. The maximum value of fp32 is not equal to the maximum value of fp16
+        # 5. The minimum value of fp32 is not equal to the minimum value of fp16
+        if self.numel is not None and self.numel > np.iinfo(np.int32).max:
+            self.is_normal = False
+            return
+
+        check_list = [
+            self.fp32_max_value,
+            self.fp32_min_value,
+            self.scaled_fp32_max_value,
+            self.scaled_fp32_min_value,
+            self.fp16_max_value,
+            self.fp16_min_value,
+        ]
+
+        for value in check_list:
+            if value is not None and is_infinite(value):
+                self.is_normal = False
+                return
+
+        if self.fp16_has_inf is not None and self.fp16_has_inf:
+            self.is_normal = False
+            return
+        if self.fp16_has_nan is not None and self.fp16_has_nan:
+            self.is_normal = False
+            return
+
+        if (
+            self.scaled_fp32_max_value is not None
+            and self.fp16_max_value is not None
+            and not is_allclose(self.fp16_max_value, self.scaled_fp32_max_value)
+        ):
+            self.is_normal = False
+            return
+        if (
+            self.scaled_fp32_min_value is not None
+            and self.fp16_min_value is not None
+            and not is_allclose(self.fp16_min_value, self.scaled_fp32_min_value)
+        ):
+            self.is_normal = False
+            return
+
+
+class ExcelWriter:
+    def __init__(self, log_fp32_dir, log_fp16_dir, output_path):
+        self.log_fp32_dir = log_fp32_dir
+        self.log_fp16_dir = log_fp16_dir
+
+        try:
+            import xlsxwriter as xlw
+        except ImportError:
+            print(
+                "import xlsxwriter failed. please run 'pip install xlsxwriter==3.0.9' to install it"
+            )
+
+        self.workbook = xlw.Workbook(output_path)
+        self.title_format = self.workbook.add_format(
+            {
+                'bold': True,
+                'border': 1,
+                'font_color': 'black',
+                'bg_color': '#6495ED',
+                'align': 'center',
+            }
+        )
+        self.tensor_name_format = self.workbook.add_format(
+            {'bold': True, 'bg_color': '#F5F5F5'}
+        )
+        self.red_bg_cell_format = self.workbook.add_format(
+            {'bold': True, 'bg_color': 'red'}
+        )
+        self.yellow_bg_cell_format = self.workbook.add_format(
+            {'bold': True, 'bg_color': 'yellow'}
+        )
+        self.orange_bg_cell_format = self.workbook.add_format(
+            {'bold': True, 'bg_color': 'orange'}
+        )
+
+    def close(self):
+        self.workbook.close()
+        self.workbook = None
+
+    def _write_dtype(self, worksheet, value, row, col):
+        if value is None:
+            worksheet.write(row, col, "--")
+        else:
+            if value == "fp16":
+                worksheet.write(row, col, value, self.yellow_bg_cell_format)
+            else:
+                worksheet.write(row, col, value)
+
+    def _write_tensor_name(self, worksheet, mp_tensor_info, row, col):
+        tensor_name = mp_tensor_info.get_tensor_name()
+        if (
+            mp_tensor_info.fp32_tensor_name is not None
+            and mp_tensor_info.fp16_tensor_name
+        ):
+            worksheet.write(row, col, tensor_name, self.tensor_name_format)
+        else:
+            worksheet.write(row, col, tensor_name)
+
+    def _write_maxmin_value(
+        self, worksheet, value, row, col, check_finite=True
+    ):
+        if value is None:
+            worksheet.write(row, col, "--")
+        else:
+            if abs(value) < 1e-5:
+                value_str = f"{value:.6E}"
+            else:
+                value_str = f"{value:.6f}"
+            if check_finite and is_infinite(value, np.float16):
+                worksheet.write(row, col, value_str, self.red_bg_cell_format)
+            else:
+                worksheet.write(row, col, value_str)
+
+    def _write_tensor_num_zero(
+        self, worksheet, value, row, col, check_finite=True
+    ):
+        if value is None:
+            worksheet.write(row, col, "--")
+        else:
+            value_str = f"{value:>10d}"
+            worksheet.write(row, col, value_str)
+
+    def _write_infinite_status(self, worksheet, value, row, col):
+        if value is None:
+            worksheet.write(row, col, "--")
+        else:
+            if value == 1:
+                worksheet.write(row, col, value, self.red_bg_cell_format)
+            else:
+                worksheet.write(row, col, value)
+
+    def _write_fp32divfp16_value(self, worksheet, value, row, col, loss_scale):
+        def _in_range(value, scale=1):
+            return value > scale * 0.95 and value < scale * 1.05
+
+        if value is None:
+            worksheet.write(row, col, "--")
+        else:
+            value_str = f"{value:.6f}"
+            if _in_range(value, scale=1) or _in_range(value, loss_scale):
+                worksheet.write(row, col, value_str)
+            else:
+                worksheet.write(row, col, value_str, self.orange_bg_cell_format)
+
+    def _write_titles(self, worksheet, loss_scale, row):
+        column_width_dict = {
+            "op_type": 24,
+            "tensor_name": 60,
+            "numel": 10,
+            "num_zero": 10,
+            "infinite": 8,
+            "dtype": 8,
+            "max_value": 16,
+            "min_value": 16,
+            "mean_value": 16,
+            "has_inf": 8,
+            "has_nan": 8,
+        }
+        title_names = ["op_type", "tensor_name", "numel", "infinite"]
+        if self.log_fp16_dir is None:
+            # only fp32 values
+            worksheet.merge_range("E1:H1", "fp32", self.title_format)
+            worksheet.merge_range(
+                "I1:J1", f"fp32 (scale={loss_scale})", self.title_format
+            )
+            title_names.extend(
+                [
+                    "dtype",
+                    "max_value",
+                    "min_value",
+                    "mean_value",
+                    "max_value",
+                    "min_value",
+                ]
+            )
+        elif self.log_fp32_dir is None:
+            # only fp16 values
+            worksheet.merge_range(
+                "E1:J1", f"fp16 (scale={loss_scale})", self.title_format
+            )
+            title_names.extend(
+                [
+                    "dtype",
+                    "max_value",
+                    "min_value",
+                    "mean_value",
+                    "num_zero",
+                    "has_inf",
+                    "has_nan",
+                ]
+            )
+        else:
+            # fp32 and fp16 values
+            worksheet.merge_range("E1:H1", "fp32", self.title_format)
+            worksheet.merge_range(
+                "I1:N1", f"fp16 (scale={loss_scale})", self.title_format
+            )
+            worksheet.merge_range("O1:Q1", "fp16 / fp32", self.title_format)
+            title_names.extend(
+                [
+                    "dtype",
+                    "max_value",
+                    "min_value",
+                    "mean_value",
+                    "num_zero",
+                    "dtype",
+                    "max_value",
+                    "min_value",
+                    "mean_value",
+                    "num_zero",
+                    "has_inf",
+                    "has_nan",
+                    "max_value",
+                    "min_value",
+                    "mean_value",
+                ]
+            )
+
+        for col in range(len(title_names)):
+            col_char = chr(ord("A") + col)
+            worksheet.set_column(
+                col_char + ":" + col_char, column_width_dict[title_names[col]]
+            )
+        for col in range(len(title_names)):
+            worksheet.write(row, col, title_names[col], self.title_format)
+
+    def add_worksheet(
+        self, mp_tensor_info_list, sheetname, loss_scale, skip_normal_tensors
+    ):
+
+        assert self.workbook is not None
+
+        worksheet = self.workbook.add_worksheet(sheetname)
+        row = 1
+
+        self._write_titles(worksheet, loss_scale, row)
+        row += 1
+
+        infinite_op_types = []
+        for tensor_info in mp_tensor_info_list:
+            if (
+                not tensor_info.is_normal
+                and tensor_info.op_type not in infinite_op_types
+            ):
+                infinite_op_types.append(tensor_info.op_type)
+
+            if skip_normal_tensors and tensor_info.is_normal:
+                continue
+
+            worksheet.write(row, 0, tensor_info.op_type)
+            self._write_tensor_name(worksheet, tensor_info, row, 1)
+
+            if tensor_info.numel > np.iinfo(np.int32).max:
+                worksheet.write(
+                    row, 2, tensor_info.numel, self.bad_value_format
+                )
+            else:
+                worksheet.write(row, 2, tensor_info.numel)
+
+            if tensor_info.is_normal:
+                worksheet.write(row, 3, "0")
+            else:
+                worksheet.write(row, 3, "1", self.red_bg_cell_format)
+
+            col = 4
+
+            if self.log_fp32_dir is not None:
+                self._write_dtype(worksheet, tensor_info.fp32_dtype, row, col)
+                self._write_maxmin_value(
+                    worksheet, tensor_info.fp32_max_value, row, col + 1
+                )
+                self._write_maxmin_value(
+                    worksheet, tensor_info.fp32_min_value, row, col + 2
+                )
+                self._write_maxmin_value(
+                    worksheet, tensor_info.fp32_mean_value, row, col + 3
+                )
+                self._write_tensor_num_zero(
+                    worksheet, tensor_info.fp32_num_zero, row, col + 4
+                )
+                col += 5
+
+                if self.log_fp16_dir is None:
+                    self._write_maxmin_value(
+                        worksheet, tensor_info.scaled_fp32_max_value, row, col
+                    )
+                    self._write_maxmin_value(
+                        worksheet,
+                        tensor_info.scaled_fp32_min_value,
+                        row,
+                        col + 1,
+                    )
+                    col += 2
+
+            if self.log_fp16_dir is not None:
+                self._write_dtype(worksheet, tensor_info.fp16_dtype, row, col)
+                self._write_maxmin_value(
+                    worksheet, tensor_info.fp16_max_value, row, col + 1
+                )
+                self._write_maxmin_value(
+                    worksheet, tensor_info.fp16_min_value, row, col + 2
+                )
+                self._write_maxmin_value(
+                    worksheet, tensor_info.fp16_mean_value, row, col + 3
+                )
+                self._write_tensor_num_zero(
+                    worksheet, tensor_info.fp32_num_zero, row, col + 4
+                )
+                col += 5
+
+                self._write_infinite_status(
+                    worksheet, tensor_info.fp16_has_inf, row, col
+                )
+                self._write_infinite_status(
+                    worksheet, tensor_info.fp16_has_nan, row, col + 1
+                )
+                col += 2
+
+            if self.log_fp32_dir is not None and self.log_fp16_dir is not None:
+                self._write_fp32divfp16_value(
+                    worksheet,
+                    tensor_info.fp32_div_fp16_max_value,
+                    row,
+                    col,
+                    loss_scale,
+                )
+                self._write_fp32divfp16_value(
+                    worksheet,
+                    tensor_info.fp32_div_fp16_min_value,
+                    row,
+                    col + 1,
+                    loss_scale,
+                )
+                self._write_fp32divfp16_value(
+                    worksheet,
+                    tensor_info.fp32_div_fp16_mean_value,
+                    row,
+                    col + 2,
+                    loss_scale,
+                )
+                col += 3
+
+            row += 1
+
+        print(f"-- OP Types produce infinite outputs: {infinite_op_types}")
+
+
+def parse_log(log_dir, filename, specified_op_list=None):
+    if log_dir is None or filename is None:
+        return None
+
+    complete_filename = log_dir + "/" + filename
+    tensor_info_list = []
+    has_tensor_name = False
+
+    try:
+        with open(complete_filename, 'r') as f:
+            lines = f.readlines()
+            for i in range(len(lines)):
+                if i % 10 == 0:
+                    print(
+                        f"-- Processing {i:-8d} / {len(lines):-8d} line",
+                        end="\r",
+                    )
+                # [op=adamw] [tensor=encoder_layer_20_multi_head_att_output_fc_0.w_0], numel: 294912, max: 0.005773, min: -0.005774
+                line = lines[i]
+                if "[PRECISION]" in line:
+                    tensor_info = TensorInfo()
+                    tensor_info.init_from_string(line)
+                    if (
+                        tensor_info.tensor_name is not None
+                        and tensor_info.tensor_name != ""
+                    ):
+                        has_tensor_name = True
+                    if (
+                        specified_op_list is None
+                        or tensor_info.op_type in specified_op_list
+                    ):
+                        tensor_info_list.append(tensor_info)
+                    # print(tensor_info)
+    except FileNotFoundError:
+        print("the file ", complete_filename, "is not found")
+        return None, has_tensor_name
+    return tensor_info_list, has_tensor_name
+
+
+def merge_tensor_info_list(
+    fp32_tensor_info_list, fp16_tensor_info_list, grad_scale
+):
+    mp_tensor_info_list = []
+    if fp16_tensor_info_list is not None:
+        fp32_tensor_info_dict = {}
+        fp32_write_count = {}
+        if fp32_tensor_info_list is not None:
+            for tensor_info in fp32_tensor_info_list:
+                tensor_info_key = tensor_info.key()
+                count = fp32_write_count.get(tensor_info_key, 0)
+                fp32_write_count[tensor_info_key] = count + 1
+                fp32_tensor_info_dict[
+                    tensor_info_key + "#" + str(count)
+                ] = tensor_info
+
+        fp32_read_count = {}
+        for i in range(len(fp16_tensor_info_list)):
+            if i % 10 == 0:
+                print(
+                    "-- Processing {:-8d} / {:-8d} FP16 Tensor Info".format(
+                        i, len(fp16_tensor_info_list)
+                    ),
+                    end="\r",
+                )
+            fp16_tensor_info = fp16_tensor_info_list[i]
+            fp32_tensor_info_key = (
+                fp16_tensor_info.key()
+                .replace(".cast_fp16", "")
+                .replace(".cast_fp32", "")
+            )
+            count = fp32_read_count.get(fp32_tensor_info_key, 0)
+            fp32_tensor_info = fp32_tensor_info_dict.get(
+                fp32_tensor_info_key + "#" + str(count), None
+            )
+            if fp32_tensor_info is not None:
+                fp32_read_count[fp32_tensor_info_key] = count + 1
+            mp_tensor_info = MixedPrecisionTensorInfo(
+                fp32_tensor_info, fp16_tensor_info, count, grad_scale
+            )
+            mp_tensor_info_list.append(mp_tensor_info)
+            # print(mp_tensor_info)
+    elif fp32_tensor_info_list is not None:
+        fp32_count = {}
+        for i in range(len(fp32_tensor_info_list)):
+            if i % 10 == 0:
+                print(
+                    "-- Processing {:-8d} / {:-8d} FP32 Tensor Info".format(
+                        i, len(fp32_tensor_info_list)
+                    ),
+                    end="\r",
+                )
+            tensor_info = fp32_tensor_info_list[i]
+            tensor_info_key = tensor_info.key()
+            count = fp32_count.get(tensor_info_key, 0)
+            fp32_count[tensor_info_key] = count + 1
+            mp_tensor_info = MixedPrecisionTensorInfo(
+                tensor_info, None, count, grad_scale
+            )
+            mp_tensor_info_list.append(mp_tensor_info)
+
+    return mp_tensor_info_list
+
+
+def compare_accuracy(
+    dump_path,
+    another_dump_path,
+    output_filename,
+    loss_scale=1,
+    dump_all_tensors=False,
+):
+    excel_writer = ExcelWriter(dump_path, another_dump_path, output_filename)
+    grad_scale = loss_scale
+    workerlog_filenames = []
+    filenames = os.listdir(dump_path)
+    for name in filenames:
+        if "worker_" in name:
+            workerlog_filenames.append(name)
+    print(
+        "-- There are {} workerlogs under {}: {}".format(
+            len(workerlog_filenames), dump_path, workerlog_filenames
+        )
+    )
+
+    for filename in sorted(workerlog_filenames):
+        print(
+            "-- [Step 1/4] Parsing FP32 logs under {}/{}".format(
+                dump_path, filename
+            )
+        )
+        fp32_tensor_info_list, fp32_has_tensor_name = parse_log(
+            dump_path, filename, None
+        )
+        print(
+            "-- [Step 2/4] Parsing FP16 logs under {}/{}".format(
+                another_dump_path, filename
+            )
+        )
+        fp16_tensor_info_list, fp16_has_tensor_name = parse_log(
+            another_dump_path, filename, None
+        )
+
+        print(
+            "-- [Step 3/4] Merge FP32 and FP16 tensor info for {}".format(
+                filename
+            )
+        )
+        mp_tensor_info_list = merge_tensor_info_list(
+            fp32_tensor_info_list, fp16_tensor_info_list, grad_scale
+        )
+        print(
+            "-- [Step 4/4] Add worksheet for mixed precision tensor info of {}".format(
+                filename
+            )
+        )
+        excel_writer.add_worksheet(
+            mp_tensor_info_list,
+            filename,
+            loss_scale,
+            False,
+        )
+
+    print(f"-- Write to {output_filename}")
+
+    print("")
+    excel_writer.close()
diff --git a/python/paddle/amp/debugging.py b/python/paddle/amp/debugging.py
index 69ee21fe4c322..c55d3506be33f 100644
--- a/python/paddle/amp/debugging.py
+++ b/python/paddle/amp/debugging.py
@@ -30,6 +30,7 @@
     "collect_operator_stats",
     "enable_tensor_checker",
     "disable_tensor_checker",
+    "compare_accuracy",
 ]
 
 
@@ -424,6 +425,67 @@ def collect_operator_stats():
     disable_operator_stats_collection()
 
 
+def compare_accuracy(
+    dump_path,
+    another_dump_path,
+    output_filename,
+    loss_scale=1,
+    dump_all_tensors=False,
+):
+    r"""
+    This is a precision comparison tool that can be used to compare log data of float16 and float32.
+
+    Args:
+        dump_path(str): The path of the running log, such as the log for execution using the fp32 type.
+        another_dump_path(str): the path of another running log ,such as the log for execution using the fp16 type.
+        output_filename(str): the excel file nmae of compare output.
+        loss_scale(float): the loss_scale during the training phase.
+        dump_all_tensors(bool, optional): dump all tensor, It is currently not support. Default is False.
+
+    Examples:
+
+        ..  code-block:: python
+
+            import paddle
+            from paddle.fluid import core
+            try:
+                import xlsxwriter as xlw
+            except ImportError:
+                import subprocess
+
+                subprocess.check_call(
+                    ['python', '-m', 'pip', 'install', 'xlsxwriter==3.0.9']
+                )
+                import xlsxwriter as xlw
+
+            if core.is_compiled_with_cuda():
+                paddle.set_flags(
+                    {"FLAGS_check_nan_inf": 1, "FLAGS_check_nan_inf_level": 3}
+                )
+                path = "workerlog_log_dir"
+                paddle.fluid.core.set_nan_inf_debug_path(path)
+                x = paddle.to_tensor(
+                    [2, 3, 4, 0], dtype="float32"
+                )
+                y = paddle.to_tensor(
+                    [1, 5, 2, 0], dtype="float32"
+                )
+                z1 = x + y
+                out_excel = "compary_accuracy_out_excel.csv"
+                paddle.amp.debugging.compare_accuracy(
+                    path, path, out_excel, loss_scale=1, dump_all_tensors=False
+                )
+    """
+    assert dump_all_tensors is False, "It is currently not supported."
+    paddle.amp.accuracy_compare.compare_accuracy(
+        dump_path,
+        another_dump_path,
+        output_filename,
+        loss_scale,
+        dump_all_tensors=False,
+    )
+
+
 def enable_tensor_checker(checker_config):
     """
     The enable_tensor_checker(checker_config) function enables model-level accuracy checking and is used in combination with disables_tensor_checker() to achieve model-level precision checking by checking the output Tensors of all operators within the specified range.
diff --git a/python/unittest_py/requirements.txt b/python/unittest_py/requirements.txt
index 3ca1baafc4ef4..213911a307caa 100644
--- a/python/unittest_py/requirements.txt
+++ b/python/unittest_py/requirements.txt
@@ -16,3 +16,4 @@ autograd==1.4
 librosa==0.8.1
 parameterized
 wandb>=0.13
+xlsxwriter==3.0.9
diff --git a/test/amp/test_compare_accuracy_api.py b/test/amp/test_compare_accuracy_api.py
new file mode 100644
index 0000000000000..83eb7af8df68e
--- /dev/null
+++ b/test/amp/test_compare_accuracy_api.py
@@ -0,0 +1,71 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import paddle
+from paddle.fluid import core
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda(), "not support cpu TestCompareAccuracyApi"
+)
+class TestCompareAccuracyApi(unittest.TestCase):
+    def calc(self, path, dtype):
+        paddle.fluid.core.set_nan_inf_debug_path(path)
+        x = paddle.to_tensor(
+            [2000, 3000, 4, 0], place=core.CUDAPlace(0), dtype=dtype
+        )
+        y = paddle.to_tensor(
+            [100, 500, 2, 10000], place=core.CUDAPlace(0), dtype=dtype
+        )
+        # normal
+        z1 = x + y
+        # inf
+        z2 = x * y
+
+    def test(self):
+        paddle.set_flags(
+            {"FLAGS_check_nan_inf": 1, "FLAGS_check_nan_inf_level": 3}
+        )
+        fp32_path = "workerlog_fp32_log_dir"
+        fp16_path = "workerlog_fp16_log_dir"
+        self.calc(fp32_path, "float32")
+        self.calc(fp16_path, "float16")
+
+        out_excel = "compary_accuracy_out_excel.csv"
+        paddle.amp.debugging.compare_accuracy(
+            fp32_path,
+            fp16_path,
+            out_excel,
+            loss_scale=1,
+            dump_all_tensors=False,
+        )
+
+    def test2(self):
+        fp32_path = "workerlog_fp32_log_dir"
+        fp16_path = "workerlog_fp16_null_log_dir"
+        self.calc(fp32_path, "float32")
+        out_excel = "compary_accuracy_out_excel_2.csv"
+        paddle.amp.debugging.compare_accuracy(
+            fp32_path,
+            fp16_path,
+            out_excel,
+            loss_scale=1,
+            dump_all_tensors=False,
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()

From 3be7a6c8230bb918422e41396674420459d0c9c0 Mon Sep 17 00:00:00 2001
From: Sanbu <96160062+sanbuphy@users.noreply.github.com>
Date: Wed, 10 May 2023 10:16:25 +0800
Subject: [PATCH 246/405] Support static graph code-gen for yolo_loss (#52946)

---
 .../fluid/operators/detection/CMakeLists.txt  |   1 -
 .../operators/detection/yolov3_loss_op.cc     | 231 ------------------
 .../generator/get_expected_kernel_func.cc     |   7 +
 .../generator/get_expected_kernel_func.h      |   4 +
 paddle/phi/api/yaml/backward.yaml             |  10 +
 paddle/phi/api/yaml/legacy_backward.yaml      |  10 -
 paddle/phi/api/yaml/legacy_ops.yaml           |  11 -
 paddle/phi/api/yaml/op_compat.yaml            |  10 +
 paddle/phi/api/yaml/ops.yaml                  |  12 +
 paddle/phi/ops/compat/yolov3_loss_sig.cc      |  59 -----
 python/paddle/vision/ops.py                   |   2 +-
 11 files changed, 44 insertions(+), 313 deletions(-)
 delete mode 100644 paddle/fluid/operators/detection/yolov3_loss_op.cc
 delete mode 100644 paddle/phi/ops/compat/yolov3_loss_sig.cc

diff --git a/paddle/fluid/operators/detection/CMakeLists.txt b/paddle/fluid/operators/detection/CMakeLists.txt
index 64f9e03d7e061..3e07b1f155452 100644
--- a/paddle/fluid/operators/detection/CMakeLists.txt
+++ b/paddle/fluid/operators/detection/CMakeLists.txt
@@ -54,7 +54,6 @@ detection_library(generate_proposal_labels_op SRCS
 detection_library(multiclass_nms_op SRCS multiclass_nms_op.cc DEPS gpc)
 detection_library(locality_aware_nms_op SRCS locality_aware_nms_op.cc DEPS gpc)
 detection_library(box_clip_op SRCS box_clip_op.cc box_clip_op.cu)
-detection_library(yolov3_loss_op SRCS yolov3_loss_op.cc)
 detection_library(box_decoder_and_assign_op SRCS box_decoder_and_assign_op.cc
                   box_decoder_and_assign_op.cu)
 detection_library(sigmoid_focal_loss_op SRCS sigmoid_focal_loss_op.cc
diff --git a/paddle/fluid/operators/detection/yolov3_loss_op.cc b/paddle/fluid/operators/detection/yolov3_loss_op.cc
deleted file mode 100644
index 21aca33f65a1a..0000000000000
--- a/paddle/fluid/operators/detection/yolov3_loss_op.cc
+++ /dev/null
@@ -1,231 +0,0 @@
-/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-   http://www.apache.org/licenses/LICENSE-2.0
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License. */
-
-#include <memory>
-
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/imperative/type_defs.h"
-#include "paddle/phi/core/infermeta_utils.h"
-#include "paddle/phi/infermeta/backward.h"
-#include "paddle/phi/infermeta/multiary.h"
-
-namespace paddle {
-namespace operators {
-
-class Yolov3LossOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(ctx, "X"),
-                          platform::CPUPlace());
-  }
-};
-
-class Yolov3LossOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput("X",
-             "The input tensor of YOLOv3 loss operator, "
-             "This is a 4-D tensor with shape of [N, C, H, W]."
-             "H and W should be same, and the second dimension(C) stores"
-             "box locations, confidence score and classification one-hot"
-             "keys of each anchor box");
-    AddInput("GTBox",
-             "The input tensor of ground truth boxes, "
-             "This is a 3-D tensor with shape of [N, max_box_num, 5], "
-             "max_box_num is the max number of boxes in each image, "
-             "In the third dimension, stores x, y, w, h coordinates, "
-             "x, y is the center coordinate of boxes and w, h is the "
-             "width and height and x, y, w, h should be divided by "
-             "input image height to scale to [0, 1].");
-    AddInput("GTLabel",
-             "The input tensor of ground truth label, "
-             "This is a 2-D tensor with shape of [N, max_box_num], "
-             "and each element should be an integer to indicate the "
-             "box class id.");
-    AddInput("GTScore",
-             "The score of GTLabel, This is a 2-D tensor in same shape "
-             "GTLabel, and score values should in range (0, 1). This "
-             "input is for GTLabel score can be not 1.0 in image mixup "
-             "augmentation.")
-        .AsDispensable();
-    AddOutput("Loss",
-              "The output yolov3 loss tensor, "
-              "This is a 1-D tensor with shape of [N]");
-    AddOutput("ObjectnessMask",
-              "This is an intermediate tensor with shape of [N, M, H, W], "
-              "M is the number of anchor masks. This parameter caches the "
-              "mask for calculate objectness loss in gradient kernel.")
-        .AsIntermediate();
-    AddOutput("GTMatchMask",
-              "This is an intermediate tensor with shape of [N, B], "
-              "B is the max box number of GT boxes. This parameter caches "
-              "matched mask index of each GT boxes for gradient calculate.")
-        .AsIntermediate();
-
-    AddAttr<int>("class_num", "The number of classes to predict.");
-    AddAttr<std::vector<int>>("anchors",
-                              "The anchor width and height, "
-                              "it will be parsed pair by pair.")
-        .SetDefault(std::vector<int>{});
-    AddAttr<std::vector<int>>("anchor_mask",
-                              "The mask index of anchors used in "
-                              "current YOLOv3 loss calculation.")
-        .SetDefault(std::vector<int>{});
-    AddAttr<int>("downsample_ratio",
-                 "The downsample ratio from network input to YOLOv3 loss "
-                 "input, so 32, 16, 8 should be set for the first, second, "
-                 "and thrid YOLOv3 loss operators.")
-        .SetDefault(32);
-    AddAttr<float>("ignore_thresh",
-                   "The ignore threshold to ignore confidence loss.")
-        .SetDefault(0.7);
-    AddAttr<bool>("use_label_smooth",
-                  "Whether to use label smooth. Default True.")
-        .SetDefault(true);
-    AddAttr<float>("scale_x_y",
-                   "Scale the center point of decoded bounding "
-                   "box. Default 1.0")
-        .SetDefault(1.);
-    AddComment(R"DOC(
-         This operator generates yolov3 loss based on given predict result and ground
-         truth boxes.
-
-         The output of previous network is in shape [N, C, H, W], while H and W
-         should be the same, H and W specify the grid size, each grid point predict
-         given number bounding boxes, this given number, which following will be represented as S,
-         is specified by the number of anchor clusters in each scale. In the second dimension(the channel
-         dimension), C should be equal to S * (class_num + 5), class_num is the object
-         category number of source dataset(such as 80 in coco dataset), so in the
-         second(channel) dimension, apart from 4 box location coordinates x, y, w, h,
-         also includes confidence score of the box and class one-hot key of each anchor box.
-
-         Assume the 4 location coordinates are :math:`t_x, t_y, t_w, t_h`, the box predictions
-         should be as follows:
-
-         $$
-         b_x = \\sigma(t_x) + c_x
-         $$
-         $$
-         b_y = \\sigma(t_y) + c_y
-         $$
-         $$
-         b_w = p_w e^{t_w}
-         $$
-         $$
-         b_h = p_h e^{t_h}
-         $$
-
-         In the equation above, :math:`c_x, c_y` is the left top corner of current grid
-         and :math:`p_w, p_h` is specified by anchors.
-
-         As for confidence score, it is the logistic regression value of IoU between
-         anchor boxes and ground truth boxes, the score of the anchor box which has
-         the max IoU should be 1, and if the anchor box has IoU bigger than ignore
-         thresh, the confidence score loss of this anchor box will be ignored.
-
-         Therefore, the yolov3 loss consists of three major parts: box location loss,
-         objectness loss and classification loss. The L1 loss is used for
-         box coordinates (w, h), sigmoid cross entropy loss is used for box
-         coordinates (x, y), objectness loss and classification loss.
-
-         Each groud truth box finds a best matching anchor box in all anchors.
-         Prediction of this anchor box will incur all three parts of losses, and
-         prediction of anchor boxes with no GT box matched will only incur objectness
-         loss.
-
-         In order to trade off box coordinate losses between big boxes and small
-         boxes, box coordinate losses will be mutiplied by scale weight, which is
-         calculated as follows.
-
-         $$
-         weight_{box} = 2.0 - t_w * t_h
-         $$
-
-         Final loss will be represented as follows.
-
-         $$
-         loss = (loss_{xy} + loss_{wh}) * weight_{box}
-              + loss_{conf} + loss_{class}
-         $$
-
-         While :attr:`use_label_smooth` is set to be :attr:`True`, the classification
-         target will be smoothed when calculating classification loss, target of
-         positive samples will be smoothed to :math:`1.0 - 1.0 / class\_num` and target of
-         negetive samples will be smoothed to :math:`1.0 / class\_num`.
-
-         While :attr:`GTScore` is given, which means the mixup score of ground truth
-         boxes, all losses incured by a ground truth box will be multiplied by its
-         mixup score.
-         )DOC");
-  }
-};
-
-class Yolov3LossOpGrad : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(ctx, "X"),
-                          platform::CPUPlace());
-  }
-};
-
-template <typename T>
-class Yolov3LossGradMaker : public framework::SingleGradOpMaker<T> {
- public:
-  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
-
- protected:
-  void Apply(GradOpPtr<T> op) const override {
-    op->SetType("yolov3_loss_grad");
-    op->SetInput("X", this->Input("X"));
-    op->SetInput("GTBox", this->Input("GTBox"));
-    op->SetInput("GTLabel", this->Input("GTLabel"));
-    op->SetInput("GTScore", this->Input("GTScore"));
-    op->SetInput(framework::GradVarName("Loss"), this->OutputGrad("Loss"));
-    op->SetInput("ObjectnessMask", this->Output("ObjectnessMask"));
-    op->SetInput("GTMatchMask", this->Output("GTMatchMask"));
-
-    op->SetAttrMap(this->Attrs());
-
-    op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
-    op->SetOutput(framework::GradVarName("GTBox"), this->EmptyInputGrad());
-    op->SetOutput(framework::GradVarName("GTLabel"), this->EmptyInputGrad());
-    op->SetOutput(framework::GradVarName("GTScore"), this->EmptyInputGrad());
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-DECLARE_INFER_SHAPE_FUNCTOR(yolov3_loss,
-                            Yolov3LossInferShapeFunctor,
-                            PD_INFER_META(phi::YoloLossInferMeta));
-DECLARE_INFER_SHAPE_FUNCTOR(yolov3_loss_grad,
-                            Yolov3LossGradInferShapeFunctor,
-                            PD_INFER_META(phi::YoloLossGradInferMeta));
-REGISTER_OPERATOR(yolov3_loss,
-                  ops::Yolov3LossOp,
-                  ops::Yolov3LossOpMaker,
-                  ops::Yolov3LossGradMaker<paddle::framework::OpDesc>,
-                  ops::Yolov3LossGradMaker<paddle::imperative::OpBase>,
-                  Yolov3LossInferShapeFunctor);
-REGISTER_OPERATOR(yolov3_loss_grad,
-                  ops::Yolov3LossOpGrad,
-                  Yolov3LossGradInferShapeFunctor);
diff --git a/paddle/fluid/operators/generator/get_expected_kernel_func.cc b/paddle/fluid/operators/generator/get_expected_kernel_func.cc
index 6085dabaed6d8..5c654d942c245 100644
--- a/paddle/fluid/operators/generator/get_expected_kernel_func.cc
+++ b/paddle/fluid/operators/generator/get_expected_kernel_func.cc
@@ -156,6 +156,13 @@ phi::KernelKey GetMatrixNmsExpectedKernelType(
                         platform::CPUPlace());
 }
 
+phi::KernelKey GetYoloLossExpectedKernelType(
+    const framework::ExecutionContext& ctx,
+    const framework::OperatorWithKernel* op_ptr) {
+  return phi::KernelKey(op_ptr->IndicateVarDataType(ctx, "X"),
+                        platform::CPUPlace());
+}
+
 phi::KernelKey GetUniqueExpectedKernelType(
     const framework::ExecutionContext& ctx,
     const framework::OperatorWithKernel* op_ptr) {
diff --git a/paddle/fluid/operators/generator/get_expected_kernel_func.h b/paddle/fluid/operators/generator/get_expected_kernel_func.h
index cbbb74e2312ed..a6fe0fc4b225a 100644
--- a/paddle/fluid/operators/generator/get_expected_kernel_func.h
+++ b/paddle/fluid/operators/generator/get_expected_kernel_func.h
@@ -48,5 +48,9 @@ phi::KernelKey GetUniqueExpectedKernelType(
     const framework::ExecutionContext& ctx,
     const framework::OperatorWithKernel* op_ptr);
 
+phi::KernelKey GetYoloLossExpectedKernelType(
+    const framework::ExecutionContext& ctx,
+    const framework::OperatorWithKernel* op_ptr);
+
 }  // namespace operators
 }  // namespace paddle
diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index 7b3b0d225af10..bde673e60b607 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -2065,6 +2065,16 @@
     func : where_grad
   no_need_buffer : x, y
 
+- backward_op : yolo_loss_grad
+  forward : yolo_loss (Tensor x, Tensor gt_box, Tensor gt_label, Tensor gt_score, int[] anchors={}, int[] anchor_mask={}, int class_num =1 , float ignore_thresh=0.7, int downsample_ratio=32, bool use_label_smooth=true, float scale_x_y=1.0) -> Tensor(loss), Tensor(objectness_mask), Tensor(gt_match_mask)
+  args : (Tensor x, Tensor gt_box, Tensor gt_label, Tensor gt_score, Tensor objectness_mask, Tensor gt_match_mask, Tensor loss_grad, int[] anchors, int[] anchor_mask, int class_num, float ignore_thresh, int downsample_ratio, bool use_label_smooth, float scale_x_y)
+  output : Tensor(x_grad), Tensor(gt_box_grad), Tensor(gt_label_grad), Tensor(gt_score_grad)
+  infer_meta :
+    func : YoloLossGradInferMeta
+  kernel :
+    func : yolo_loss_grad
+  optional : gt_score
+
 - backward_op: unpool3d_grad
   forward: unpool3d (Tensor x, Tensor indices, int[] ksize, int[] strides={1,1,1}, int[] paddings={0,0,0}, int[] output_size={0,0,0}, str data_format="NCDHW") -> Tensor(out)
   args: (Tensor x, Tensor indices, Tensor out, Tensor out_grad, int[] ksize, int[] strides, int[] paddings, int[] output_size, str data_format)
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index b844f76290c0e..9d1b2ce5b4933 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -1032,13 +1032,3 @@
     param : [out_grad]
   kernel :
     func : triu_grad
-
-- backward_op : yolo_loss_grad
-  forward : yolo_loss(Tensor x, Tensor gt_box, Tensor gt_label, Tensor gt_score, int[] anchors, int[] anchor_mask, int class_num, float ignore_thresh, int downsample_ratio, bool use_label_smooth=true, float scale_x_y=1.0) -> Tensor(loss), Tensor(objectness_mask), Tensor(gt_match_mask)
-  args : (Tensor x, Tensor gt_box, Tensor gt_label, Tensor gt_score, Tensor objectness_mask, Tensor gt_match_mask, Tensor loss_grad, int[] anchors, int[] anchor_mask, int class_num, float ignore_thresh, int downsample_ratio, bool use_label_smooth=true, float scale_x_y=1.0)
-  output : Tensor(x_grad), Tensor(gt_box_grad), Tensor(gt_label_grad), Tensor(gt_score_grad)
-  infer_meta :
-    func : YoloLossGradInferMeta
-  kernel :
-    func : yolo_loss_grad
-  optional : gt_score
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index e7b0a5ca4f330..6eff624761c89 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -1198,17 +1198,6 @@
     func : unique
     data_type : x
 
-- op : yolo_loss
-  args : (Tensor x, Tensor gt_box, Tensor gt_label, Tensor gt_score, int[] anchors, int[] anchor_mask, int class_num, float ignore_thresh, int downsample_ratio, bool use_label_smooth=true, float scale_x_y=1.0)
-  output : Tensor(loss), Tensor(objectness_mask), Tensor(gt_match_mask)
-  infer_meta :
-    func : YoloLossInferMeta
-  kernel :
-    func : yolo_loss
-    data_type : x
-  optional : gt_score
-  backward : yolo_loss_grad
-
 - op : zeros
   args : (IntArray shape, DataType dtype=DataType::FLOAT32, Place place=CPUPlace())
   output : Tensor(out)
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index 928797a3c3941..1e81fcac8eb5e 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -2511,6 +2511,16 @@
   outputs :
     {boxes : Boxes, scores : Scores}
 
+- op : yolo_loss (yolov3_loss)
+  backward: yolo_loss_grad (yolov3_loss_grad)
+  inputs :
+    {x : X,  gt_box : GTBox, gt_label : GTLabel ,gt_score : GTScore}
+  outputs :
+    {loss : Loss , objectness_mask : ObjectnessMask, gt_match_mask : GTMatchMask}
+  get_expected_kernel_type :
+    yolo_loss : GetYoloLossExpectedKernelType
+    yolo_loss_grad : GetYoloLossExpectedKernelType
+
 - op: lu
   backward: lu_grad
   inputs:
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index eaaf9f61fa7ac..e9624cbca06e6 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -2265,3 +2265,15 @@
   kernel :
     func : yolo_box
     data_type : x
+
+- op : yolo_loss
+  args : (Tensor x, Tensor gt_box, Tensor gt_label, Tensor gt_score, int[] anchors={}, int[] anchor_mask={}, int class_num =1 , float ignore_thresh=0.7, int downsample_ratio=32, bool use_label_smooth=true, float scale_x_y=1.0)
+  output : Tensor(loss), Tensor(objectness_mask), Tensor(gt_match_mask)
+  infer_meta :
+    func : YoloLossInferMeta
+  kernel :
+    func : yolo_loss
+    data_type : x
+  optional : gt_score
+  intermediate : objectness_mask, gt_match_mask
+  backward : yolo_loss_grad
diff --git a/paddle/phi/ops/compat/yolov3_loss_sig.cc b/paddle/phi/ops/compat/yolov3_loss_sig.cc
deleted file mode 100644
index f98709a9fdf33..0000000000000
--- a/paddle/phi/ops/compat/yolov3_loss_sig.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/core/compat/op_utils.h"
-
-namespace phi {
-
-KernelSignature Yolov3LossOpArgumentMapping(const ArgumentMappingContext& ctx) {
-  return KernelSignature("yolo_loss",
-                         {"X", "GTBox", "GTLabel", "GTScore"},
-                         {"anchors",
-                          "anchor_mask",
-                          "class_num",
-                          "ignore_thresh",
-                          "downsample_ratio",
-                          "use_label_smooth",
-                          "scale_x_y"},
-                         {"Loss", "ObjectnessMask", "GTMatchMask"});
-}
-
-KernelSignature Yolov3LossGradOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature(
-      "yolo_loss_grad",
-      {"X",
-       "GTBox",
-       "GTLabel",
-       "GTScore",
-       "ObjectnessMask",
-       "GTMatchMask",
-       "Loss@GRAD"},
-      {"anchors",
-       "anchor_mask",
-       "class_num",
-       "ignore_thresh",
-       "downsample_ratio",
-       "use_label_smooth",
-       "scale_x_y"},
-      {"X@GRAD", "GTBox@GRAD", "GTLabel@GRAD", "GTScore@GRAD"});
-}
-}  // namespace phi
-
-PD_REGISTER_BASE_KERNEL_NAME(yolov3_loss, yolo_loss);
-PD_REGISTER_BASE_KERNEL_NAME(yolov3_loss_grad, yolo_loss_grad);
-
-PD_REGISTER_ARG_MAPPING_FN(yolov3_loss, phi::Yolov3LossOpArgumentMapping);
-PD_REGISTER_ARG_MAPPING_FN(yolov3_loss_grad,
-                           phi::Yolov3LossGradOpArgumentMapping);
diff --git a/python/paddle/vision/ops.py b/python/paddle/vision/ops.py
index e6ff2c02dcd96..b02451f53b332 100755
--- a/python/paddle/vision/ops.py
+++ b/python/paddle/vision/ops.py
@@ -192,7 +192,7 @@ def yolo_loss(
     """
 
     if in_dygraph_mode():
-        loss, _, _ = _C_ops.yolo_loss(
+        loss = _C_ops.yolo_loss(
             x,
             gt_box,
             gt_label,

From 26fe2dcb874ff0747c0667f45c12c556ae21734b Mon Sep 17 00:00:00 2001
From: Yiqun Liu <Xreki@users.noreply.github.com>
Date: Wed, 10 May 2023 10:44:16 +0800
Subject: [PATCH 247/405] Fix the index calculation in cross_entroy_kernel.
 (#53659)

---
 paddle/phi/kernels/gpu/cross_entropy_kernel.cu | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/paddle/phi/kernels/gpu/cross_entropy_kernel.cu b/paddle/phi/kernels/gpu/cross_entropy_kernel.cu
index 0920c15d35889..38600733569aa 100644
--- a/paddle/phi/kernels/gpu/cross_entropy_kernel.cu
+++ b/paddle/phi/kernels/gpu/cross_entropy_kernel.cu
@@ -14,6 +14,8 @@ limitations under the License. */
 
 #include "paddle/phi/kernels/cross_entropy_kernel.h"
 
+#include "glog/logging.h"
+
 #ifdef __NVCC__
 #include "cub/cub.cuh"
 #endif
@@ -468,8 +470,8 @@ __global__ void VectorizedSoftmaxForward(T* loss,
   using VecT = kps::details::VectorType<T, VecSize>;
 
   // each block deal with one batch
-  logits += blockIdx.x * mid_dim;
-  softmax += blockIdx.x * mid_dim;
+  logits += static_cast<int64_t>(blockIdx.x) * static_cast<int64_t>(mid_dim);
+  softmax += static_cast<int64_t>(blockIdx.x) * static_cast<int64_t>(mid_dim);
 
   const int input_offset = ((uint64_t)logits) % ALIGN_BYTES / sizeof(T);
   const int output_offset = ((uint64_t)softmax) % ALIGN_BYTES / sizeof(T);
@@ -1165,6 +1167,8 @@ static void SoftmaxWithCrossEntropyHardLabel(const GPUContext& dev_ctx,
                                              int dim,
                                              int D,
                                              const int ignore_index) {
+  VLOG(7) << "rank=" << rank << ", axis = " << axis << ", N = " << N
+          << ", dim = " << dim << ", D = " << D;
   auto stream = dev_ctx.stream();
   constexpr int max_dim = 320;
   if (D == 1) {
@@ -1247,11 +1251,11 @@ void CrossEntropyWithSoftmaxCUDAKernel(const GPUContext& dev_ctx,
                                        int axis,
                                        DenseTensor* softmax,
                                        DenseTensor* loss) {
-  PADDLE_ENFORCE_EQ(
-      dev_ctx.GetPlace().GetType(),
-      AllocationType::GPU,
-      phi::errors::Unavailable("softmax_with_cross_entropy operator's "
-                               "CUDA kernel only runs on GPU device."));
+  VLOG(7) << "logits.shape={" << logits.dims() << "}, label.shape={"
+          << label.dims() << "}, soft_label=" << soft_label
+          << ", use_softmax=" << use_softmax
+          << ", numeric_stable_mode=" << numeric_stable_mode
+          << ", ignore_index=" << ignore_index << ", axis=" << axis;
 
   // do not with softmax op, and input is softmax
   if (!use_softmax) {

From ee1aa69cb2218764944379c4735a5e8be2db844a Mon Sep 17 00:00:00 2001
From: Winters Montagne
 <118546135+WintersMontagne10335@users.noreply.github.com>
Date: Wed, 10 May 2023 10:59:00 +0800
Subject: [PATCH 248/405] fix two minor issues (#53658)

---
 paddle/phi/kernels/funcs/dims_simplifier.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/paddle/phi/kernels/funcs/dims_simplifier.h b/paddle/phi/kernels/funcs/dims_simplifier.h
index fa0953ec46aab..35621ed0abddb 100644
--- a/paddle/phi/kernels/funcs/dims_simplifier.h
+++ b/paddle/phi/kernels/funcs/dims_simplifier.h
@@ -50,7 +50,7 @@ struct BroadcastDimsSimplifier {
         in_dims[j] = phi::vectorize<int64_t>(ins[j]->dims());
       }
     }
-    ExtendInputDimensions(N, axis);
+    ExtendInputDimensions(axis);
 
     // To Merge the dimensions of input_tensors while the consequtive
     // equal-dimensions appears. Example below :
@@ -103,7 +103,7 @@ struct BroadcastDimsSimplifier {
 
  private:
   // To compensate the lackage of input_tensors' dimension with axis.
-  void ExtendInputDimensions(int N, int axis) {
+  void ExtendInputDimensions(int axis) {
     for (auto &in_dim : in_dims) {
       if (in_dim.size() < rank) {
         DimVector extended_in_dim(rank, 1);
@@ -120,7 +120,7 @@ struct BroadcastDimsSimplifier {
                 "{%s}.",
                 in_idx,
                 out_idx,
-                out_dims[axis],
+                out_dims[out_idx],
                 in_dim[in_idx],
                 phi::make_ddim(in_dim),
                 phi::make_ddim(out_dims)));

From 7a8635d82300278916dbf91e812b87fdfc9a30c7 Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Wed, 10 May 2023 11:42:43 +0800
Subject: [PATCH 249/405] Add elementwise_heaviside tests (#53549)

---
 .../kernels/gpu/elementwise_grad_kernel.cu    |  1 +
 paddle/phi/kernels/kps/elementwise_kernel.cu  |  1 +
 .../test_elementwise_heaviside_op.py          | 54 +++++++++++++++++--
 3 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu b/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu
index efd15adc38346..c0646e71a9add 100644
--- a/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/elementwise_grad_kernel.cu
@@ -139,6 +139,7 @@ PD_REGISTER_KERNEL(heaviside_grad,
                    double,
                    int,
                    phi::dtype::float16,
+                   phi::dtype::bfloat16,
                    int64_t) {}
 
 PD_REGISTER_KERNEL(elementwise_pow_grad,
diff --git a/paddle/phi/kernels/kps/elementwise_kernel.cu b/paddle/phi/kernels/kps/elementwise_kernel.cu
index e56cb4d845911..e1eb5f8528839 100644
--- a/paddle/phi/kernels/kps/elementwise_kernel.cu
+++ b/paddle/phi/kernels/kps/elementwise_kernel.cu
@@ -177,5 +177,6 @@ PD_REGISTER_KERNEL(heaviside,
                    double,
                    int,
                    float16,
+                   bfloat16,
                    int64_t) {}
 #endif
diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_heaviside_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_heaviside_op.py
index 06439b4962c5b..a034c74fb8415 100644
--- a/python/paddle/fluid/tests/unittests/test_elementwise_heaviside_op.py
+++ b/python/paddle/fluid/tests/unittests/test_elementwise_heaviside_op.py
@@ -15,15 +15,19 @@
 import unittest
 
 import numpy as np
-from eager_op_test import OpTest
+from eager_op_test import OpTest, convert_float_to_uint16
 
 import paddle
+from paddle.fluid import core
 
 
-def Heaviside_grad(x, y, dout):
-    tmp = np.zeros(x.shape).astype("float16")
+def Heaviside_grad(x, y, dout, astype="float16", is_bfloat16=False):
+    tmp = np.zeros(x.shape).astype(astype)
     dx = np.multiply(tmp, dout)
-    dy = np.multiply(np.equal(x, 0), dout).astype("float16")
+    dy = np.multiply(np.equal(x, 0), dout).astype(astype)
+    if is_bfloat16:
+        dx = convert_float_to_uint16(dx)
+        dy = convert_float_to_uint16(dy)
     return dx, dy
 
 
@@ -161,7 +165,7 @@ def setUp(self):
         self.dtype = "int32"
 
 
-class TestHeavisideAPI_float16(OpTest):
+class TestHeavisideFP16Op(OpTest):
     def setUp(self):
         self.dtype = np.float16
         self.op_type = "elementwise_heaviside"
@@ -185,6 +189,46 @@ def test_check_grad(self):
         )
 
 
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support bfloat16",
+)
+class TestHeavisideBF16Op(OpTest):
+    def setUp(self):
+        self.dtype = np.uint16
+        self.np_dtype = np.float32
+        self.op_type = "elementwise_heaviside"
+        self.python_api = paddle.heaviside
+        self.inputs = {
+            'X': np.random.uniform(1, 2, [20, 5]).astype(self.np_dtype),
+            'Y': np.random.uniform(1, 2, [20, 5]).astype(self.np_dtype),
+        }
+        self.outputs = {'Out': np.heaviside(self.inputs['X'], self.inputs['Y'])}
+
+        self.place = core.CUDAPlace(0)
+        self.inputs['X'] = convert_float_to_uint16(self.inputs['X'])
+        self.inputs['Y'] = convert_float_to_uint16(self.inputs['Y'])
+        self.outputs['Out'] = convert_float_to_uint16(self.outputs['Out'])
+
+    def test_check_output(self):
+        self.check_output_with_place(self.place)
+
+    def test_check_grad(self):
+        self.check_grad_with_place(
+            self.place,
+            ['X', 'Y'],
+            'Out',
+            user_defined_grads=Heaviside_grad(
+                self.inputs['X'],
+                self.inputs['Y'],
+                1 / self.inputs['X'].size,
+                self.np_dtype,
+                True,
+            ),
+        )
+
+
 class TestHeavisideError(unittest.TestCase):
     def test_input(self):
         paddle.disable_static()

From 2eea311a3d5e1912e5dc61260cc138aaab331aba Mon Sep 17 00:00:00 2001
From: Yiqun Liu <Xreki@users.noreply.github.com>
Date: Wed, 10 May 2023 12:55:24 +0800
Subject: [PATCH 250/405] Add einsum to the default white_list. (#53586)

---
 python/paddle/amp/amp_lists.py         | 3 ++-
 python/paddle/static/amp/fp16_lists.py | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/python/paddle/amp/amp_lists.py b/python/paddle/amp/amp_lists.py
index 51c557b9481c6..f406456f5d352 100644
--- a/python/paddle/amp/amp_lists.py
+++ b/python/paddle/amp/amp_lists.py
@@ -16,6 +16,7 @@
 # safe and performance-critical. These ops are always converted to fp16.
 FP16_WHITE_LIST = {
     'conv2d',
+    'einsum',
     'matmul',
     'matmul_v2',
     'max_pool2d_with_index',
@@ -87,7 +88,7 @@
     'depthwise_conv2d',
 }
 
-BF16_WHITE_LIST = {'conv2d', 'matmul_v2'}
+BF16_WHITE_LIST = {'conv2d', 'einsum', 'matmul_v2'}
 BF16_BLACK_LIST = set()
 
 
diff --git a/python/paddle/static/amp/fp16_lists.py b/python/paddle/static/amp/fp16_lists.py
index 79abcc5e6c522..23da3f1c7469a 100644
--- a/python/paddle/static/amp/fp16_lists.py
+++ b/python/paddle/static/amp/fp16_lists.py
@@ -135,6 +135,7 @@ def _get_unsupported_list(dtype):
 
 white_list = {
     'conv2d',
+    'einsum',
     'matmul',
     'matmul_v2',
     'mul',

From aafaad9099ee80493b27b72f82bc8d66cc3fdf0b Mon Sep 17 00:00:00 2001
From: Qi Shao <450864116@qq.com>
Date: Wed, 10 May 2023 13:49:56 +0800
Subject: [PATCH 251/405] Revert "Optimize the implementation of the argsort
 operator. (#47738)" (#53631)

This reverts commit 9e9b705aa842542d9bcf69b9039b555cae982b80.
---
 paddle/phi/kernels/gpu/argsort_kernel.cu | 424 ++++++-----------------
 1 file changed, 110 insertions(+), 314 deletions(-)

diff --git a/paddle/phi/kernels/gpu/argsort_kernel.cu b/paddle/phi/kernels/gpu/argsort_kernel.cu
index 0ad699cdc6409..5cf3f2894a36c 100644
--- a/paddle/phi/kernels/gpu/argsort_kernel.cu
+++ b/paddle/phi/kernels/gpu/argsort_kernel.cu
@@ -65,10 +65,8 @@ struct SegmentOffsetIter {
   int num_cols_;
 };
 
-#define PADDLE_CUDA_NUM_THREADS 1024
-
 template <typename T>
-static __global__ void FillIndex(T *indices, T num_rows, T num_cols) {
+static __global__ void FillIndex(T* indices, T num_rows, T num_cols) {
   int col_id = threadIdx.x;
   int row_id = blockIdx.x;
 
@@ -81,246 +79,23 @@ static __global__ void FillIndex(T *indices, T num_rows, T num_cols) {
 
 // Sort by flag descending, True: descending. False: Ascending.
 // Default is false.
-static __global__ void FillIndexAndSegmentKernel(int2 *data,
-                                                 int numel,
-                                                 int nsort) {
-  CUDA_KERNEL_LOOP(idx, numel) {
-    auto segment = static_cast<int>(idx / nsort);
-    auto sort = static_cast<int>(idx % nsort);
-    data[idx] = int2{segment, sort};
-  }
-}
-
-#define CUB_WRAPPER(func, ctx, ...)                                            \
-  do {                                                                         \
-    size_t temp_storage_bytes = 0;                                             \
-    gpuError_t err;                                                            \
-    err = func(nullptr, temp_storage_bytes, __VA_ARGS__);                      \
-    PADDLE_ENFORCE_GPU_SUCCESS(err);                                           \
-    DenseTensor temp_storage;                                                  \
-    int64_t temp_size = temp_storage_bytes;                                    \
-    temp_storage.Resize({temp_size});                                          \
-    ctx.template Alloc<uint8_t>(&temp_storage);                                \
-    err = func(temp_storage.data<uint8_t>(), temp_storage_bytes, __VA_ARGS__); \
-    PADDLE_ENFORCE_GPU_SUCCESS(err);                                           \
-  } while (false)
-
-template <typename KT, typename VT>
-static void RadixSortPairs(const phi::GPUContext &ctx,
-                           const KT *keys_in,
-                           const VT *values_in,
-                           KT *keys_out,
-                           VT *values_out,
-                           int64_t n,
-                           bool descending = false,
-                           int64_t begin_bit = 0,
-                           int64_t end_bit = sizeof(KT) * 8) {
-  if (keys_out == nullptr) {
-    DenseTensor key_out_owner;
-    key_out_owner.Resize({n});
-    ctx.template Alloc<KT>(&key_out_owner);
-    keys_out = key_out_owner.data<KT>();
-  }
-
-  if (descending) {
-    CUB_WRAPPER(cub::DeviceRadixSort::SortPairsDescending,
-                ctx,
-                keys_in,
-                keys_out,
-                values_in,
-                values_out,
-                n,
-                begin_bit,
-                end_bit,
-                ctx.stream());
-  } else {
-    CUB_WRAPPER(cub::DeviceRadixSort::SortPairs,
-                ctx,
-                keys_in,
-                keys_out,
-                values_in,
-                values_out,
-                n,
-                begin_bit,
-                end_bit,
-                ctx.stream());
-  }
-}
-
-template <typename KT>
-static void RadixSortKeys(const phi::GPUContext &ctx,
-                          const KT *keys_in,
-                          KT *keys_out,
-                          int64_t n,
-                          bool descending,
-                          int64_t begin_bit,
-                          int64_t end_bit) {
-  if (descending) {
-    CUB_WRAPPER(cub::DeviceRadixSort::SortKeysDescending,
-                ctx,
-                keys_in,
-                keys_out,
-                n,
-                begin_bit,
-                end_bit,
-                ctx.stream());
-  } else {
-    CUB_WRAPPER(cub::DeviceRadixSort::SortKeys,
-                ctx,
-                keys_in,
-                keys_out,
-                n,
-                begin_bit,
-                end_bit,
-                ctx.stream());
-  }
-}
-
-template <typename T>
-static __global__ void SortPostprocessKernel(const T *in,
-                                             const int2 *i_s_ptr,
-                                             T *out,
-                                             int64_t *index,
-                                             int nsegments,
-                                             int nsort) {
-  CUDA_KERNEL_LOOP(i, nsegments * nsort) {
-    int segment = i / nsort;  // segment_id
-    int j = i % nsort;
-
-    int offset = segment * nsort;
-    const T *in_ = in + offset;
-    T *out_ = out + offset;
-    int64_t *index_ = index + offset;
-    const int2 *i_s_ptr_ = i_s_ptr + offset;
-
-    int idx = i_s_ptr_[j].y;
-    index_[j] = idx;
-    out_[j] = in_[idx];
-  }
-}
-
-template <typename T>
-inline void SegmentedSortPairsByFullSort(const phi::GPUContext &ctx,
-                                         const T *const self_ptr,
-                                         T *const values_ptr,
-                                         int64_t *const indices_ptr,
-                                         const int64_t nsegments,
-                                         const int64_t nsort,
-                                         const int64_t n,
-                                         const bool descending) {
-  int64_t segment_bits = std::max<int64_t>(
-      1L, static_cast<int64_t>(std::ceil(std::log2(nsegments))));
-
-  const auto numel = nsort * nsegments;
-
-  DenseTensor indices_and_segment;
-  int64_t indices_and_segment_size = numel;
-  indices_and_segment.Resize({indices_and_segment_size * 2});
-  ctx.template Alloc<int64_t>(&indices_and_segment);
-  auto i_s_ptr_base = indices_and_segment.data<int64_t>();
-  auto i_s_ptr = reinterpret_cast<int2 *>(i_s_ptr_base);
-
-  dim3 block = PADDLE_CUDA_NUM_THREADS;
-  auto block_num = (numel - 1) / PADDLE_CUDA_NUM_THREADS + 1;
-  dim3 grid = static_cast<int>(block_num);
-
-  auto cu_stream = ctx.stream();
-
-  FillIndexAndSegmentKernel<<<grid, block, 0, cu_stream>>>(
-      i_s_ptr, numel, nsort);
-
-  DenseTensor indices_and_segment2;
-  int64_t indices_and_segment2_size = numel;
-  indices_and_segment2.Resize({indices_and_segment2_size * 2});
-  ctx.template Alloc<int64_t>(&indices_and_segment2);
-  auto i_s_ptr2_base = indices_and_segment2.data<int64_t>();
-  auto i_s_ptr2 = reinterpret_cast<int2 *>(i_s_ptr2_base);
-
-  RadixSortPairs<T, int2>(
-      ctx, self_ptr, i_s_ptr, nullptr, i_s_ptr2, n, descending);
-
-  RadixSortKeys<int64_t>(ctx,
-                         reinterpret_cast<int64_t *>(i_s_ptr2),
-                         reinterpret_cast<int64_t *>(i_s_ptr),
-                         n,
-                         false,
-                         0,
-                         segment_bits);
-
-  SortPostprocessKernel<<<grid, block, 0, cu_stream>>>(
-      self_ptr, i_s_ptr, values_ptr, indices_ptr, nsegments, nsort);
-}
-
-// The method is called when # of the rows of the input is less than or equal to
-// 4
-template <typename T, typename IndexType>
-void ArgFullSortForTinyRows(const phi::GPUContext &ctx,
-                            const DenseTensor *input,
-                            DenseTensor *output,
-                            DenseTensor *indices,
-                            const IndexType num_rows,
-                            const IndexType num_cols,
-                            const bool descending) {
-  auto gpu_stream = ctx.stream();
-  size_t temp_storage_bytes = -1;
-
-  IndexType numel = num_rows * num_cols;
-  if (numel == 0) {
-    return;
-  }
-
-  IndexType numel_or_intmax =
-      std::min(numel, static_cast<int64_t>(std::numeric_limits<int>::max()));
-  IndexType nsort = num_cols;
-  IndexType nbatch = (numel_or_intmax / nsort) * nsort;
-
-  T *sorted_out_ptr;
-  IndexType *sorted_indices_ptr;
-  const T *input_data = input->data<T>();
-  T *out = ctx.template Alloc<T>(output);
-  IndexType *ind = ctx.template Alloc<IndexType>(indices);
-  sorted_out_ptr = out;
-  sorted_indices_ptr = ind;
-
-  int64_t remaining = numel;
-
-  while (remaining > 0) {
-    int64_t n = std::min(remaining, nbatch);
-    IndexType nsegments = n / nsort;
-
-    SegmentedSortPairsByFullSort(ctx,
-                                 input_data,
-                                 sorted_out_ptr,
-                                 sorted_indices_ptr,
-                                 nsegments,
-                                 nsort,
-                                 n,
-                                 descending);
-
-    remaining -= n;
-    input_data += n;
-    sorted_out_ptr += n;
-    sorted_indices_ptr += n;
-  }
-}
-
-template <typename T, typename IndexType>
-void ArgFullSort(const phi::GPUContext &ctx,
-                 const DenseTensor *input,
-                 DenseTensor *output,
-                 DenseTensor *indices,
-                 const IndexType num_rows,
-                 const IndexType num_cols,
+template <typename T, typename IndType>
+void ArgFullSort(const phi::GPUContext& ctx,
+                 const DenseTensor* input,
+                 DenseTensor* output,
+                 DenseTensor* indices,
+                 const IndType num_rows,
+                 const IndType num_cols,
                  const bool descending) {
   auto cu_stream = ctx.stream();
   DenseTensor input_indices;
-  const std::vector<IndexType> dims = {num_rows, num_cols};
+  const std::vector<IndType> dims = {num_rows, num_cols};
   auto dim = phi::make_ddim(dims);
   input_indices.Resize(dim);
-  ctx.template Alloc<IndexType>(&input_indices);
+  ctx.template Alloc<IndType>(&input_indices);
   size_t temp_storage_bytes = -1;
 
-  auto ComputeBlockSize = [](IndexType col) {
+  auto ComputeBlockSize = [](IndType col) {
     if (col > 512)
       return 1024;
     else if (col > 256 && col <= 512)
@@ -339,71 +114,112 @@ void ArgFullSort(const phi::GPUContext &ctx,
   int grid_size = num_rows < maxGridDimX ? num_rows : maxGridDimX;
   // Init a index array
   FillIndex<<<grid_size, block_size, 0, cu_stream>>>(
-      input_indices.data<IndexType>(), num_rows, num_cols);
+      input_indices.data<IndType>(), num_rows, num_cols);
 
-  T *sorted_out_ptr;
-  IndexType *sorted_indices_ptr;
-  const T *inp = input->data<T>();
-  T *out = ctx.template Alloc<T>(output);
-  IndexType *ind = ctx.template Alloc<IndexType>(indices);
+  T* sorted_out_ptr;
+  IndType* sorted_indices_ptr;
+  const T* inp = input->data<T>();
+  T* out = ctx.template Alloc<T>(output);
+  IndType* ind = ctx.template Alloc<IndType>(indices);
   sorted_out_ptr = out;
   sorted_indices_ptr = ind;
 
   // create iter for counting input
-  cub::CountingInputIterator<IndexType> counting_iter(0);
+  cub::CountingInputIterator<IndType> counting_iter(0);
   // segment_offset is used for move to next row
-  cub::TransformInputIterator<IndexType,
+  cub::TransformInputIterator<IndType,
                               SegmentOffsetIter,
-                              cub::CountingInputIterator<IndexType>>
+                              cub::CountingInputIterator<IndType>>
       segment_offsets_t(counting_iter, SegmentOffsetIter(num_cols));
 
   gpuError_t err;
   if (descending) {
-    CUB_WRAPPER(cub::DeviceSegmentedRadixSort::SortPairsDescending,
-                ctx,
-                inp,
-                sorted_out_ptr,
-                input_indices.data<IndexType>(),
-                sorted_indices_ptr,
-                num_cols * num_rows,
-                num_rows,
-                segment_offsets_t,
-                segment_offsets_t + 1,
-                0,
-                sizeof(T) * 8,
-                ctx.stream());
+    err = cub::DeviceSegmentedRadixSort::SortPairsDescending(
+        nullptr,
+        temp_storage_bytes,
+        inp,
+        sorted_out_ptr,
+        input_indices.data<IndType>(),
+        sorted_indices_ptr,
+        num_cols * num_rows,
+        num_rows,
+        segment_offsets_t,
+        segment_offsets_t + 1,
+        0,
+        sizeof(T) * 8,
+        cu_stream);
   } else {
-    CUB_WRAPPER(cub::DeviceSegmentedRadixSort::SortPairs,
-                ctx,
-                inp,
-                sorted_out_ptr,
-                input_indices.data<IndexType>(),
-                sorted_indices_ptr,
-                num_cols * num_rows,
-                num_rows,
-                segment_offsets_t,
-                segment_offsets_t + 1,
-                0,
-                sizeof(T) * 8,
-                ctx.stream());
+    err =
+        cub::DeviceSegmentedRadixSort::SortPairs(nullptr,
+                                                 temp_storage_bytes,
+                                                 inp,
+                                                 sorted_out_ptr,
+                                                 input_indices.data<IndType>(),
+                                                 sorted_indices_ptr,
+                                                 num_cols * num_rows,
+                                                 num_rows,
+                                                 segment_offsets_t,
+                                                 segment_offsets_t + 1,
+                                                 0,
+                                                 sizeof(T) * 8,
+                                                 cu_stream);
   }
+  PADDLE_ENFORCE_GPU_SUCCESS(err);
+
+  DenseTensor temp_storage;
+  int64_t temp_size = temp_storage_bytes;
+  temp_storage.Resize({temp_size});
+  ctx.template Alloc<uint8_t>(&temp_storage);
+
+  if (descending) {
+    err = cub::DeviceSegmentedRadixSort::SortPairsDescending(
+        temp_storage.data<uint8_t>(),
+        temp_storage_bytes,
+        inp,
+        sorted_out_ptr,
+        input_indices.data<IndType>(),
+        sorted_indices_ptr,
+        num_cols * num_rows,
+        num_rows,
+        segment_offsets_t,
+        segment_offsets_t + 1,
+        0,
+        sizeof(T) * 8,
+        cu_stream);
+  } else {
+    err =
+        cub::DeviceSegmentedRadixSort::SortPairs(temp_storage.data<uint8_t>(),
+                                                 temp_storage_bytes,
+                                                 inp,
+                                                 sorted_out_ptr,
+                                                 input_indices.data<IndType>(),
+                                                 sorted_indices_ptr,
+                                                 num_cols * num_rows,
+                                                 num_rows,
+                                                 segment_offsets_t,
+                                                 segment_offsets_t + 1,
+                                                 0,
+                                                 sizeof(T) * 8,
+                                                 cu_stream);
+  }
+
+  PADDLE_ENFORCE_GPU_SUCCESS(err);
 }
 
 template <typename T, typename Context>
-void ArgsortKernel(const Context &dev_ctx,
-                   const DenseTensor &input,
+void ArgsortKernel(const Context& dev_ctx,
+                   const DenseTensor& input,
                    int axis,
                    bool descending,
-                   DenseTensor *output,
-                   DenseTensor *indices) {
+                   DenseTensor* output,
+                   DenseTensor* indices) {
   auto in_dims = input.dims();
   auto rank = in_dims.size();
   axis = (axis < 0) ? (in_dims.size() + axis) : axis;
-
-  const T *in_data = input.data<T>();
+  const T* in_data = input.data<T>();
   auto size = input.numel();
-  T *out_data = dev_ctx.template Alloc<T>(output);
-  int64_t *ids_data = dev_ctx.template Alloc<int64_t>(indices);
+  T* out_data = dev_ctx.template Alloc<T>(output);
+  int64_t* ids_data = dev_ctx.template Alloc<int64_t>(indices);
 
   if (rank == 0) {
     phi::Copy<Context>(dev_ctx, input, dev_ctx.GetPlace(), false, output);
@@ -431,23 +247,13 @@ void ArgsortKernel(const Context &dev_ctx,
     const int64_t input_height =
         phi::product(phi::slice_ddim(in_dims, 0, in_dims.size() - 1));
     const int64_t input_width = in_dims[in_dims.size() - 1];
-    if (input_height <= 4) {
-      ArgFullSortForTinyRows<T, int64_t>(dev_ctx,
-                                         &input,
-                                         output,
-                                         indices,
-                                         input_height,
-                                         input_width,
-                                         descending);
-    } else {
-      ArgFullSort<T, int64_t>(dev_ctx,
-                              &input,
-                              output,
-                              indices,
-                              input_height,
-                              input_width,
-                              descending);
-    }
+    ArgFullSort<T, int64_t>(dev_ctx,
+                            &input,
+                            output,
+                            indices,
+                            input_height,
+                            input_width,
+                            descending);
   } else {
     // if not full sort, do transpose first
     std::vector<int> trans;
@@ -466,7 +272,7 @@ void ArgsortKernel(const Context &dev_ctx,
 
     DenseTensor trans_inp;
     trans_inp.Resize(trans_dims);
-    T *trans_inp_data = dev_ctx.template Alloc<T>(&trans_inp);
+    T* trans_inp_data = dev_ctx.template Alloc<T>(&trans_inp);
     // Do transpose
     TransposeKernel<T, Context>(dev_ctx, input, trans, &trans_inp);
 
@@ -484,23 +290,13 @@ void ArgsortKernel(const Context &dev_ctx,
     dev_ctx.template Alloc<int64_t>(&tmp_indices);
     dev_ctx.template Alloc<int64_t>(indices);
 
-    if (input_height <= 4) {
-      ArgFullSortForTinyRows<T, int64_t>(dev_ctx,
-                                         &trans_inp,
-                                         &tmp_out,
-                                         &tmp_indices,
-                                         input_height,
-                                         input_width,
-                                         descending);
-    } else {
-      ArgFullSort<T, int64_t>(dev_ctx,
-                              &trans_inp,
-                              &tmp_out,
-                              &tmp_indices,
-                              input_height,
-                              input_width,
-                              descending);
-    }
+    ArgFullSort<T, int64_t>(dev_ctx,
+                            &trans_inp,
+                            &tmp_out,
+                            &tmp_indices,
+                            input_height,
+                            input_width,
+                            descending);
 
     TransposeKernel<int64_t, Context>(dev_ctx, tmp_indices, trans, indices);
     // transpose back

From e077678c5d7cb5c5c9de6e641dd7d8b2cb95515e Mon Sep 17 00:00:00 2001
From: ronnywang <ronny1996@163.com>
Date: Wed, 10 May 2023 14:09:20 +0800
Subject: [PATCH 252/405] [CustomDevice] fix reducer when input on cpu (#53662)

---
 paddle/fluid/distributed/collective/reducer.cc | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/paddle/fluid/distributed/collective/reducer.cc b/paddle/fluid/distributed/collective/reducer.cc
index 260da509a17cb..b06ed3578df72 100644
--- a/paddle/fluid/distributed/collective/reducer.cc
+++ b/paddle/fluid/distributed/collective/reducer.cc
@@ -215,7 +215,11 @@ struct ConcatTensorsForAllReduce<platform::CustomDeviceContext, T> {
       const uint8_t *in_data =
           reinterpret_cast<const uint8_t *>(tensor.data<T>());
       auto sz = tensor.numel() * sizeof(T);
-      device->MemoryCopyD2D(out_data + offset, in_data, sz, &stream);
+      if (tensor.place().GetType() == phi::AllocationType::CPU) {
+        device->MemoryCopyH2D(out_data + offset, in_data, sz, &stream);
+      } else {
+        device->MemoryCopyD2D(out_data + offset, in_data, sz, &stream);
+      }
       offset += sz;
     }
   }
@@ -237,7 +241,11 @@ struct SplitTensorsForAllReduce<platform::CustomDeviceContext, T> {
     for (auto &tensor : *p_dense_tensors) {
       uint8_t *out_data = reinterpret_cast<uint8_t *>(tensor.data<T>());
       auto sz = tensor.numel() * sizeof(T);
-      device->MemoryCopyD2D(out_data, in_data + offset, sz, &stream);
+      if (tensor.place().GetType() == phi::AllocationType::CPU) {
+        device->MemoryCopyD2H(out_data, in_data + offset, sz, &stream);
+      } else {
+        device->MemoryCopyD2D(out_data, in_data + offset, sz, &stream);
+      }
       offset += sz;
     }
   }

From 65e57a7df00dc9429df92dcad4b4e7a1be46e3e8 Mon Sep 17 00:00:00 2001
From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com>
Date: Wed, 10 May 2023 14:22:54 +0800
Subject: [PATCH 253/405] remove some [-Wunused-parameter] warning and
 WITH_DISTRIBUT flags (#53650)

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop
---
 paddle/fluid/framework/CMakeLists.txt         |  8 +---
 .../fluid/framework/details/nccl_op_handle.h  |  4 +-
 paddle/fluid/operators/save_combine_op.h      |  4 +-
 paddle/phi/common/cpstring_impl.h             |  8 +++-
 paddle/phi/kernels/cpu/p_recv_kernel.cc       | 20 ++++----
 paddle/phi/kernels/cpu/p_send_kernel.cc       | 18 +++----
 paddle/phi/kernels/cpu/rnn_grad_kernel.cc     | 48 +++++++++----------
 paddle/phi/kernels/cpu/rnn_kernel.cc          | 40 ++++++++--------
 .../kernels/cpu/send_ue_recv_grad_kernel.cc   |  8 ++--
 paddle/phi/kernels/cpu/send_uv_grad_kernel.cc |  2 +-
 .../kernels/cpu/unique_consecutive_functor.h  |  2 +-
 .../phi/kernels/cpu/viterbi_decode_kernel.cc  |  2 +-
 paddle/phi/kernels/funcs/activation_functor.h |  4 +-
 paddle/phi/kernels/funcs/compound_functors.h  |  2 +-
 paddle/phi/kernels/funcs/cpu_vec.h            |  2 +-
 .../phi/kernels/funcs/detail/strided_memcpy.h | 10 ++--
 paddle/phi/kernels/funcs/jit/more/mkl/mkl.cc  |  8 ++--
 paddle/phi/kernels/funcs/scatter.h            |  4 +-
 paddle/phi/kernels/sparse/cpu/conv.h          |  2 +-
 19 files changed, 98 insertions(+), 98 deletions(-)

diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt
index b95d996b88a94..3cab75f0971dd 100755
--- a/paddle/fluid/framework/CMakeLists.txt
+++ b/paddle/fluid/framework/CMakeLists.txt
@@ -771,9 +771,7 @@ if(WITH_DISTRIBUTE)
            heter_service_proto
            fleet_executor
            ${BRPC_DEP})
-    set(DISTRIBUTE_COMPILE_FLAGS
-        "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=parentheses"
-    )
+    set(DISTRIBUTE_COMPILE_FLAGS "")
     if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
       set(DISTRIBUTE_COMPILE_FLAGS "${DISTRIBUTE_COMPILE_FLAGS} -faligned-new")
     endif()
@@ -855,9 +853,7 @@ if(WITH_DISTRIBUTE)
            brpc
            fleet_executor
            flags)
-    set(DISTRIBUTE_COMPILE_FLAGS
-        "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=parentheses"
-    )
+    set(DISTRIBUTE_COMPILE_FLAGS "")
     if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
       set(DISTRIBUTE_COMPILE_FLAGS "${DISTRIBUTE_COMPILE_FLAGS} -faligned-new")
     endif()
diff --git a/paddle/fluid/framework/details/nccl_op_handle.h b/paddle/fluid/framework/details/nccl_op_handle.h
index db7fc45b246c6..e4472e8d989dd 100644
--- a/paddle/fluid/framework/details/nccl_op_handle.h
+++ b/paddle/fluid/framework/details/nccl_op_handle.h
@@ -225,7 +225,7 @@ class NCCLOpHandleBase : public OpHandleBase {
                    void* recvbuff,
                    size_t count,
                    ncclDataType_t datatype,
-                   ncclRedOp_t op) {
+                   ncclRedOp_t op UNUSED) {
     auto nccl_ctxs = nccl_ctxs_->GetHierarchicalInterCtx(run_order_);
     int dev_id = place.device;
     auto& nccl_ctx = nccl_ctxs->at(dev_id);
@@ -297,7 +297,7 @@ class NCCLOpHandleBase : public OpHandleBase {
                       void* sendbuff,
                       size_t count,
                       ncclDataType_t datatype,
-                      ncclRedOp_t op) {
+                      ncclRedOp_t op UNUSED) {
     auto nccl_ctxs = nccl_ctxs_->GetHierarchicalInterCtx(run_order_);
     int dev_id = place.device;
     auto& nccl_ctx = nccl_ctxs->at(dev_id);
diff --git a/paddle/fluid/operators/save_combine_op.h b/paddle/fluid/operators/save_combine_op.h
index 817e18791f9a3..1888ce5b57493 100644
--- a/paddle/fluid/operators/save_combine_op.h
+++ b/paddle/fluid/operators/save_combine_op.h
@@ -121,11 +121,11 @@ void SaveCombineTensorKernel(const Context& dev_ctx,
 
 template <typename T, typename Context>
 void SaveCombineVocabKernel(
-    const Context& dev_ctx,
+    const Context& dev_ctx UNUSED,
     const std::vector<const phi::ExtendedTensor*>& inputs,
     const std::string& file_path,
     bool overwrite,
-    bool save_as_fp16,
+    bool save_as_fp16 UNUSED,
     bool save_to_memory,
     phi::ExtendedTensor* out) {
   std::string* y = nullptr;
diff --git a/paddle/phi/common/cpstring_impl.h b/paddle/phi/common/cpstring_impl.h
index 99a04e7ce4924..6783799026d44 100644
--- a/paddle/phi/common/cpstring_impl.h
+++ b/paddle/phi/common/cpstring_impl.h
@@ -24,6 +24,8 @@ limitations under the License. */
 #include <stdlib.h>
 #include <string.h>
 
+#include "paddle/phi/core/macros.h"
+
 #if (defined(__NVCC__) || defined(__HIPCC__))
 #define HOSTDEVICE __host__ __device__
 #define DEVICE __device__
@@ -181,7 +183,9 @@ HOSTDEVICE static inline size_t PD_PString_ToInternalSizeT(
 /*
  * Need to implement in other source file.
  */
-HOSTDEVICE static inline void PD_Free(void *ptr, size_t size) { free(ptr); }
+HOSTDEVICE static inline void PD_Free(void *ptr, size_t size UNUSED) {
+  free(ptr);
+}
 
 HOSTDEVICE static inline void *PD_Memset(void *src, int ch, size_t size) {
   char *dst = (char *)src;  // NOLINT
@@ -203,7 +207,7 @@ HOSTDEVICE static inline void *PD_Memcpy(void *dst,
 HOSTDEVICE static inline void *PD_Malloc(size_t size) { return malloc(size); }
 
 HOSTDEVICE static inline void *PD_Realloc(void *ptr,
-                                          size_t old_size,
+                                          size_t old_size UNUSED,
                                           size_t new_size) {
 #if (defined(__NVCC__) || defined(__HIPCC__))
   if (old_size >= new_size) {
diff --git a/paddle/phi/kernels/cpu/p_recv_kernel.cc b/paddle/phi/kernels/cpu/p_recv_kernel.cc
index 72d312e71ca37..10526e6935e1e 100644
--- a/paddle/phi/kernels/cpu/p_recv_kernel.cc
+++ b/paddle/phi/kernels/cpu/p_recv_kernel.cc
@@ -24,20 +24,20 @@
 namespace phi {
 
 template <typename T, typename Context>
-void PRecvKernel(const Context& dev_ctx,
-                 int peer,
-                 DataType dtype,
-                 bool dynamic_shape,
-                 DenseTensor* out) {
+void PRecvKernel(const Context& dev_ctx UNUSED,
+                 int peer UNUSED,
+                 DataType dtype UNUSED,
+                 bool dynamic_shape UNUSED,
+                 DenseTensor* out UNUSED) {
   PADDLE_THROW(errors::Unavailable("Do not support recv for cpu kernel now."));
 }
 
 template <typename T, typename Context>
-void PRecvArrayKernel(const Context& dev_ctx,
-                      int peer,
-                      DataType dtype,
-                      const std::vector<int>& out_shape,
-                      TensorArray* out_array) {
+void PRecvArrayKernel(const Context& dev_ctx UNUSED,
+                      int peer UNUSED,
+                      DataType dtype UNUSED,
+                      const std::vector<int>& out_shape UNUSED,
+                      TensorArray* out_array UNUSED) {
   PADDLE_THROW(
       errors::Unavailable("Do not support recv array for cpu kernel now."));
 }
diff --git a/paddle/phi/kernels/cpu/p_send_kernel.cc b/paddle/phi/kernels/cpu/p_send_kernel.cc
index 60f9d458ce6e3..a786de7ecaf3b 100644
--- a/paddle/phi/kernels/cpu/p_send_kernel.cc
+++ b/paddle/phi/kernels/cpu/p_send_kernel.cc
@@ -24,19 +24,19 @@
 namespace phi {
 
 template <typename T, typename Context>
-void PSendKernel(const Context& dev_ctx,
-                 const DenseTensor& x,
-                 int peer,
-                 bool dynamic_shape) {
+void PSendKernel(const Context& dev_ctx UNUSED,
+                 const DenseTensor& x UNUSED,
+                 int peer UNUSED,
+                 bool dynamic_shape UNUSED) {
   PADDLE_THROW(errors::Unavailable("Do not support send for cpu kernel now."));
 }
 
 template <typename T, typename Context>
-void PSendArrayKernel(const Context& dev_ctx,
-                      const TensorArray& x,
-                      int peer,
-                      bool dynamic_shape,
-                      DenseTensor* out) {
+void PSendArrayKernel(const Context& dev_ctx UNUSED,
+                      const TensorArray& x UNUSED,
+                      int peer UNUSED,
+                      bool dynamic_shape UNUSED,
+                      DenseTensor* out UNUSED) {
   PADDLE_THROW(
       errors::Unavailable("Do not support send array for cpu kernel now."));
 }
diff --git a/paddle/phi/kernels/cpu/rnn_grad_kernel.cc b/paddle/phi/kernels/cpu/rnn_grad_kernel.cc
index b4ec6652eb975..cb479cb755cfa 100644
--- a/paddle/phi/kernels/cpu/rnn_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/rnn_grad_kernel.cc
@@ -553,28 +553,28 @@ struct GradLayer {
   }
 
   virtual void operator()(
-      const CPUContext& dev_ctx,
-      const DenseTensor* input,
-      const DenseTensor* output,
-      const std::vector<DenseTensor>& init_h_unbind,
-      const std::vector<DenseTensor>& init_c_unbind,
-      const std::vector<DenseTensor>& last_h_grad_unbind,
-      const std::vector<DenseTensor>& last_c_grad_unbind,
-      const std::vector<DenseTensor>& gate_tensor_unbind,
-      const std::vector<DenseTensor>& state_tensor_unbind,
-      const std::vector<DenseTensor>& act_state_tensor_unbind,
-      const DenseTensor* output_grad,
-      const std::vector<std::vector<DenseTensor>>& parameter_lists,
-      const DenseTensor* sequence_length,
-      DenseTensor* input_grad,
-      std::vector<DenseTensor>* init_h_grad_unbind,
-      std::vector<DenseTensor>* init_c_grad_unbind,
-      const std::vector<std::vector<DenseTensor>>& weight_list_grad,
-      int layer_idx,
-      bool is_bidirec,
-      int hidden_size,
-      const std::string& mode,
-      int gate_num) {}
+      const CPUContext& dev_ctx UNUSED,
+      const DenseTensor* input UNUSED,
+      const DenseTensor* output UNUSED,
+      const std::vector<DenseTensor>& init_h_unbind UNUSED,
+      const std::vector<DenseTensor>& init_c_unbind UNUSED,
+      const std::vector<DenseTensor>& last_h_grad_unbind UNUSED,
+      const std::vector<DenseTensor>& last_c_grad_unbind UNUSED,
+      const std::vector<DenseTensor>& gate_tensor_unbind UNUSED,
+      const std::vector<DenseTensor>& state_tensor_unbind UNUSED,
+      const std::vector<DenseTensor>& act_state_tensor_unbind UNUSED,
+      const DenseTensor* output_grad UNUSED,
+      const std::vector<std::vector<DenseTensor>>& parameter_lists UNUSED,
+      const DenseTensor* sequence_length UNUSED,
+      DenseTensor* input_grad UNUSED,
+      std::vector<DenseTensor>* init_h_grad_unbind UNUSED,
+      std::vector<DenseTensor>* init_c_grad_unbind UNUSED,
+      const std::vector<std::vector<DenseTensor>>& weight_list_grad UNUSED,
+      int layer_idx UNUSED,
+      bool is_bidirec UNUSED,
+      int hidden_size UNUSED,
+      const std::string& mode UNUSED,
+      int gate_num UNUSED) {}
 
   void preprocess(const CPUContext& dev_ctx,
                   const DenseTensor* grad_output,
@@ -978,11 +978,11 @@ void RnnGradFunc(const CPUContext& dev_ctx,
                  const std::vector<const DenseTensor*>& state_grad,
                  float dropout_prob,
                  bool is_bidirec,
-                 int input_size,
+                 int input_size UNUSED,
                  int hidden_size,
                  int num_layers,
                  const std::string& mode,
-                 int seed,
+                 int seed UNUSED,
                  bool is_test,
                  int gate_num,
                  DenseTensor* x_grad,
diff --git a/paddle/phi/kernels/cpu/rnn_kernel.cc b/paddle/phi/kernels/cpu/rnn_kernel.cc
index ae3a4402f5138..de1caf16f621e 100644
--- a/paddle/phi/kernels/cpu/rnn_kernel.cc
+++ b/paddle/phi/kernels/cpu/rnn_kernel.cc
@@ -292,22 +292,22 @@ struct Layer {
     }
   }
 
-  virtual void operator()(const CPUContext& dev_ctx,
-                          const DenseTensor* input,
-                          const std::vector<DenseTensor>& vec,
-                          const std::vector<DenseTensor>& init_h,
-                          const std::vector<DenseTensor>& init_c,
-                          const DenseTensor* sequence_length,
-                          std::vector<DenseTensor> last_h,
-                          std::vector<DenseTensor> last_c,
-                          DenseTensor* output,
-                          const int& layer_idx,
-                          const int& gate_num,
-                          DenseTensor* gate_value,
-                          DenseTensor* cell_value,
-                          DenseTensor* cell_act_value,
-                          const std::string& mode,
-                          bool is_test) {}
+  virtual void operator()(const CPUContext& dev_ctx UNUSED,
+                          const DenseTensor* input UNUSED,
+                          const std::vector<DenseTensor>& vec UNUSED,
+                          const std::vector<DenseTensor>& init_h UNUSED,
+                          const std::vector<DenseTensor>& init_c UNUSED,
+                          const DenseTensor* sequence_length UNUSED,
+                          std::vector<DenseTensor> last_h UNUSED,
+                          std::vector<DenseTensor> last_c UNUSED,
+                          DenseTensor* output UNUSED,
+                          const int& layer_idx UNUSED,
+                          const int& gate_num UNUSED,
+                          DenseTensor* gate_value UNUSED,
+                          DenseTensor* cell_value UNUSED,
+                          DenseTensor* cell_act_value UNUSED,
+                          const std::string& mode UNUSED,
+                          bool is_test UNUSED) {}
 
   void RunTestIter(const CPUContext& dev_ctx,
                    const DenseTensor* input,
@@ -320,8 +320,8 @@ struct Layer {
                    DenseTensor* output,
                    int layer_idx,
                    DenseTensor* gate_value,
-                   DenseTensor* cell_value,
-                   DenseTensor* cell_act_value,
+                   DenseTensor* cell_value UNUSED,
+                   DenseTensor* cell_act_value UNUSED,
                    bool is_bidirect,
                    int offset,
                    const std::string& mode) {
@@ -701,7 +701,7 @@ struct SingleLayer : public Layer<T, CellType> {
                   std::vector<DenseTensor> last_c,
                   DenseTensor* output,
                   const int& layer_idx,
-                  const int& gate_num,
+                  const int& gate_num UNUSED,
                   DenseTensor* gate_value,
                   DenseTensor* cell_value,
                   DenseTensor* cell_act_value,
@@ -740,7 +740,7 @@ struct BidirLayer : public Layer<T, CellType> {
                   std::vector<DenseTensor> last_c,
                   DenseTensor* output,
                   const int& layer_idx,
-                  const int& gate_num,
+                  const int& gate_num UNUSED,
                   DenseTensor* gate_value,
                   DenseTensor* cell_value,
                   DenseTensor* cell_act_value,
diff --git a/paddle/phi/kernels/cpu/send_ue_recv_grad_kernel.cc b/paddle/phi/kernels/cpu/send_ue_recv_grad_kernel.cc
index 659c6502803f0..0d06245e068fc 100644
--- a/paddle/phi/kernels/cpu/send_ue_recv_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/send_ue_recv_grad_kernel.cc
@@ -32,7 +32,7 @@ namespace phi {
 template <typename Context, typename T, typename IndexT>
 void CalculateXGrad(const Context& ctx,
                     const T* out_grad,
-                    const T* x_data,
+                    const T* x_data UNUSED,
                     const T* e_data,
                     const phi::DDim& out_grad_dims,
                     const phi::DDim& x_dims,
@@ -46,7 +46,7 @@ void CalculateXGrad(const Context& ctx,
                     const DenseTensor& out_grad_tensor,
                     DenseTensor* x_grad_tensor,
                     const DenseTensor* dst_count = nullptr,
-                    const DenseTensor* out = nullptr) {
+                    const DenseTensor* out UNUSED = nullptr) {
   std::vector<int64_t> reduce_idx;
   bool reduce = ReduceGrad(out_grad_dims, x_dims, reduce_idx);
 
@@ -232,7 +232,7 @@ void CalculateXGrad(const Context& ctx,
 template <typename T, typename IndexT>
 void CalculateEGrad(const T* out_grad_data,
                     const T* x_data,
-                    const T* e_data,
+                    const T* e_data UNUSED,
                     const phi::DDim& x_dims,
                     const phi::DDim& e_dims,
                     const IndexT* s_index,
@@ -308,7 +308,7 @@ void CalculateXEGradForMinMax(const T* out_grad,
                               const IndexT* s_index,
                               const IndexT* d_index,
                               const std::string& message_op,
-                              const std::string& reduce_op,
+                              const std::string& reduce_op UNUSED,
                               int64_t index_size,
                               T* x_grad,
                               T* e_grad,
diff --git a/paddle/phi/kernels/cpu/send_uv_grad_kernel.cc b/paddle/phi/kernels/cpu/send_uv_grad_kernel.cc
index 8e5873ac01769..c04bdaec0177e 100644
--- a/paddle/phi/kernels/cpu/send_uv_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/send_uv_grad_kernel.cc
@@ -35,7 +35,7 @@ void CalculateGrad(const Context& ctx,
                    int64_t index_size,
                    int64_t slice_size,
                    T* x_grad,
-                   const DenseTensor& out_grad_tensor,
+                   const DenseTensor& out_grad_tensor UNUSED,
                    const DenseTensor& y) {
   std::vector<int64_t> reduce_idx;
   bool reduce = ReduceGrad(out_grad_dims, x_grad_dims, reduce_idx);
diff --git a/paddle/phi/kernels/cpu/unique_consecutive_functor.h b/paddle/phi/kernels/cpu/unique_consecutive_functor.h
index 73d196bbb98d9..2daee69eed92d 100644
--- a/paddle/phi/kernels/cpu/unique_consecutive_functor.h
+++ b/paddle/phi/kernels/cpu/unique_consecutive_functor.h
@@ -111,7 +111,7 @@ struct UniqueConsecutiveFlattenedTensorFunctor {
 
 template <typename Context, class ForwardIt, typename InT, typename IndexT>
 static ForwardIt UniqueConsecutiveDimImpl(
-    const Context& context,
+    const Context& context UNUSED,
     ForwardIt first,
     ForwardIt last,
     const std::vector<IndexT>& sorted_indices_vec,
diff --git a/paddle/phi/kernels/cpu/viterbi_decode_kernel.cc b/paddle/phi/kernels/cpu/viterbi_decode_kernel.cc
index 42fdd78c6205f..f129000e0a97a 100644
--- a/paddle/phi/kernels/cpu/viterbi_decode_kernel.cc
+++ b/paddle/phi/kernels/cpu/viterbi_decode_kernel.cc
@@ -113,7 +113,7 @@ template <typename Context,
           class CompareFunctor,
           typename T>
 struct GetMask {
-  void operator()(const Context& dev_ctx,
+  void operator()(const Context& dev_ctx UNUSED,
                   const DenseTensor& lhs,
                   const DenseTensor& rhs,
                   DenseTensor* mask) {
diff --git a/paddle/phi/kernels/funcs/activation_functor.h b/paddle/phi/kernels/funcs/activation_functor.h
index bc1b21c1c8cce..ccc2c474499d7 100644
--- a/paddle/phi/kernels/funcs/activation_functor.h
+++ b/paddle/phi/kernels/funcs/activation_functor.h
@@ -597,7 +597,7 @@ struct SquareGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = dout * static_cast<T>(2) * x;
   }
 
@@ -1087,7 +1087,7 @@ struct ExpGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x UNUSED, Out out, dOut dout, dX dx) const {
     dx.device(d) = dout * out;
   }
 
diff --git a/paddle/phi/kernels/funcs/compound_functors.h b/paddle/phi/kernels/funcs/compound_functors.h
index d51c13950aa33..121597bca6873 100644
--- a/paddle/phi/kernels/funcs/compound_functors.h
+++ b/paddle/phi/kernels/funcs/compound_functors.h
@@ -47,7 +47,7 @@ struct UnaryCompoundFunctor {
 
   inline HOSTDEVICE T GetOut(T x, T y) { return func1_(func2_(x, y)); }
 
-  inline HOSTDEVICE T GetOutUseIntermediateOut(T x, T intermediat_out) {
+  inline HOSTDEVICE T GetOutUseIntermediateOut(T x UNUSED, T intermediat_out) {
     return func1_(intermediat_out);
   }
 
diff --git a/paddle/phi/kernels/funcs/cpu_vec.h b/paddle/phi/kernels/funcs/cpu_vec.h
index e7dc6535c1547..6774cd391dd5d 100644
--- a/paddle/phi/kernels/funcs/cpu_vec.h
+++ b/paddle/phi/kernels/funcs/cpu_vec.h
@@ -475,7 +475,7 @@ inline void vec_add_bias<float, backends::cpu::avx512f>(const int n,
 }
 
 template <typename T, backends::cpu::cpu_isa_t isa = backends::cpu::isa_any>
-inline void vec_identity(const int n, const T* x, T* y) {
+inline void vec_identity(const int n UNUSED, const T* x UNUSED, T* y UNUSED) {
   // do nothing
   return;
 }
diff --git a/paddle/phi/kernels/funcs/detail/strided_memcpy.h b/paddle/phi/kernels/funcs/detail/strided_memcpy.h
index 209d463065b97..0cd07fdfd0e1a 100644
--- a/paddle/phi/kernels/funcs/detail/strided_memcpy.h
+++ b/paddle/phi/kernels/funcs/detail/strided_memcpy.h
@@ -32,9 +32,9 @@ template <typename T>
 struct StridedMemcpyFunctor<T, 0> {
   void operator()(const phi::DeviceContext& dev_ctx,
                   const T* src,
-                  const int64_t* src_stride,
-                  const int64_t* dst_dim,
-                  const int64_t* dst_stride,
+                  const int64_t* src_stride UNUSED,
+                  const int64_t* dst_dim UNUSED,
+                  const int64_t* dst_stride UNUSED,
                   T* dst) const {
     auto place = dev_ctx.GetPlace();
     if (place.GetType() == phi::AllocationType::CPU) {
@@ -58,9 +58,9 @@ template <typename T>
 struct StridedMemcpyFunctor<T, 1> {
   void operator()(const phi::DeviceContext& dev_ctx,
                   const T* src,
-                  const int64_t* src_stride,
+                  const int64_t* src_stride UNUSED,
                   const int64_t* dst_dim,
-                  const int64_t* dst_stride,
+                  const int64_t* dst_stride UNUSED,
                   T* dst) const {
     auto place = dev_ctx.GetPlace();
     if (place.GetType() == phi::AllocationType::CPU) {
diff --git a/paddle/phi/kernels/funcs/jit/more/mkl/mkl.cc b/paddle/phi/kernels/funcs/jit/more/mkl/mkl.cc
index daf9eac988a16..deaeba0224fb9 100644
--- a/paddle/phi/kernels/funcs/jit/more/mkl/mkl.cc
+++ b/paddle/phi/kernels/funcs/jit/more/mkl/mkl.cc
@@ -246,10 +246,10 @@ bool MatMulKernel<double>::CanBeUsed(const matmul_attr_t& attr) const {
   return true;
 }
 
-#define AWALYS_USE_ME_WITH_DOUBLE(func)                      \
-  template <>                                                \
-  bool func##Kernel<double>::CanBeUsed(const int& d) const { \
-    return true;                                             \
+#define AWALYS_USE_ME_WITH_DOUBLE(func)                             \
+  template <>                                                       \
+  bool func##Kernel<double>::CanBeUsed(const int& d UNUSED) const { \
+    return true;                                                    \
   }
 
 AWALYS_USE_ME_WITH_DOUBLE(VMul);
diff --git a/paddle/phi/kernels/funcs/scatter.h b/paddle/phi/kernels/funcs/scatter.h
index d430588541837..64bca648251b9 100644
--- a/paddle/phi/kernels/funcs/scatter.h
+++ b/paddle/phi/kernels/funcs/scatter.h
@@ -72,7 +72,7 @@ elementwise_inner_add(const phi::CPUContext& ctx UNUSED,
  * return: output tensor
  */
 template <typename T, typename IndexT = int>
-void ScatterAssign(const phi::CPUContext& ctx,
+void ScatterAssign(const phi::CPUContext& ctx UNUSED,
                    const DenseTensor& src,
                    const DenseTensor& index,
                    DenseTensor* output) {
@@ -241,7 +241,7 @@ void ScatterAssignAdd(const phi::CPUContext& ctx,
 // The function is only for scatter grad x,
 // however update grad use gather
 template <typename T, typename IndexT = int>
-void CPUScatterGradForX(const phi::CPUContext& ctx,
+void CPUScatterGradForX(const phi::CPUContext& ctx UNUSED,
                         const DenseTensor& index,
                         DenseTensor* output) {
   int64_t index_size = index.dims()[0];
diff --git a/paddle/phi/kernels/sparse/cpu/conv.h b/paddle/phi/kernels/sparse/cpu/conv.h
index bbbfc625c127a..abbedd6b491cb 100644
--- a/paddle/phi/kernels/sparse/cpu/conv.h
+++ b/paddle/phi/kernels/sparse/cpu/conv.h
@@ -136,7 +136,7 @@ void ProductRuleBook(const Context& dev_ctx,
 template <typename T, typename Context, typename IntT = int>
 void UpdateRulebookAndOutIndex(const Context& dev_ctx,
                                const SparseCooTensor& x,
-                               const int kernel_size,
+                               const int kernel_size UNUSED,
                                const int out_channels,
                                const DDim& out_dims,
                                DenseTensor* rulebook,

From 6a279dfd59a2e927cd383792efd5309968176633 Mon Sep 17 00:00:00 2001
From: Yuanle Liu <yuanlehome@163.com>
Date: Wed, 10 May 2023 14:34:16 +0800
Subject: [PATCH 254/405] scale, square, sum, swish trt op converter support
 zero dim (#53660)

---
 paddle/fluid/inference/tensorrt/op_teller.cc | 62 ++++++++++++-------
 test/ir/inference/test_trt_convert_scale.py  | 23 +++----
 test/ir/inference/test_trt_convert_square.py | 64 +++++++++++---------
 test/ir/inference/test_trt_convert_sum.py    | 42 +++++++++++--
 test/ir/inference/test_trt_convert_swish.py  | 14 +++--
 5 files changed, 133 insertions(+), 72 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index 56d7fba985ce1..84712e9aa6294 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -105,7 +105,7 @@ struct SimpleOpTypeSetTeller : public Teller {
         "erf",        "floor",       "round",
         "sign",       "silu",        "logical_not",
         "reciprocal", "tanh_shrink", "logsigmoid",
-        "rsqrt"};
+        "rsqrt",      "swish"};
     std::unordered_set<std::string> unary_list = {
         "exp",        "log",  "sqrt",        "abs",        "sin",
         "cos",        "tan",  "tanh",        "sinh",       "cosh",
@@ -1194,9 +1194,9 @@ struct SimpleOpTypeSetTeller : public Teller {
               dtype == framework::proto::VarType::FP16)) {
           return false;
         }
-        if (x_shape.size() == 1) {
-          VLOG(3)
-              << "Scale op does not support 1-dimensional input in tensorrt";
+        if (x_shape.size() == 1 || x_shape.size() == 0) {
+          VLOG(3) << "Scale op does not support 0 or 1-dimensional input in "
+                     "tensorrt";
           return false;
         }
       } else {
@@ -1548,8 +1548,24 @@ struct SimpleOpTypeSetTeller : public Teller {
         return false;
       }
     }
-    // remember that 1D input in static shape mode is filtered at the beginning
+
     if (op_type == "sum") {
+      auto* block = desc.Block();
+      if (block == nullptr) {
+        VLOG(3) << "The block desc is nullptr, we can't continue to analyze. "
+                   "Developers need to check whether block_desc is passed in "
+                   "the pass.";
+        return false;
+      }
+      auto x_var_name = desc.Input("X")[0];
+      auto* x_var = block->FindVar(x_var_name);
+      const auto x_shape = x_var->GetShape();
+      if (!with_dynamic_shape && (x_shape.size() == 0 || x_shape.size() == 1)) {
+        VLOG(3) << op_type
+                << " op does not support input's dim is 0 or 1 in tensorrt "
+                   "with static shape.";
+        return false;
+      }
       return true;
     }
 
@@ -1803,22 +1819,7 @@ struct SimpleOpTypeSetTeller : public Teller {
         }
       }
     }
-    if (op_type == "swish") {
-      auto* block = desc.Block();
-      if (block == nullptr) {
-        VLOG(3) << "The block desc is nullptr, we can't continue to analyze. "
-                   "Developers need to check whether block_desc is passed in "
-                   "the pass.";
-        return false;
-      }
-      auto x_var_name = desc.Input("X")[0];
-      auto* x_var_desc = block->FindVar(x_var_name);
-      const auto x_shape = x_var_desc->GetShape();
-      if (x_shape.size() == 1) {
-        VLOG(3) << "swish op does not support input's dim is 1 in tensorrt.";
-        return false;
-      }
-    }
+
     if (op_type == "prelu") {
       if (desc.Input("X").size() != 1) {
         VLOG(3) << "Invalid input X's size of prelu TRT converter. "
@@ -2180,6 +2181,25 @@ struct SimpleOpTypeSetTeller : public Teller {
       }
     }
 
+    if (op_type == "square") {
+      auto* block = desc.Block();
+      if (block == nullptr) {
+        VLOG(3) << "The block desc is nullptr, we can't continue to analyze. "
+                   "Developers need to check whether block_desc is passed in "
+                   "the pass.";
+        return false;
+      }
+      auto x_var_name = desc.Input("X")[0];
+      auto* x_var = block->FindVar(x_var_name);
+      const auto x_shape = x_var->GetShape();
+      if (!with_dynamic_shape && x_shape.size() == 0) {
+        VLOG(3) << op_type
+                << " op does not support input's dim is 0 in tensorrt "
+                   "with static shape.";
+        return false;
+      }
+    }
+
     if (op_type == "clip") {
       // Paddle-TRT does not support the input tensors: Min and Max
       auto clip_inputs = desc.Inputs();
diff --git a/test/ir/inference/test_trt_convert_scale.py b/test/ir/inference/test_trt_convert_scale.py
index eba6a5be445f8..55d1752d49903 100644
--- a/test/ir/inference/test_trt_convert_scale.py
+++ b/test/ir/inference/test_trt_convert_scale.py
@@ -43,12 +43,14 @@ def generate_input1(attrs: List[Dict[str, Any]], batch, is_int):
                 )
             elif self.dims == 1:
                 return np.ones([24]).astype(np.int32 if is_int else np.float32)
+            elif self.dims == 0:
+                return np.ones([]).astype(np.int32 if is_int else np.float32)
 
         def generate_weight1(attrs: List[Dict[str, Any]], is_int):
             return np.ones([1]).astype(np.int32 if is_int else np.float32)
 
         for num_input in [0, 1]:
-            for dims in [1, 2, 3, 4]:
+            for dims in [0, 1, 2, 3, 4]:
                 for batch in [1, 2]:
                     for scale in [0.1, -1.0]:
                         for bias in [0.0, 1.2]:
@@ -141,6 +143,10 @@ def generate_dynamic_shape(attrs):
                 self.dynamic_shape.min_input_shape = {"scale_input": [24]}
                 self.dynamic_shape.max_input_shape = {"scale_input": [48]}
                 self.dynamic_shape.opt_input_shape = {"scale_input": [24]}
+            elif self.dims == 0:
+                self.dynamic_shape.min_input_shape = {"scale_input": []}
+                self.dynamic_shape.max_input_shape = {"scale_input": []}
+                self.dynamic_shape.opt_input_shape = {"scale_input": []}
 
         def clear_dynamic_shape():
             self.dynamic_shape.min_input_shape = {}
@@ -148,6 +154,8 @@ def clear_dynamic_shape():
             self.dynamic_shape.opt_input_shape = {}
 
         def generate_trt_nodes_num(attrs, dynamic_shape):
+            if not dynamic_shape and (self.dims == 1 or self.dims == 0):
+                return 0, 3
             return 1, 2
 
         attrs = [
@@ -189,23 +197,12 @@ def teller1(program_config, predictor_config):
         )
 
         def teller2(program_config, predictor_config):
-            if self.dims == 1 and len(self.dynamic_shape.min_input_shape) == 0:
-                return True
-            return False
-
-        self.add_skip_case(
-            teller2,
-            SkipReasons.TRT_NOT_SUPPORT,
-            "INPUT DIM EQUAL TO 1 OF STATIC SHAPE NOT SUPPORT",
-        )
-
-        def teller3(program_config, predictor_config):
             if self.is_int and len(self.dynamic_shape.min_input_shape) == 0:
                 return True
             return False
 
         self.add_skip_case(
-            teller3,
+            teller2,
             SkipReasons.TRT_NOT_SUPPORT,
             "INTEGER INPUT OF STATIC SHAPE NOT SUPPORT",
         )
diff --git a/test/ir/inference/test_trt_convert_square.py b/test/ir/inference/test_trt_convert_square.py
index a3cd487881b09..4dd30b4016f9f 100644
--- a/test/ir/inference/test_trt_convert_square.py
+++ b/test/ir/inference/test_trt_convert_square.py
@@ -29,7 +29,9 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool:
 
     def sample_program_configs(self):
         def generate_input1(dims):
-            if dims == 1:
+            if dims == 0:
+                return np.ones([]).astype(np.float32)
+            elif dims == 1:
                 return np.ones([3]).astype(np.float32)
             elif dims == 2:
                 return np.ones([3, 64]).astype(np.float32)
@@ -38,40 +40,42 @@ def generate_input1(dims):
             else:
                 return np.ones([1, 3, 64, 64]).astype(np.float32)
 
-        for dims in [1, 2, 3, 4]:
-            for alpha in [1.0, 2.0, 3.0]:
-                self.dims = dims
-
-                ops_config = [
-                    {
-                        "op_type": "square",
-                        "op_inputs": {
-                            "X": ["input_data"],
-                        },
-                        "op_outputs": {"Out": ["output_data"]},
-                        "op_attrs": {},
-                    }
-                ]
-                ops = self.generate_op_config(ops_config)
-
-                program_config = ProgramConfig(
-                    ops=ops,
-                    weights={},
-                    inputs={
-                        "input_data": TensorConfig(
-                            data_gen=partial(generate_input1, dims)
-                        )
+        for dims in [0, 1, 2, 3, 4]:
+            self.dims = dims
+            ops_config = [
+                {
+                    "op_type": "square",
+                    "op_inputs": {
+                        "X": ["input_data"],
                     },
-                    outputs=["output_data"],
-                )
-
-                yield program_config
+                    "op_outputs": {"Out": ["output_data"]},
+                    "op_attrs": {},
+                }
+            ]
+            ops = self.generate_op_config(ops_config)
+
+            program_config = ProgramConfig(
+                ops=ops,
+                weights={},
+                inputs={
+                    "input_data": TensorConfig(
+                        data_gen=partial(generate_input1, dims)
+                    )
+                },
+                outputs=["output_data"],
+            )
+
+            yield program_config
 
     def sample_predictor_configs(
         self, program_config
     ) -> (paddle_infer.Config, List[int], float):
         def generate_dynamic_shape(attrs):
-            if self.dims == 1:
+            if self.dims == 0:
+                self.dynamic_shape.min_input_shape = {"input_data": []}
+                self.dynamic_shape.max_input_shape = {"input_data": []}
+                self.dynamic_shape.opt_input_shape = {"input_data": []}
+            elif self.dims == 1:
                 self.dynamic_shape.min_input_shape = {"input_data": [1]}
                 self.dynamic_shape.max_input_shape = {"input_data": [128]}
                 self.dynamic_shape.opt_input_shape = {"input_data": [64]}
@@ -102,7 +106,7 @@ def clear_dynamic_shape():
             self.dynamic_shape.opt_input_shape = {}
 
         def generate_trt_nodes_num(attrs, dynamic_shape):
-            if not dynamic_shape and self.dims == 1:
+            if not dynamic_shape and (self.dims == 1 or self.dims == 0):
                 return 0, 3
             return 1, 2
 
diff --git a/test/ir/inference/test_trt_convert_sum.py b/test/ir/inference/test_trt_convert_sum.py
index ccc04cc84d48f..231921d641b73 100644
--- a/test/ir/inference/test_trt_convert_sum.py
+++ b/test/ir/inference/test_trt_convert_sum.py
@@ -37,6 +37,8 @@ def generate_input1(batch):
                 return np.ones([batch, 24]).astype(np.float32)
             elif self.dims == 1:
                 return np.ones([24]).astype(np.float32)
+            elif self.dims == 0:
+                return np.ones([]).astype(np.float32)
 
         def generate_input2(batch):
             if self.dims == 4:
@@ -47,6 +49,8 @@ def generate_input2(batch):
                 return np.ones([batch, 24]).astype(np.float32)
             elif self.dims == 1:
                 return np.ones([24]).astype(np.float32)
+            elif self.dims == 0:
+                return np.ones([]).astype(np.float32)
 
         def generate_input3(batch):
             if self.dims == 4:
@@ -57,8 +61,10 @@ def generate_input3(batch):
                 return np.ones([batch, 24]).astype(np.float32)
             elif self.dims == 1:
                 return np.ones([24]).astype(np.float32)
+            elif self.dims == 0:
+                return np.ones([]).astype(np.float32)
 
-        for dims in [1, 2, 3, 4]:
+        for dims in [0, 1, 2, 3, 4]:
             for batch in [1, 4]:
                 self.dims = dims
                 ops_config = [
@@ -157,6 +163,22 @@ def generate_dynamic_shape():
                     "input2": [24],
                     "input3": [24],
                 }
+            elif self.dims == 0:
+                self.dynamic_shape.min_input_shape = {
+                    "input1": [],
+                    "input2": [],
+                    "input3": [],
+                }
+                self.dynamic_shape.max_input_shape = {
+                    "input1": [],
+                    "input2": [],
+                    "input3": [],
+                }
+                self.dynamic_shape.opt_input_shape = {
+                    "input1": [],
+                    "input2": [],
+                    "input3": [],
+                }
 
         def clear_dynamic_shape():
             self.dynamic_shape.min_input_shape = {}
@@ -164,7 +186,7 @@ def clear_dynamic_shape():
             self.dynamic_shape.opt_input_shape = {}
 
         def generate_trt_nodes_num(dynamic_shape):
-            if self.dims == 1 and not dynamic_shape:
+            if (self.dims == 1 or self.dims == 0) and not dynamic_shape:
                 return 0, 5
             return 1, 4
 
@@ -205,8 +227,10 @@ def generate_input1(batch):
                 return np.ones([batch, 24]).astype(np.float32)
             elif self.dims == 1:
                 return np.ones([24]).astype(np.float32)
+            else:
+                return np.ones([]).astype(np.float32)
 
-        for dims in [1, 2, 3, 4]:
+        for dims in [0, 1, 2, 3, 4]:
             for batch in [1, 4]:
                 self.dims = dims
                 ops_config = [
@@ -263,6 +287,16 @@ def generate_dynamic_shape():
                 self.dynamic_shape.opt_input_shape = {
                     "input1": [24],
                 }
+            elif self.dims == 0:
+                self.dynamic_shape.min_input_shape = {
+                    "input1": [],
+                }
+                self.dynamic_shape.max_input_shape = {
+                    "input1": [],
+                }
+                self.dynamic_shape.opt_input_shape = {
+                    "input1": [],
+                }
 
         def clear_dynamic_shape():
             self.dynamic_shape.min_input_shape = {}
@@ -270,7 +304,7 @@ def clear_dynamic_shape():
             self.dynamic_shape.opt_input_shape = {}
 
         def generate_trt_nodes_num(dynamic_shape):
-            if self.dims == 1 and not dynamic_shape:
+            if (self.dims == 1 or self.dims == 0) and not dynamic_shape:
                 return 0, 3
             return 1, 2
 
diff --git a/test/ir/inference/test_trt_convert_swish.py b/test/ir/inference/test_trt_convert_swish.py
index 0ae2939a0d732..c52dd29fcf7b3 100755
--- a/test/ir/inference/test_trt_convert_swish.py
+++ b/test/ir/inference/test_trt_convert_swish.py
@@ -29,7 +29,9 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool:
 
     def sample_program_configs(self):
         def generate_input1(dims, attrs: List[Dict[str, Any]]):
-            if dims == 1:
+            if dims == 0:
+                return np.ones([]).astype(np.float32)
+            elif dims == 1:
                 return np.ones([3]).astype(np.float32)
             elif dims == 2:
                 return np.ones([3, 64]).astype(np.float32)
@@ -38,7 +40,7 @@ def generate_input1(dims, attrs: List[Dict[str, Any]]):
             else:
                 return np.ones([1, 3, 64, 64]).astype(np.float32)
 
-        for dims in [1, 2, 3, 4]:
+        for dims in [0, 1, 2, 3, 4]:
             for beta in [1.0, 2.0, 3.0]:
                 self.dims = dims
 
@@ -73,7 +75,11 @@ def sample_predictor_configs(
         self, program_config
     ) -> (paddle_infer.Config, List[int], float):
         def generate_dynamic_shape(attrs):
-            if self.dims == 1:
+            if self.dims == 0:
+                self.dynamic_shape.min_input_shape = {"input_data": []}
+                self.dynamic_shape.max_input_shape = {"input_data": []}
+                self.dynamic_shape.opt_input_shape = {"input_data": []}
+            elif self.dims == 1:
                 self.dynamic_shape.min_input_shape = {"input_data": [1]}
                 self.dynamic_shape.max_input_shape = {"input_data": [128]}
                 self.dynamic_shape.opt_input_shape = {"input_data": [64]}
@@ -104,7 +110,7 @@ def clear_dynamic_shape():
             self.dynamic_shape.opt_input_shape = {}
 
         def generate_trt_nodes_num(attrs, dynamic_shape):
-            if self.dims == 1:
+            if (self.dims == 1 or self.dims == 0) and not dynamic_shape:
                 return 0, 3
             return 1, 2
 

From f023d42f36ee0e7bfb1c8162987a5b40dd97cae1 Mon Sep 17 00:00:00 2001
From: Roc <30228238+sljlp@users.noreply.github.com>
Date: Wed, 10 May 2023 14:42:53 +0800
Subject: [PATCH 255/405] [NPU] PP for npu (#53501)

* revert p2p communication for xpu

* pp for npu

* update

* update

* fix xpuplace

* add ut for sync send

* Revert "fix xpuplace"

This reverts commit f89c1d7622426686bc153a3414a42c39e0f4a647.

* add ut for pp sync send

* rm unusable ut

* update
---
 .../pp_utils/p2p_communication.py             | 302 +++++++++++++-----
 .../unittests/collective/fleet/CMakeLists.txt |  13 +
 ...lel_dygraph_pipeline_parallel_sync_send.py |  37 +++
 .../unittests/collective/fleet/testslist.csv  |   1 +
 4 files changed, 274 insertions(+), 79 deletions(-)
 create mode 100644 python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_sync_send.py

diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
index 7d7a02d3e25a7..da20e312a1dc8 100644
--- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
+++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
+import os
+
 import numpy as np
 
 import paddle
@@ -24,6 +27,9 @@
 _use_cache = False
 _enable_partial_send_recv = True
 
+_sync_send = os.environ.get("PADDLE_P2P_SYNC_SEND", "0")
+_sync_send = _sync_send.lower() in ['1', 'true']
+
 
 def initialize_p2p_groups(hcg, use_cache=True, enable_partial_send_recv=True):
     global _hcg, _use_cache, _enable_partial_send_recv
@@ -351,43 +357,111 @@ def _p2p_helper(
     # TODO(Yuang Liu): use batch_isend_irecv replace all these comm ops
     tasks = []
     # start to p2p communicate
-    if tensor_send_prev is not None:
-        if isinstance(tensor_send_prev, tuple):
-            for d in tensor_send_prev:
-                paddle.distributed.wait(d, use_calc_stream=True)
+    if _sync_send:
+        # Some devices(NPU for example) do not support asynchronized send op, So the order is
+        # recv_prev -> send_next -> recv_next -> send_prev
+        # When using this order, the environment variable
+        # 'PADDLE_P2P_SYNC_SEND' should be set True
+        if tensor_recv_prev is not None:
+            if isinstance(tensor_recv_prev, tuple):
+                for d in tensor_recv_prev:
+                    task = recv_partial(
+                        d,
+                        src=0,
+                        nranks=mp_degree,
+                        rank_id=mp_rank,
+                        group=_hcg.recv_prev_group,
+                        use_calc_stream=sync_recv,
+                    )
+                    if sync_recv:
+                        allgather_partial(
+                            d,
+                            nranks=mp_degree,
+                            rank_id=mp_rank,
+                            group=mp_group,
+                            use_calc_stream=True,
+                        )
+                    else:
+                        tasks.append(task)
+            else:
+                task = recv_partial(
+                    tensor_recv_prev,
+                    src=0,
+                    nranks=mp_degree,
+                    rank_id=mp_rank,
+                    group=_hcg.recv_prev_group,
+                    use_calc_stream=sync_recv,
+                )
+
+                if sync_recv:
+                    allgather_partial(
+                        tensor_recv_prev,
+                        nranks=mp_degree,
+                        rank_id=mp_rank,
+                        group=mp_group,
+                        use_calc_stream=True,
+                    )
+                else:
+                    tasks.append(task)
+
+        if tensor_send_next is not None:
+            if isinstance(tensor_send_next, tuple):
+                for d in tensor_send_next:
+                    paddle.distributed.wait(d, use_calc_stream=True)
+                    send_partial(
+                        d,
+                        dst=1,
+                        nranks=mp_degree,
+                        rank_id=mp_rank,
+                        group=_hcg.send_next_group,
+                        use_calc_stream=False,
+                    )
+            else:
+                paddle.distributed.wait(tensor_send_next, use_calc_stream=True)
                 send_partial(
-                    d,
-                    dst=0,
+                    tensor_send_next,
+                    dst=1,
                     nranks=mp_degree,
                     rank_id=mp_rank,
-                    group=_hcg.send_prev_group,
+                    group=_hcg.send_next_group,
                     use_calc_stream=False,
                 )
-        else:
-            paddle.distributed.wait(tensor_send_prev, use_calc_stream=True)
-            send_partial(
-                tensor_send_prev,
-                dst=0,
-                nranks=mp_degree,
-                rank_id=mp_rank,
-                group=_hcg.send_prev_group,
-                use_calc_stream=False,
-            )
 
-    if tensor_recv_prev is not None:
-        if isinstance(tensor_recv_prev, tuple):
-            for d in tensor_recv_prev:
+        if tensor_recv_next is not None:
+            if isinstance(tensor_recv_next, tuple):
+                for d in tensor_recv_next:
+                    task = recv_partial(
+                        d,
+                        src=1,
+                        nranks=mp_degree,
+                        rank_id=mp_rank,
+                        group=_hcg.recv_next_group,
+                        use_calc_stream=sync_recv,
+                    )
+
+                    if sync_recv:
+                        allgather_partial(
+                            d,
+                            nranks=mp_degree,
+                            rank_id=mp_rank,
+                            group=mp_group,
+                            use_calc_stream=True,
+                        )
+                    else:
+                        tasks.append(task)
+
+            else:
                 task = recv_partial(
-                    d,
-                    src=0,
+                    tensor_recv_next,
+                    src=1,
                     nranks=mp_degree,
                     rank_id=mp_rank,
-                    group=_hcg.recv_prev_group,
+                    group=_hcg.recv_next_group,
                     use_calc_stream=sync_recv,
                 )
                 if sync_recv:
                     allgather_partial(
-                        d,
+                        tensor_recv_next,
                         nranks=mp_degree,
                         rank_id=mp_rank,
                         group=mp_group,
@@ -395,65 +469,154 @@ def _p2p_helper(
                     )
                 else:
                     tasks.append(task)
-        else:
-            task = recv_partial(
-                tensor_recv_prev,
-                src=0,
-                nranks=mp_degree,
-                rank_id=mp_rank,
-                group=_hcg.recv_prev_group,
-                use_calc_stream=sync_recv,
-            )
 
-            if sync_recv:
-                allgather_partial(
-                    tensor_recv_prev,
+        if tensor_send_prev is not None:
+            if isinstance(tensor_send_prev, tuple):
+                for d in tensor_send_prev:
+                    paddle.distributed.wait(d, use_calc_stream=True)
+                    send_partial(
+                        d,
+                        dst=0,
+                        nranks=mp_degree,
+                        rank_id=mp_rank,
+                        group=_hcg.send_prev_group,
+                        use_calc_stream=False,
+                    )
+            else:
+                paddle.distributed.wait(tensor_send_prev, use_calc_stream=True)
+                send_partial(
+                    tensor_send_prev,
+                    dst=0,
+                    nranks=mp_degree,
+                    rank_id=mp_rank,
+                    group=_hcg.send_prev_group,
+                    use_calc_stream=False,
+                )
+
+    else:
+        if tensor_send_prev is not None:
+            if isinstance(tensor_send_prev, tuple):
+                for d in tensor_send_prev:
+                    paddle.distributed.wait(d, use_calc_stream=True)
+                    send_partial(
+                        d,
+                        dst=0,
+                        nranks=mp_degree,
+                        rank_id=mp_rank,
+                        group=_hcg.send_prev_group,
+                        use_calc_stream=False,
+                    )
+            else:
+                paddle.distributed.wait(tensor_send_prev, use_calc_stream=True)
+                send_partial(
+                    tensor_send_prev,
+                    dst=0,
                     nranks=mp_degree,
                     rank_id=mp_rank,
-                    group=mp_group,
-                    use_calc_stream=True,
+                    group=_hcg.send_prev_group,
+                    use_calc_stream=False,
                 )
+
+        if tensor_recv_prev is not None:
+            if isinstance(tensor_recv_prev, tuple):
+                for d in tensor_recv_prev:
+                    task = recv_partial(
+                        d,
+                        src=0,
+                        nranks=mp_degree,
+                        rank_id=mp_rank,
+                        group=_hcg.recv_prev_group,
+                        use_calc_stream=sync_recv,
+                    )
+                    if sync_recv:
+                        allgather_partial(
+                            d,
+                            nranks=mp_degree,
+                            rank_id=mp_rank,
+                            group=mp_group,
+                            use_calc_stream=True,
+                        )
+                    else:
+                        tasks.append(task)
             else:
-                tasks.append(task)
+                task = recv_partial(
+                    tensor_recv_prev,
+                    src=0,
+                    nranks=mp_degree,
+                    rank_id=mp_rank,
+                    group=_hcg.recv_prev_group,
+                    use_calc_stream=sync_recv,
+                )
+
+                if sync_recv:
+                    allgather_partial(
+                        tensor_recv_prev,
+                        nranks=mp_degree,
+                        rank_id=mp_rank,
+                        group=mp_group,
+                        use_calc_stream=True,
+                    )
+                else:
+                    tasks.append(task)
 
-    if tensor_send_next is not None:
-        if isinstance(tensor_send_next, tuple):
-            for d in tensor_send_next:
-                paddle.distributed.wait(d, use_calc_stream=True)
+        if tensor_send_next is not None:
+            if isinstance(tensor_send_next, tuple):
+                for d in tensor_send_next:
+                    paddle.distributed.wait(d, use_calc_stream=True)
+                    send_partial(
+                        d,
+                        dst=1,
+                        nranks=mp_degree,
+                        rank_id=mp_rank,
+                        group=_hcg.send_next_group,
+                        use_calc_stream=False,
+                    )
+            else:
+                paddle.distributed.wait(tensor_send_next, use_calc_stream=True)
                 send_partial(
-                    d,
+                    tensor_send_next,
                     dst=1,
                     nranks=mp_degree,
                     rank_id=mp_rank,
                     group=_hcg.send_next_group,
                     use_calc_stream=False,
                 )
-        else:
-            paddle.distributed.wait(tensor_send_next, use_calc_stream=True)
-            send_partial(
-                tensor_send_next,
-                dst=1,
-                nranks=mp_degree,
-                rank_id=mp_rank,
-                group=_hcg.send_next_group,
-                use_calc_stream=False,
-            )
 
-    if tensor_recv_next is not None:
-        if isinstance(tensor_recv_next, tuple):
-            for d in tensor_recv_next:
+        if tensor_recv_next is not None:
+            if isinstance(tensor_recv_next, tuple):
+                for d in tensor_recv_next:
+                    task = recv_partial(
+                        d,
+                        src=1,
+                        nranks=mp_degree,
+                        rank_id=mp_rank,
+                        group=_hcg.recv_next_group,
+                        use_calc_stream=sync_recv,
+                    )
+
+                    if sync_recv:
+                        allgather_partial(
+                            d,
+                            nranks=mp_degree,
+                            rank_id=mp_rank,
+                            group=mp_group,
+                            use_calc_stream=True,
+                        )
+                    else:
+                        tasks.append(task)
+
+            else:
                 task = recv_partial(
-                    d,
+                    tensor_recv_next,
                     src=1,
                     nranks=mp_degree,
                     rank_id=mp_rank,
                     group=_hcg.recv_next_group,
                     use_calc_stream=sync_recv,
                 )
-
                 if sync_recv:
                     allgather_partial(
-                        d,
+                        tensor_recv_next,
                         nranks=mp_degree,
                         rank_id=mp_rank,
                         group=mp_group,
@@ -462,25 +625,6 @@ def _p2p_helper(
                 else:
                     tasks.append(task)
 
-        else:
-            task = recv_partial(
-                tensor_recv_next,
-                src=1,
-                nranks=mp_degree,
-                rank_id=mp_rank,
-                group=_hcg.recv_next_group,
-                use_calc_stream=sync_recv,
-            )
-            if sync_recv:
-                allgather_partial(
-                    tensor_recv_next,
-                    nranks=mp_degree,
-                    rank_id=mp_rank,
-                    group=mp_group,
-                    use_calc_stream=True,
-                )
-            else:
-                tasks.append(task)
     if not sync_recv:
         if framework.in_dygraph_mode():
             # wait irecv tasks in eager dygraph mode with new comm library
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt b/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt
index 17408305e5992..de76e969b55f3 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt
@@ -162,6 +162,19 @@ if((WITH_GPU) AND LOCAL_ALL_PLAT)
   set_tests_properties(test_parallel_dygraph_pipeline_parallel
                        PROPERTIES TIMEOUT "500")
 endif()
+if((WITH_GPU OR WITH_XPU) AND LOCAL_ALL_PLAT)
+  bash_test_modules(
+    test_parallel_dygraph_pipeline_parallel_sync_send
+    START_BASH
+    ../../dist_test.sh
+    LABELS
+    "RUN_TYPE=DIST"
+    ENVS
+    "PADDLE_DIST_UT_PORT=21992;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python;PADDLE_P2P_SYNC_SEND=1"
+  )
+  set_tests_properties(test_parallel_dygraph_pipeline_parallel_sync_send
+                       PROPERTIES TIMEOUT "300")
+endif()
 if((WITH_GPU) AND LOCAL_ALL_PLAT)
   bash_test_modules(
     test_parallel_dygraph_pipeline_parallel_with_virtual_stage
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_sync_send.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_sync_send.py
new file mode 100644
index 0000000000000..2ea7fa2372232
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_sync_send.py
@@ -0,0 +1,37 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+
+from test_parallel_dygraph_dataparallel import TestMultipleGpus
+
+# os.environ["PADDLE_P2P_SYNC_SEND"] = "1"
+
+
+class TestHybridPipeParallel(TestMultipleGpus):
+    def test_hybrid_parallel_pp_layer(self):
+        self.run_mnist_2gpu(
+            os.path.abspath('../../hybrid_parallel_pp_layer.py')
+        )
+
+    def test_hybrid_parallel_pp_tuple_inputs(self):
+        self.run_mnist_2gpu('hybrid_parallel_pp_embedding.py')
+
+    def test_hybrid_parallel_shared_weight(self):
+        self.run_mnist_2gpu('hybrid_parallel_shared_weight.py')
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv b/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv
index 688b5e759b1db..751a67754cf55 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv
@@ -13,6 +13,7 @@ test_tcp_store,LINUX;APPLE,,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=
 test_dygraph_sharding_stage3_for_eager,,,350,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_communicator_half_async,,,120,DIST,test_runner.py,2,,FLAGS_communicator_send_queue_size=1;FLAGS_communicator_max_merge_var_num=1;http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL
 test_parallel_dygraph_pipeline_parallel,,GPU,500,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_parallel_dygraph_pipeline_parallel_sync_send,,GPU;XPU,300,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..;PADDLE_P2P_SYNC_SEND=1,
 test_parallel_dygraph_pipeline_parallel_with_virtual_stage,,GPU,500,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_pp_adaptor,,GPU,500,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_localsgd_meta_optimizer,LINUX,GPU;XPU,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,

From c828934e94af66ee6539a7a978e59cdeae79671a Mon Sep 17 00:00:00 2001
From: jiangcheng <thisjiang@qq.com>
Date: Wed, 10 May 2023 15:46:56 +0800
Subject: [PATCH 256/405] [CINN] fix check_cinn in Optest check condition error
 bug (#53676)

---
 python/paddle/fluid/tests/unittests/eager_op_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/eager_op_test.py b/python/paddle/fluid/tests/unittests/eager_op_test.py
index 469caf4feae87..63a14a66efa62 100644
--- a/python/paddle/fluid/tests/unittests/eager_op_test.py
+++ b/python/paddle/fluid/tests/unittests/eager_op_test.py
@@ -557,9 +557,9 @@ def disable_cal_ref_output(self):
     def _enable_check_cinn_test(self, place, inputs, outputs):
         # if the test not run in cuda or the paddle not compile with CINN, skip cinn test
         if (
-            core.is_compiled_with_cinn()
-            and core.is_compiled_with_cuda()
-            and isinstance(place, fluid.CUDAPlace)
+            not core.is_compiled_with_cinn()
+            or not core.is_compiled_with_cuda()
+            or not isinstance(place, fluid.CUDAPlace)
         ):
             return False
         # CINN not support bfloat16 now, skip cinn test

From 0f319f8afa1cfc7441d7d00b8c5ef5d3a235fd38 Mon Sep 17 00:00:00 2001
From: Zhang Zheng <32410583+ZzSean@users.noreply.github.com>
Date: Wed, 10 May 2023 15:51:33 +0800
Subject: [PATCH 257/405] Fix bug in log_softmax kernel when lastdim is larger
 than 100000 (#53654)

---
 paddle/phi/kernels/gpudnn/softmax_gpudnn.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/paddle/phi/kernels/gpudnn/softmax_gpudnn.h b/paddle/phi/kernels/gpudnn/softmax_gpudnn.h
index 8ba329301c916..9db814f198643 100644
--- a/paddle/phi/kernels/gpudnn/softmax_gpudnn.h
+++ b/paddle/phi/kernels/gpudnn/softmax_gpudnn.h
@@ -499,8 +499,7 @@ __global__ void KeMatrixSoftmaxForward(T* softmax, const T* src, int dim_size) {
 
   // write data to softmax_output according to the LogMode
   if (LogMode) {
-    LogSoftmaxForwardFunctor<AccT, T> reduction(thread_max,
-                                                std::log(thread_exp));
+    LogSoftmaxForwardFunctor<AccT, T> reduction(thread_max, thread_exp);
     if (input_align_shift == output_align_shift) {
       ThreadVecWriteVec<LogSoftmaxForwardFunctor, T, AccT, VecSize>(
           batch_output, batch_input, dim_size, input_align_shift, reduction);

From 7f39bcd1cc6cf4f31b5f17ecbf3c5b5b59235787 Mon Sep 17 00:00:00 2001
From: Chitsing KUI <kuizhiqing@msn.com>
Date: Wed, 10 May 2023 16:05:01 +0800
Subject: [PATCH 258/405] [LAUNCH] add log overwrite flag (#53608)

* add log overwrite flag

* use strtobool
---
 .../paddle/distributed/launch/context/args_envs.py   | 11 ++++++++++-
 .../distributed/launch/controllers/controller.py     |  3 ++-
 python/paddle/distributed/launch/job/container.py    | 12 +++++++-----
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/python/paddle/distributed/launch/context/args_envs.py b/python/paddle/distributed/launch/context/args_envs.py
index de91d90119633..9e0565b1c45e5 100644
--- a/python/paddle/distributed/launch/context/args_envs.py
+++ b/python/paddle/distributed/launch/context/args_envs.py
@@ -14,6 +14,7 @@
 
 import os
 from argparse import REMAINDER, ArgumentParser
+from distutils.util import strtobool
 
 env_args_mapping = {
     'POD_IP': 'host',
@@ -22,6 +23,7 @@
     'PADDLE_NNODES': 'nnodes',
     'PADDLE_RUN_MODE': 'run_mode',
     'PADDLE_LOG_LEVEL': 'log_level',
+    'PADDLE_LOG_OVERWRITE': 'log_overwrite',
     'PADDLE_NPROC_PER_NODE': 'nproc_per_node',
     'PADDLE_JOB_ID': 'job_id',
     'PADDLE_RANK': 'rank',
@@ -61,7 +63,7 @@ def parse_args():
     )
 
     base_group.add_argument(
-        "--legacy", type=bool, default=False, help="use legacy launch"
+        "--legacy", type=strtobool, default=False, help="use legacy launch"
     )
 
     base_group.add_argument(
@@ -72,6 +74,13 @@ def parse_args():
         "--log_level", type=str, default="INFO", help="log level. Default INFO"
     )
 
+    base_group.add_argument(
+        "--log_overwrite",
+        type=strtobool,
+        default=False,
+        help="overwrite exits logfiles. Default False",
+    )
+
     base_group.add_argument(
         "--nnodes",
         type=str,
diff --git a/python/paddle/distributed/launch/controllers/controller.py b/python/paddle/distributed/launch/controllers/controller.py
index 585d65a41657e..0b8dd72ca6453 100644
--- a/python/paddle/distributed/launch/controllers/controller.py
+++ b/python/paddle/distributed/launch/controllers/controller.py
@@ -205,6 +205,7 @@ def new_container(
         c = Container(
             entrypoint=(entrypoint or self._get_entrypoint()),
             env=(self.ctx.get_envs() if use_ctx_env else {}),
+            overwrite_log=self.ctx.args.log_overwrite,
         )
         c.outfile, c.errfile = self._get_out_err_file(out, err)
         c.update_env(envs)
@@ -286,7 +287,7 @@ def _save_container_env(self, container, is_init=False):
         )
         try:
             os.makedirs(os.path.dirname(f), exist_ok=True)
-            with open(f, 'w') as fd:
+            with open(f, container.log_mode) as fd:
                 for k, v in sorted(container.env.items()):
                     fd.write(str(f"{k}={v}\n"))
         except Exception as e:
diff --git a/python/paddle/distributed/launch/job/container.py b/python/paddle/distributed/launch/job/container.py
index 1e42c84b5b2b1..caadcdd1d70cd 100644
--- a/python/paddle/distributed/launch/job/container.py
+++ b/python/paddle/distributed/launch/job/container.py
@@ -25,7 +25,7 @@ class Container:
     TODO(kuizhiqing) A container can be run by process/thread or just a callable function
     '''
 
-    def __init__(self, entrypoint=[], rank=-1, env={}):
+    def __init__(self, entrypoint=[], rank=-1, env={}, overwrite_log=False):
         self._entrypoint = entrypoint
         self._rank = rank
         self._out = None
@@ -39,6 +39,8 @@ def __init__(self, entrypoint=[], rank=-1, env={}):
         self._log_handler = None
         self._shell = False
 
+        self.log_mode = 'w' if overwrite_log else 'a'
+
     @property
     def env(self):
         return self._env
@@ -104,7 +106,7 @@ def _get_fd(self, pth):
             d = os.path.dirname(pth)
             if not os.path.isdir(d):
                 os.makedirs(d, exist_ok=True)
-            return open(pth, 'a')
+            return open(pth, self.log_mode)
         except:
             return None
 
@@ -120,7 +122,7 @@ def start(self):
         elif self._err:
             self._stderr = self._get_fd(self._err) or sys.stderr
 
-        if not self._log_handler:
+        if self._out and not self._log_handler:
             self._log_handler = open(self._out)
             self._log_handler.seek(0, 2)
             self._log_start_offset = self._log_handler.tell()
@@ -179,7 +181,7 @@ def __str__(self):
 
     def logs(self, fn=None, offset=0, whence=1, limit=1000):
         if not self._log_handler:
-            self._log_handler = open(self._out)
+            return
 
         if fn is None:
             fn = sys.stdout
@@ -201,7 +203,7 @@ def logs(self, fn=None, offset=0, whence=1, limit=1000):
 
     def tail(self, length=3000):
         if not self._log_handler:
-            self._log_handler = open(self._out)
+            return
 
         try:
             self._log_handler.seek(0, 2)

From 4f33f44b546ccfcfed6b615a6782abbe37e09bda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?=
 <83450930+Liyulingyue@users.noreply.github.com>
Date: Wed, 10 May 2023 16:20:38 +0800
Subject: [PATCH 259/405] [static op generation] lstsq (#53290)

---
 paddle/fluid/operators/lstsq_op.cc  | 96 -----------------------------
 paddle/phi/api/yaml/legacy_ops.yaml |  9 ---
 paddle/phi/api/yaml/op_compat.yaml  | 10 +++
 paddle/phi/api/yaml/op_version.yaml |  7 +++
 paddle/phi/api/yaml/ops.yaml        | 10 +++
 paddle/phi/ops/compat/lstsq_sig.cc  | 28 ---------
 6 files changed, 27 insertions(+), 133 deletions(-)
 delete mode 100644 paddle/fluid/operators/lstsq_op.cc
 delete mode 100644 paddle/phi/ops/compat/lstsq_sig.cc

diff --git a/paddle/fluid/operators/lstsq_op.cc b/paddle/fluid/operators/lstsq_op.cc
deleted file mode 100644
index bf19e28af0adc..0000000000000
--- a/paddle/fluid/operators/lstsq_op.cc
+++ /dev/null
@@ -1,96 +0,0 @@
-// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/framework/op_version_registry.h"
-#include "paddle/phi/infermeta/binary.h"
-
-namespace paddle {
-namespace operators {
-
-class LstsqOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
-  // The output of lstsq is always complex-valued even for real-valued inputs
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    auto dtype = OperatorWithKernel::IndicateVarDataType(ctx, "X");
-    if (dtype != framework::proto::VarType::FP32 &&
-        dtype != framework::proto::VarType::FP64) {
-      PADDLE_THROW(platform::errors::InvalidArgument(
-          "unsupported data type: %s!", dtype));
-    }
-    return phi::KernelKey(dtype, ctx.GetPlace());
-  }
-};
-
-class LstsqOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput("X",
-             "(Tensor), A real-valued tensor with shape (*, m, n). "
-             "The accepted datatype is one of float32, float64");
-    AddInput("Y",
-             "(Tensor), A real-valued tensor with shape (*, m, k). "
-             "The accepted datatype is one of float32, float64");
-    AddAttr<float>(
-        "rcond",
-        "(float, default 0.0), A float value used to determine the effective "
-        "rank of A.")
-        .SetDefault(0.0f);
-    AddAttr<std::string>("driver",
-                         "(string, default \"gels\"). "
-                         "name of the LAPACK method to be used.")
-        .SetDefault("gels");
-    AddOutput("Solution",
-              "(Tensor), The output Solution tensor with shape (*, n, k).");
-    AddOutput("Residuals",
-              "(Tensor), The output Residuals tensor with shape (*, k).")
-        .AsDispensable();
-    AddOutput("Rank", "(Tensor), The output Rank tensor with shape (*).");
-    AddOutput(
-        "SingularValues",
-        "(Tensor), The output SingularValues tensor with shape (*, min(m,n)).");
-    AddComment(R"DOC(
-        Lstsq Operator.
-This API processes Lstsq functor for general matrices.
-)DOC");
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-
-DECLARE_INFER_SHAPE_FUNCTOR(lstsq,
-                            LstsqInferShapeFunctor,
-                            PD_INFER_META(phi::LstsqInferMeta));
-
-REGISTER_OPERATOR(lstsq,
-                  ops::LstsqOp,
-                  ops::LstsqOpMaker,
-                  LstsqInferShapeFunctor);
-
-REGISTER_OP_VERSION(lstsq).AddCheckpoint(
-    R"ROC(
-        Upgrade lstsq, add 1 outputs [Residuals].
-      )ROC",
-    paddle::framework::compatible::OpVersionDesc().NewOutput(
-        "Residuals",
-        "Output tensor of lstsq operator, "
-        "meaning the squared residuals of the calculated solutions."));
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index 6eff624761c89..9b76564c83259 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -669,15 +669,6 @@
     func : logsumexp
   backward : logsumexp_grad
 
-- op : lstsq
-  args : (Tensor x, Tensor y, Scalar rcond, str driver)
-  output : Tensor(solution), Tensor(residuals), Tensor(rank), Tensor(singular_values)
-  infer_meta :
-    func : LstsqInferMeta
-  kernel :
-    func : lstsq
-    data_type : x
-
 - op : matmul
   args : (Tensor x, Tensor y, bool transpose_x = false, bool transpose_y = false)
   output : Tensor
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index 1e81fcac8eb5e..74bdb3c53dde1 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -1415,6 +1415,16 @@
   extra :
     attrs : [bool use_mkldnn = false, bool is_test = false]
 
+- op : lstsq
+  inputs :
+    {x : X, y : Y}
+  outputs :
+    {solution : Solution, residuals : Residuals, rank : Rank, singular_values : SingularValues}
+  scalar :
+    rcond :
+      data_type : float
+      support_tensor : true
+
 - op : lu_unpack
   backward : lu_unpack_grad
   inputs :
diff --git a/paddle/phi/api/yaml/op_version.yaml b/paddle/phi/api/yaml/op_version.yaml
index b13b432c460f7..9d9cb6187b218 100644
--- a/paddle/phi/api/yaml/op_version.yaml
+++ b/paddle/phi/api/yaml/op_version.yaml
@@ -200,6 +200,13 @@
           comment : In order to change output data type
           default : 5
 
+- op : lstsq
+  version :
+    - checkpoint : Upgrade lstsq, add 1 outputs [Residuals].
+      action :
+        - add_output : Residuals
+          comment : Output tensor of lstsq operator, meaning the squared residuals of the calculated solutions.
+
 - op : matrix_nms
   version :
     - checkpoint : Upgrade matrix_nms, add a new output [RoisNum].
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index e9624cbca06e6..c254a75f98e17 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -1231,6 +1231,16 @@
     func : logsigmoid
   backward : logsigmoid_grad
 
+- op : lstsq
+  args : (Tensor x, Tensor y, Scalar rcond=0.0f, str driver="gels")
+  output : Tensor(solution), Tensor(residuals), Tensor(rank), Tensor(singular_values)
+  infer_meta :
+    func : LstsqInferMeta
+  kernel :
+    func : lstsq
+    data_type : x
+  optional : residuals
+
 - op : lu
   args : (Tensor x, bool pivot = true)
   output : Tensor(out), Tensor(pivots), Tensor(infos)
diff --git a/paddle/phi/ops/compat/lstsq_sig.cc b/paddle/phi/ops/compat/lstsq_sig.cc
deleted file mode 100644
index f36dfb1917caf..0000000000000
--- a/paddle/phi/ops/compat/lstsq_sig.cc
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/core/compat/op_utils.h"
-
-namespace phi {
-
-KernelSignature LstsqOpArgumentMapping(const ArgumentMappingContext& ctx) {
-  return KernelSignature("lstsq",
-                         {"X", "Y"},
-                         {"rcond", "driver"},
-                         {"Solution", "Residuals", "Rank", "SingularValues"});
-}
-
-}  // namespace phi
-
-PD_REGISTER_ARG_MAPPING_FN(lstsq, phi::LstsqOpArgumentMapping);

From 38d664b725dc80b605d7f2c5fd18e892a7bb5f13 Mon Sep 17 00:00:00 2001
From: wz1qqx <55830058+wz1qqx@users.noreply.github.com>
Date: Wed, 10 May 2023 17:39:07 +0800
Subject: [PATCH 260/405] [XPU]Conv transpose fp16 && fix unittest (#53626)

* fix as review, add fp16 conv2d_transpose

* fix unittest of bn and reduce_mean

* fix bn unittest

* fix ci

* fix ci
---
 .../framework/ir/xpu/conv2d_xpu_fuse_pass.cc  |  24 +--
 paddle/phi/backends/xpu/xpu2_op_list.cc       |   3 +-
 paddle/phi/kernels/reduce_mean_kernel.cc      |   3 +-
 paddle/phi/kernels/xpu/activation_kernel.cc   |  12 --
 .../phi/kernels/xpu/conv_transpose_kernel.cc  |  16 +-
 test/xpu/test_batch_norm_op_xpu.py            | 175 ++++++++++++------
 6 files changed, 131 insertions(+), 102 deletions(-)

diff --git a/paddle/fluid/framework/ir/xpu/conv2d_xpu_fuse_pass.cc b/paddle/fluid/framework/ir/xpu/conv2d_xpu_fuse_pass.cc
index c80fda30990ff..91fe7dae50b54 100644
--- a/paddle/fluid/framework/ir/xpu/conv2d_xpu_fuse_pass.cc
+++ b/paddle/fluid/framework/ir/xpu/conv2d_xpu_fuse_pass.cc
@@ -34,25 +34,6 @@ class Scope;
 }  // namespace framework
 }  // namespace paddle
 
-namespace {
-
-template <typename T1, typename T2>
-void ConvertTensorType(phi::DenseTensor* tensor) {
-  phi::DenseTensor tmp_tensor;
-  tmp_tensor.set_type(phi::CppTypeToDataType<T2>::Type());
-  tmp_tensor.Resize(tensor->dims());
-  auto* tmp_data = tmp_tensor.mutable_data<T2>(paddle::platform::CPUPlace());
-  auto* data = tensor->mutable_data<T1>(paddle::platform::CPUPlace());
-  for (int i = 0; i < tensor->numel(); i++) {
-    tmp_data[i] = static_cast<T2>(data[i]);
-  }
-  tensor->clear();
-  paddle::framework::TensorCopySync(
-      tmp_tensor, paddle::platform::CPUPlace(), tensor);
-}
-
-}  // namespace
-
 namespace paddle {
 namespace framework {
 namespace ir {
@@ -450,7 +431,7 @@ int Conv2dXPUFusePass::ApplyImpl(ir::Graph* graph,
     // conv_filter fp16 --> fp32
     auto tensor_type = filter_t->dtype();
     if (tensor_type == phi::DataType::FLOAT16) {
-      ConvertTensorType<float16, float>(filter_t);
+      CastToFp32(filter_t, nullptr);
     }
     auto filter_dims = filter_t->dims();
     bool has_bias = with_bn || with_conv_bias;
@@ -529,9 +510,6 @@ int Conv2dXPUFusePass::ApplyImpl(ir::Graph* graph,
         }
       }
     }
-    if (tensor_type == phi::DataType::FLOAT16) {
-      ConvertTensorType<float, float16>(filter_t);
-    }
     // filter max
     Node* filter_int16 = nullptr;
     Node* filter_max = nullptr;
diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc
index 9bde77ec8bf0b..4c3688b0badfa 100644
--- a/paddle/phi/backends/xpu/xpu2_op_list.cc
+++ b/paddle/phi/backends/xpu/xpu2_op_list.cc
@@ -164,7 +164,8 @@ XPUOpMap& get_kl2_ops() {
       {"conv3d",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"conv2d_transpose_grad", XPUKernelSet({phi::DataType::FLOAT32})},
-      {"conv2d_transpose", XPUKernelSet({phi::DataType::FLOAT32})},
+      {"conv2d_transpose",
+       XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
       {"cumsum",
        XPUKernelSet({phi::DataType::FLOAT32,
                      phi::DataType::FLOAT16,
diff --git a/paddle/phi/kernels/reduce_mean_kernel.cc b/paddle/phi/kernels/reduce_mean_kernel.cc
index 9ff9bfc933a48..21b02412d31ca 100644
--- a/paddle/phi/kernels/reduce_mean_kernel.cc
+++ b/paddle/phi/kernels/reduce_mean_kernel.cc
@@ -67,5 +67,6 @@ PD_REGISTER_KERNEL(
 #endif
 
 #if defined(PADDLE_WITH_XPU)
-PD_REGISTER_KERNEL(mean, XPU, ALL_LAYOUT, phi::MeanKernel, float) {}
+PD_REGISTER_KERNEL(
+    mean, XPU, ALL_LAYOUT, phi::MeanKernel, float, phi::dtype::float16) {}
 #endif
diff --git a/paddle/phi/kernels/xpu/activation_kernel.cc b/paddle/phi/kernels/xpu/activation_kernel.cc
index 41a1cceb11935..cc2c76c93a387 100644
--- a/paddle/phi/kernels/xpu/activation_kernel.cc
+++ b/paddle/phi/kernels/xpu/activation_kernel.cc
@@ -195,13 +195,6 @@ void PowKernel(const Context& dev_ctx,
                const DenseTensor& x,
                const Scalar& factor,
                DenseTensor* out) {
-  // using XPUType = typename XPUTypeTrait<T>::Type;
-  // // dev_ctx.template Alloc<T>(out);
-  // auto pow_factor = factor.to<T>();
-  // const auto* x_data = reinterpret_cast<const XPUType*>(x.data<T>());
-  // auto* y_data = reinterpret_cast<XPUType*>(dev_ctx.template Alloc<T>(out));
-  // // const T* x_data = x.data<T>();
-  // // T* y_data = out->data<T>();
   dev_ctx.template Alloc<T>(out);
   float pow_factor = factor.to<float>();
   const T* x_data = x.data<T>();
@@ -578,16 +571,11 @@ PD_REGISTER_KERNEL(
 
 PD_REGISTER_ACTIVATION_KERNEL(exp, ExpKernel)  // no grad
 PD_REGISTER_ACTIVATION_KERNEL(floor, FloorKernel)
-// PD_REGISTER_ACTIVATION_KERNEL(leaky_relu, LeakyReluKernel)
-// PD_REGISTER_ACTIVATION_KERNEL(hard_sigmoid, HardSigmoidKernel)
 PD_REGISTER_ACTIVATION_KERNEL(hardswish, HardSwishKernel)
 PD_REGISTER_ACTIVATION_KERNEL(mish, MishKernel)
 PD_REGISTER_ACTIVATION_KERNEL(pow, PowKernel)
 PD_REGISTER_ACTIVATION_KERNEL(reciprocal, ReciprocalKernel)
 PD_REGISTER_ACTIVATION_KERNEL(relu6_raw, Relu6RawKernel)
-// PD_REGISTER_ACTIVATION_KERNEL(sigmoid, SigmoidKernel)
-// PD_REGISTER_ACTIVATION_KERNEL(sqrt, SqrtKernel)
-// PD_REGISTER_ACTIVATION_KERNEL(swish_raw, SwishRawKernel)
 PD_REGISTER_ACTIVATION_KERNEL(softplus, SoftplusKernel)
 PD_REGISTER_ACTIVATION_KERNEL(sin, SinKernel)
 PD_REGISTER_ACTIVATION_KERNEL(cos, CosKernel)
diff --git a/paddle/phi/kernels/xpu/conv_transpose_kernel.cc b/paddle/phi/kernels/xpu/conv_transpose_kernel.cc
index dec56dab994fc..1b3c31f665c7c 100644
--- a/paddle/phi/kernels/xpu/conv_transpose_kernel.cc
+++ b/paddle/phi/kernels/xpu/conv_transpose_kernel.cc
@@ -123,11 +123,11 @@ void Conv2dTransposeKernel(const Context& ctx,
         true);
     PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d_transpose_v2");
   } else {
-    int r = xpu::conv2d_transpose_v2<float, float, float, int16_t>(
+    int r = xpu::conv2d_transpose_v2<XPUT, XPUT, XPUT, int16_t>(
         ctx.x_context(),
-        x.data<float>(),
-        filter_.data<float>(),
-        out->data<float>(),
+        reinterpret_cast<const XPUT*>(x.data<T>()),
+        reinterpret_cast<const XPUT*>(filter.data<T>()),
+        reinterpret_cast<XPUT*>(out->data<T>()),
         batch_size,
         img_yc,
         img_xh,
@@ -148,5 +148,9 @@ void Conv2dTransposeKernel(const Context& ctx,
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(
-    conv2d_transpose, XPU, ALL_LAYOUT, phi::Conv2dTransposeKernel, float) {}
+PD_REGISTER_KERNEL(conv2d_transpose,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::Conv2dTransposeKernel,
+                   float,
+                   phi::dtype::float16) {}
diff --git a/test/xpu/test_batch_norm_op_xpu.py b/test/xpu/test_batch_norm_op_xpu.py
index 6cf666c8094c9..c7a6efecad406 100644
--- a/test/xpu/test_batch_norm_op_xpu.py
+++ b/test/xpu/test_batch_norm_op_xpu.py
@@ -154,7 +154,6 @@ def __init__(self):
     class TestBatchNormOp(unittest.TestCase):
         def setUp(self):
             self.op_type = "batch_norm"
-            self.dtype = np.float32
             self.shape = [2, 3, 4, 5]
             self.data_layout = "NCHW"
             self.epsilon = 1e-05
@@ -162,6 +161,9 @@ def setUp(self):
             self.init_dtype()
             self.set_xpu()
             self.set_attrs()
+            self.rtol = 1e-5
+            if self.dtype == np.float16:
+                self.rtol = 1e-2
 
             if self.data_layout == "NHWC":
                 channel_size = self.shape[3]
@@ -175,15 +177,15 @@ def setUp(self):
             np.random.seed(1024)
             self.x_np = np.random.random_sample(self.shape).astype(self.dtype)
             self.scale_np = np.random.random_sample([channel_size]).astype(
-                self.dtype
+                np.float32
             )
             self.bias_np = np.random.random_sample([channel_size]).astype(
-                self.dtype
+                np.float32
             )
-            self.mean_np = np.zeros([channel_size]).astype(self.dtype)
-            self.variance_np = np.ones([channel_size]).astype(self.dtype)
-            self.saved_mean_np = np.zeros([channel_size]).astype(self.dtype)
-            self.saved_variance_np = np.ones([channel_size]).astype(self.dtype)
+            self.mean_np = np.zeros([channel_size]).astype(np.float32)
+            self.variance_np = np.ones([channel_size]).astype(np.float32)
+            self.saved_mean_np = np.zeros([channel_size]).astype(np.float32)
+            self.saved_variance_np = np.ones([channel_size]).astype(np.float32)
 
         def set_attrs(self):
             pass
@@ -244,7 +246,110 @@ def test_infer(self):
                 self.epsilon,
                 self.data_layout,
             )
-            np.testing.assert_allclose(y_np_ref, y_np, rtol=1e-05)
+            np.testing.assert_allclose(y_np_ref, y_np, rtol=self.rtol)
+
+    class TestBatchNormOpUseGlobalStats(unittest.TestCase):
+        def setUp(self):
+            self.places = [paddle.XPUPlace(0)]
+            self.init_test()
+
+        # train mode
+        def init_test(self):
+            self.use_global_stats = True
+            self.trainable_statistics = False
+
+        def test_global_stats(self):
+            for p in self.places:
+                with fluid.dygraph.guard(p):
+                    x = paddle.randn([2, 6, 6, 4])
+                    net1 = paddle.nn.BatchNorm(
+                        6,
+                        param_attr=fluid.ParamAttr(
+                            initializer=paddle.nn.initializer.Constant(1.0)
+                        ),
+                        use_global_stats=self.use_global_stats,
+                        trainable_statistics=self.trainable_statistics,
+                    )
+                    net2 = paddle.nn.BatchNorm2D(
+                        6, use_global_stats=self.use_global_stats
+                    )
+                    net2.weight = net1.weight
+                    net2.bias = net1.bias
+                    if self.trainable_statistics:
+                        net1.training = False
+                        net2.training = False
+                    y1 = net1(x)
+                    y2 = net2(x)
+                    np.testing.assert_allclose(
+                        y1.numpy(), y2.numpy(), rtol=1e-5
+                    )
+
+    class TestBatchNormOpUseGlobalStats1(TestBatchNormOpUseGlobalStats):
+        # test mode
+        def init_test(self):
+            self.use_global_stats = True
+            self.trainable_statistics = True
+
+    class TestBatchNormUseGlobalStats2(TestBatchNormOpUseGlobalStats):
+        # train mode
+        def init_test(self):
+            self.use_global_stats = True
+            self.trainable_statistics = False
+
+
+support_types = get_xpu_op_support_types('batch_norm')
+for stype in support_types:
+    create_test_class(globals(), XPUTestBatchNormOp, stype)
+
+
+class XPUTestBatchNormGradOp(XPUOpTestWrapper):
+    def __init__(self):
+        self.op_name = 'batch_norm'
+        self.use_dynamic_create_class = False
+
+    class TestBatchNormGradOp(unittest.TestCase):
+        def setUp(self):
+            self.op_type = "batch_norm"
+            self.shape = [2, 3, 4, 5]
+            self.data_layout = "NCHW"
+            self.epsilon = 1e-05
+            self.momentum = 0.9
+            self.init_dtype()
+            self.set_xpu()
+            self.set_attrs()
+
+            if self.data_layout == "NHWC":
+                channel_size = self.shape[3]
+            elif self.data_layout == "NCHW":
+                channel_size = self.shape[1]
+            else:
+                raise ValueError(
+                    "Unsupported data layout! Only NCHW and NHWC is supported, but received "
+                    + self.data_layout
+                )
+            np.random.seed(1024)
+            self.x_np = np.random.random_sample(self.shape).astype(self.dtype)
+            self.scale_np = np.random.random_sample([channel_size]).astype(
+                np.float32
+            )
+            self.bias_np = np.random.random_sample([channel_size]).astype(
+                np.float32
+            )
+            self.mean_np = np.zeros([channel_size]).astype(np.float32)
+            self.variance_np = np.ones([channel_size]).astype(np.float32)
+            self.saved_mean_np = np.zeros([channel_size]).astype(np.float32)
+            self.saved_variance_np = np.ones([channel_size]).astype(np.float32)
+
+        def set_attrs(self):
+            pass
+
+        def init_dtype(self):
+            self.dtype = self.in_type
+
+        def set_xpu(self):
+            self.__class__.use_xpu = True
+            self.__class__.op_type = self.in_type
+            self.place = paddle.XPUPlace(0)
 
         def test_train(self):
             y_grad_np = np.random.random_sample(self.shape).astype(self.dtype)
@@ -349,58 +454,10 @@ def test_train(self):
                         outputs[name], outs[id], rtol=1e-05, atol=1e-4
                     )
 
-    class TestBatchNormOpUseGlobalStats(unittest.TestCase):
-        def setUp(self):
-            self.places = [paddle.XPUPlace(0)]
-            self.init_test()
-
-        # train mode
-        def init_test(self):
-            self.use_global_stats = True
-            self.trainable_statistics = False
-
-        def test_global_stats(self):
-            for p in self.places:
-                with fluid.dygraph.guard(p):
-                    x = paddle.randn([2, 6, 6, 4])
-                    net1 = paddle.nn.BatchNorm(
-                        6,
-                        param_attr=fluid.ParamAttr(
-                            initializer=paddle.nn.initializer.Constant(1.0)
-                        ),
-                        use_global_stats=self.use_global_stats,
-                        trainable_statistics=self.trainable_statistics,
-                    )
-                    net2 = paddle.nn.BatchNorm2D(
-                        6, use_global_stats=self.use_global_stats
-                    )
-                    net2.weight = net1.weight
-                    net2.bias = net1.bias
-                    if self.trainable_statistics:
-                        net1.training = False
-                        net2.training = False
-                    y1 = net1(x)
-                    y2 = net2(x)
-                    np.testing.assert_allclose(
-                        y1.numpy(), y2.numpy(), rtol=1e-05
-                    )
-
-    class TestBatchNormOpUseGlobalStats1(TestBatchNormOpUseGlobalStats):
-        # test mode
-        def init_test(self):
-            self.use_global_stats = True
-            self.trainable_statistics = True
-
-    class TestBatchNormUseGlobalStats2(TestBatchNormOpUseGlobalStats):
-        # train mode
-        def init_test(self):
-            self.use_global_stats = True
-            self.trainable_statistics = False
 
-
-support_types = get_xpu_op_support_types('batch_norm')
-for stype in support_types:
-    create_test_class(globals(), XPUTestBatchNormOp, stype)
+support_types_grad = get_xpu_op_support_types('batch_norm_grad')
+for stype_grad in support_types_grad:
+    create_test_class(globals(), XPUTestBatchNormGradOp, stype_grad)
 
 if __name__ == '__main__':
     unittest.main()

From 65a3a58405b206189b31c054741371accabf4585 Mon Sep 17 00:00:00 2001
From: Yuang Liu <liuyuang@baidu.com>
Date: Wed, 10 May 2023 19:18:38 +0800
Subject: [PATCH 261/405] no value clip for parallel cross entropy (#53547)

---
 .../operators/collective/c_softmax_with_cross_entropy_op.cu | 6 ++----
 .../tests/unittests/c_softmax_with_cross_entropy_op.py      | 6 +++---
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu
index 5a78689819714..3c70b997a7fd8 100644
--- a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu
+++ b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu
@@ -188,8 +188,7 @@ struct CSoftmaxWithCrossEntropyFunctor<phi::GPUContext, T> {
 
     eigen_softmax.device(*dev_ctx.eigen_device()) =
         (eigen_logits -
-         eigen_logits_max.reshape(batch_by_one).broadcast(one_by_class))
-            .unaryExpr(phi::funcs::ValueClip<T>());
+         eigen_logits_max.reshape(batch_by_one).broadcast(one_by_class));
 
     // step 3, obtain predict target
     phi::DenseTensor predicted_logits;
@@ -346,8 +345,7 @@ struct CSoftmaxWithCrossEntropyProcessGroupFunctor<phi::GPUContext, T> {
 
     eigen_softmax.device(*dev_ctx.eigen_device()) =
         (eigen_logits -
-         eigen_logits_max.reshape(batch_by_one).broadcast(one_by_class))
-            .unaryExpr(phi::funcs::ValueClip<T>());
+         eigen_logits_max.reshape(batch_by_one).broadcast(one_by_class));
 
     // step 3, obtain predict target
     phi::DenseTensor predicted_logits;
diff --git a/python/paddle/fluid/tests/unittests/c_softmax_with_cross_entropy_op.py b/python/paddle/fluid/tests/unittests/c_softmax_with_cross_entropy_op.py
index 0217f7ee710f5..684c1c20bb1d7 100644
--- a/python/paddle/fluid/tests/unittests/c_softmax_with_cross_entropy_op.py
+++ b/python/paddle/fluid/tests/unittests/c_softmax_with_cross_entropy_op.py
@@ -26,7 +26,7 @@ def stable_softmax(x):
     """Compute the softmax of vector x in a numerically stable way."""
     # clip to shiftx, otherwise, when calc loss with
     # log(exp(shiftx)), may get log(0)=INF
-    shiftx = (x - np.max(x)).clip(-64.0)
+    shiftx = x - np.max(x)
     exps = np.exp(shiftx)
     return exps / np.sum(exps)
 
@@ -88,13 +88,13 @@ def test_model(self, data_type="float32"):
         # get input data for rank 0
         np.random.seed(0)
         input0 = np.random.uniform(
-            low=-10.0, high=10.0, size=(self.batch_size, local_elements)
+            low=-40.0, high=40.0, size=(self.batch_size, local_elements)
         ).astype(data_type)
 
         # get input data for rank 1
         np.random.seed(1)
         input1 = np.random.uniform(
-            low=-10.0, high=10.0, size=(self.batch_size, local_elements)
+            low=-40.0, high=40.0, size=(self.batch_size, local_elements)
         ).astype(data_type)
 
         # get combined input data

From f3393f494541aca303e3292c1cb78104e90d89c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=82=85=E5=89=91=E5=AF=92?= <Xs1580802568@gmail.com>
Date: Wed, 10 May 2023 19:23:32 +0800
Subject: [PATCH 262/405] add index_put api (#52886)

* add index_put api

* fix value broadcast in backward and add test case in static

* add timeout=120s for index_put

* add op_compat for index_put

* add inplace index_put test

* add test case when index tensor in indices is int32 when indices.size less than x.dims

* add index_put api backward in cpu place

* add backward test case

* refactor code to delete some duplicated code

* replace reshape with resize for decrease extra memcpy

* add datatype flag in backward yaml

* fix bug in documentation

* Update python/paddle/tensor/manipulation.py

---------

Co-authored-by: Ligoml <39876205+Ligoml@users.noreply.github.com>
---
 paddle/phi/api/yaml/backward.yaml             |  11 +
 paddle/phi/api/yaml/ops.yaml                  |  11 +
 paddle/phi/infermeta/multiary.cc              |  16 +-
 paddle/phi/infermeta/multiary.h               |   6 +
 .../phi/kernels/cpu/index_put_grad_kernel.cc  | 225 +++++
 paddle/phi/kernels/cpu/index_put_kernel.cc    | 166 ++++
 paddle/phi/kernels/funcs/index_put_utils.h    | 348 ++++++++
 .../phi/kernels/gpu/index_put_grad_kernel.cu  | 287 ++++++
 paddle/phi/kernels/gpu/index_put_kernel.cu    | 198 +++++
 paddle/phi/kernels/index_put_grad_kernel.h    |  30 +
 paddle/phi/kernels/index_put_kernel.h         |  29 +
 python/paddle/__init__.py                     |   4 +
 .../fluid/tests/unittests/CMakeLists.txt      |   1 +
 .../tests/unittests/test_index_put_op.py      | 826 ++++++++++++++++++
 python/paddle/tensor/__init__.py              |   4 +
 python/paddle/tensor/manipulation.py          | 103 ++-
 16 files changed, 2263 insertions(+), 2 deletions(-)
 create mode 100644 paddle/phi/kernels/cpu/index_put_grad_kernel.cc
 create mode 100644 paddle/phi/kernels/cpu/index_put_kernel.cc
 create mode 100644 paddle/phi/kernels/funcs/index_put_utils.h
 create mode 100644 paddle/phi/kernels/gpu/index_put_grad_kernel.cu
 create mode 100644 paddle/phi/kernels/gpu/index_put_kernel.cu
 create mode 100644 paddle/phi/kernels/index_put_grad_kernel.h
 create mode 100644 paddle/phi/kernels/index_put_kernel.h
 create mode 100644 python/paddle/fluid/tests/unittests/test_index_put_op.py

diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index bde673e60b607..6ae8c19006410 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -796,6 +796,17 @@
     data_type : out_grad
   inplace : (out_grad -> x_grad)
 
+- backward_op : index_put_grad
+  forward : index_put (Tensor x, Tensor[] indices, Tensor value, bool accumulate=false) -> Tensor(out)
+  args : (Tensor x, Tensor[] indices, Tensor value, Tensor out_grad, bool accumulate=false)
+  output : Tensor(x_grad), Tensor(value_grad)
+  infer_meta :
+    func : GeneralBinaryGradInferMeta
+    param : [x, value]
+  kernel :
+    func : index_put_grad
+    data_type : out_grad
+
 - backward_op : index_sample_grad
   forward : index_sample (Tensor x, Tensor index) -> Tensor(out)
   args : (Tensor x, Tensor index, Tensor out_grad)
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index c254a75f98e17..de9d6c4f29c57 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -940,6 +940,17 @@
   inplace : (x -> out)
   backward : index_add_grad
 
+- op : index_put
+  args : (Tensor x, Tensor[] indices, Tensor value, bool accumulate=false)
+  output : Tensor(out)
+  infer_meta :
+    func : IndexPutInferMeta
+  kernel :
+    func : index_put
+    data_type : x
+  inplace : (x -> out)
+  backward : index_put_grad
+
 - op : index_sample
   args : (Tensor x, Tensor index)
   output : Tensor
diff --git a/paddle/phi/infermeta/multiary.cc b/paddle/phi/infermeta/multiary.cc
index 5a8e38e21fd72..8ea2dc65d9a54 100644
--- a/paddle/phi/infermeta/multiary.cc
+++ b/paddle/phi/infermeta/multiary.cc
@@ -1962,6 +1962,21 @@ void InterpolateInferMeta(
   }
 }
 
+void IndexPutInferMeta(const MetaTensor& x,
+                       const std::vector<const MetaTensor*>& indices,
+                       const MetaTensor& value,
+                       bool accumulate,
+                       MetaTensor* out) {
+  auto in_dims = x.dims();
+  PADDLE_ENFORCE_LT(
+      in_dims.size(),
+      7,
+      phi::errors::InvalidArgument(
+          "The rank of input should be less than 7, but received %d.",
+          in_dims.size()));
+  out->share_meta(x);
+}
+
 void LambInferMeta(const MetaTensor& param,
                    const MetaTensor& grad,
                    const MetaTensor& learning_rate,
@@ -3295,6 +3310,5 @@ void WeightedSampleNeighborsInferMeta(const MetaTensor& row,
   out_count->set_dims({-1});
   out_count->set_dtype(DataType::INT32);
 }
-
 }  // namespace phi
 PD_REGISTER_INFER_META_FN(batch_norm_infer, phi::BatchNormInferInferMeta);
diff --git a/paddle/phi/infermeta/multiary.h b/paddle/phi/infermeta/multiary.h
index 993e6c21ff6ff..2a924ecbb30a0 100644
--- a/paddle/phi/infermeta/multiary.h
+++ b/paddle/phi/infermeta/multiary.h
@@ -332,6 +332,12 @@ void InterpolateInferMeta(
     MetaTensor* output,
     MetaConfig config = MetaConfig());
 
+void IndexPutInferMeta(const MetaTensor& x,
+                       const std::vector<const MetaTensor*>& indices,
+                       const MetaTensor& value,
+                       bool accumulate,
+                       MetaTensor* out);
+
 void LambInferMeta(const MetaTensor& param,
                    const MetaTensor& grad,
                    const MetaTensor& learning_rate,
diff --git a/paddle/phi/kernels/cpu/index_put_grad_kernel.cc b/paddle/phi/kernels/cpu/index_put_grad_kernel.cc
new file mode 100644
index 0000000000000..7374bcd403d12
--- /dev/null
+++ b/paddle/phi/kernels/cpu/index_put_grad_kernel.cc
@@ -0,0 +1,225 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/index_put_grad_kernel.h"
+#include <numeric>
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/cast_kernel.h"
+#include "paddle/phi/kernels/funcs/index_put_utils.h"
+#include "paddle/phi/kernels/reduce_sum_kernel.h"
+
+namespace phi {
+
+template <typename T>
+void set_zero_kernel(const int64_t N,
+                     const int64_t** indices,
+                     const phi::DDim& stride,
+                     const phi::DDim& shape,
+                     T* out) {
+#ifdef PADDLE_WITH_MKLML
+#pragma omp parallel for
+#endif
+  for (int64_t idx = 0; idx < N; ++idx) {
+    int64_t cur_ix = 0;
+    int64_t offset = 0;
+
+    for (int i = 0; i < shape.size(); ++i) {
+      cur_ix = (static_cast<int64_t>(*(indices[i] + idx)));
+      if (cur_ix < 0) {
+        cur_ix += shape[i];
+      }
+      offset += stride[i] * cur_ix;
+    }
+    *(out + offset) = 0;
+  }
+}
+
+template <typename T>
+void index_put_grad_kernel(const int64_t N,
+                           const T* out_grad,
+                           const int64_t** indices,
+                           const phi::DDim& stride,
+                           const phi::DDim& shape,
+                           T* value_grad) {
+#ifdef PADDLE_WITH_MKLML
+#pragma omp parallel for
+#endif
+  for (int64_t idx = 0; idx < N; ++idx) {
+    int64_t cur_ix = 0;
+    int64_t offset = 0;
+
+    for (int i = 0; i < shape.size(); ++i) {
+      cur_ix = (static_cast<int64_t>(*(indices[i] + idx)));
+      if (cur_ix < 0) {
+        cur_ix += shape[i];
+      }
+      offset += stride[i] * cur_ix;
+    }
+    *(value_grad + idx) = *(out_grad + offset);
+  }
+}
+
+template <typename T, typename Context>
+void LaunchIndexPutGradKernel(const Context& dev_ctx,
+                              const std::vector<const DenseTensor*>& indices,
+                              const DenseTensor& out_grad,
+                              bool accumulate,
+                              DenseTensor* value_grad,
+                              DenseTensor* x_grad) {
+  const int64_t* pd_indices[7];
+  for (size_t i = 0; i < indices.size(); ++i) {
+    pd_indices[i] = indices[i]->data<int64_t>();
+  }
+
+  if (x_grad) {
+    phi::Copy(dev_ctx, out_grad, dev_ctx.GetPlace(), false, x_grad);
+    if (!accumulate) {
+      T* x_grad_data = x_grad->data<T>();
+
+      auto x_grad_dims = x_grad->dims();
+      const int64_t numel = indices[0]->numel();
+      auto x_grad_stride = phi::stride(x_grad_dims);
+
+      set_zero_kernel<T>(
+          numel, pd_indices, x_grad_stride, x_grad_dims, x_grad_data);
+    }
+  }
+
+  auto out_grad_dims = out_grad.dims();
+  const int64_t numel = indices[0]->numel();
+  auto out_grad_stride = phi::stride(out_grad_dims);
+
+  if (value_grad) {
+    if (value_grad->numel() == 1) {
+      DenseTensor tmp_value_grad(value_grad->dtype());
+      tmp_value_grad.Resize(indices[0]->dims());
+
+      T* tmp_value_grad_data = dev_ctx.template Alloc<T>(&tmp_value_grad);
+      auto out_grad_data = out_grad.data<T>();
+
+      index_put_grad_kernel<T>(numel,
+                               out_grad_data,
+                               pd_indices,
+                               out_grad_stride,
+                               out_grad_dims,
+                               tmp_value_grad_data);
+
+      std::vector<int> v_dims(tmp_value_grad.dims().size());
+      std::iota(v_dims.begin(), v_dims.end(), 0);
+      IntArray v_axis(v_dims);
+      SumKernel<T>(dev_ctx,
+                   tmp_value_grad,
+                   v_axis,
+                   value_grad->dtype(),
+                   false,
+                   value_grad);
+    } else if (value_grad->numel() == indices[0]->numel()) {
+      T* value_grad_data = dev_ctx.template Alloc<T>(value_grad);
+      auto out_grad_data = out_grad.data<T>();
+
+      index_put_grad_kernel<T>(numel,
+                               out_grad_data,
+                               pd_indices,
+                               out_grad_stride,
+                               out_grad_dims,
+                               value_grad_data);
+    } else {
+      DenseTensor tmp_value_grad(value_grad->dtype());
+      tmp_value_grad.Resize(indices[0]->dims());
+
+      T* tmp_value_grad_data = dev_ctx.template Alloc<T>(&tmp_value_grad);
+      auto out_grad_data = out_grad.data<T>();
+
+      index_put_grad_kernel<T>(numel,
+                               out_grad_data,
+                               pd_indices,
+                               out_grad_stride,
+                               out_grad_dims,
+                               tmp_value_grad_data);
+
+      std::vector<int64_t> after_dims = phi::vectorize(tmp_value_grad.dims());
+      std::vector<int64_t> before_dims = phi::vectorize(value_grad->dims());
+      std::vector<int64_t> compress_dims;
+      std::vector<int64_t> dims_without_1;
+
+      funcs::CalCompressedDimsWith1AndWithout1(
+          &after_dims, &before_dims, &compress_dims, &dims_without_1);
+
+      auto pre_dims = value_grad->dims();
+      value_grad->Resize(phi::make_ddim(dims_without_1));
+      IntArray v_axis(compress_dims);
+      SumKernel<T>(dev_ctx,
+                   tmp_value_grad,
+                   v_axis,
+                   value_grad->dtype(),
+                   false,
+                   value_grad);
+      value_grad->Resize(pre_dims);
+    }
+  }
+}
+
+template <typename T, typename Context>
+void IndexPutGradKernel(const Context& dev_ctx,
+                        const DenseTensor& x,
+                        const std::vector<const DenseTensor*>& indices,
+                        const DenseTensor& value,
+                        const DenseTensor& out_grad,
+                        bool accumulate,
+                        DenseTensor* x_grad,
+                        DenseTensor* value_grad) {
+  PADDLE_ENFORCE_EQ(
+      x.dtype(),
+      value.dtype(),
+      phi::errors::InvalidArgument(
+          "The data type of tensor in indices must be same to the data type "
+          "of tensor x."));
+  std::vector<DenseTensor> tmp_args;
+  std::vector<const phi::DenseTensor*> int_indices_v =
+      funcs::DealWithBoolIndices<T, Context>(dev_ctx, indices, &tmp_args);
+  auto bd_dim = funcs::BroadCastTensorsDims(int_indices_v);
+
+  std::vector<int64_t> res_dim_v(phi::vectorize(bd_dim));
+  std::vector<const phi::DenseTensor*> res_indices_v(x.dims().size(), nullptr);
+  std::vector<DenseTensor> tmp_res_indices_v;
+  std::vector<DenseTensor> range_tensor_v;
+
+  for (int i = indices.size(); i < x.dims().size(); ++i) {
+    range_tensor_v.emplace_back(funcs::GetRangeTensor<int64_t, Context>(
+        dev_ctx, x.dims()[i], phi::DataType::INT64));
+  }
+
+  funcs::DealWithIndices<T, Context>(dev_ctx,
+                                     x,
+                                     int_indices_v,
+                                     &res_indices_v,
+                                     &tmp_res_indices_v,
+                                     range_tensor_v,
+                                     bd_dim,
+                                     &res_dim_v);
+
+  LaunchIndexPutGradKernel<T, Context>(
+      dev_ctx, res_indices_v, out_grad, accumulate, value_grad, x_grad);
+}
+}  // namespace phi
+
+PD_REGISTER_KERNEL(index_put_grad,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::IndexPutGradKernel,
+                   float,
+                   double,
+                   int,
+                   int64_t,
+                   bool) {}
diff --git a/paddle/phi/kernels/cpu/index_put_kernel.cc b/paddle/phi/kernels/cpu/index_put_kernel.cc
new file mode 100644
index 0000000000000..da3e37ac24236
--- /dev/null
+++ b/paddle/phi/kernels/cpu/index_put_kernel.cc
@@ -0,0 +1,166 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/index_put_kernel.h"
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/cast_kernel.h"
+#include "paddle/phi/kernels/funcs/index_put_utils.h"
+
+namespace phi {
+
+template <typename T>
+void index_put_kernel(const int64_t N,
+                      const T* x,
+                      const T* vals,
+                      const int64_t** indices,
+                      const phi::DDim& stride,
+                      const phi::DDim& shape,
+                      int64_t is_single_val_tensor,
+                      bool accumulate,
+                      T* out) {
+#ifdef PADDLE_WITH_MKLML
+#pragma omp parallel for
+#endif
+  for (int64_t idx = 0; idx < N; ++idx) {
+    int64_t cur_ix = 0;
+    int64_t offset = 0;
+
+    for (int i = 0; i < shape.size(); ++i) {
+      cur_ix = (static_cast<int64_t>(*(indices[i] + idx)));
+      if (cur_ix < 0) {
+        cur_ix += shape[i];
+      }
+      offset += stride[i] * cur_ix;
+    }
+
+    if (accumulate) {
+      *(out + offset) += *(vals + (idx & is_single_val_tensor));
+    } else {
+      *(out + offset) = *(vals + (idx & is_single_val_tensor));
+    }
+  }
+}
+
+template <typename T, typename Context>
+void LaunchIndexPutKernel(const Context& dev_ctx,
+                          const DenseTensor& x,
+                          const std::vector<const DenseTensor*>& indices,
+                          const DenseTensor& value,
+                          bool accumulate,
+                          DenseTensor* out) {
+  auto* x_data = x.data<T>();
+  auto* val_data = value.data<T>();
+  bool is_initialized = out->initialized();
+  T* out_data = dev_ctx.template Alloc<T>(out);
+
+  if (!is_initialized) {
+    phi::Copy(dev_ctx, x, dev_ctx.GetPlace(), false, out);
+  }
+
+  auto x_dims = x.dims();
+  const int64_t numel = indices[0]->numel();
+  auto x_stride = phi::stride(x_dims);
+
+  int64_t is_single_val_tensor = (value.numel() == 1) ? 0 : INT64_MAX;
+
+  const int64_t* pd_indices[7];
+  for (size_t i = 0; i < indices.size(); ++i) {
+    pd_indices[i] = indices[i]->data<int64_t>();
+  }
+
+  index_put_kernel<T>(numel,
+                      x_data,
+                      val_data,
+                      pd_indices,
+                      x_stride,
+                      x_dims,
+                      is_single_val_tensor,
+                      accumulate,
+                      out_data);
+}
+
+template <typename T, typename Context>
+void IndexPutKernel(const Context& dev_ctx,
+                    const DenseTensor& x,
+                    const std::vector<const DenseTensor*>& indices,
+                    const DenseTensor& value,
+                    bool accumulate,
+                    DenseTensor* out) {
+  PADDLE_ENFORCE_EQ(
+      x.dtype(),
+      value.dtype(),
+      phi::errors::InvalidArgument(
+          "The data type of tensor in indices must be same to the data type "
+          "of tensor x."));
+  PADDLE_ENFORCE_EQ(indices.empty(),
+                    false,
+                    phi::errors::InvalidArgument("Indices cannot be empty."));
+
+  const size_t total_dims = x.dims().size();
+  PADDLE_ENFORCE_LE(total_dims,
+                    6,
+                    phi::errors::InvalidArgument(
+                        "Dims of input tensor should be less than 7."));
+
+  std::vector<DenseTensor> tmp_args;
+  std::vector<const phi::DenseTensor*> int_indices_v =
+      funcs::DealWithBoolIndices<T, Context>(dev_ctx, indices, &tmp_args);
+
+  auto bd_dim = funcs::BroadCastTensorsDims(int_indices_v);
+
+  std::vector<int64_t> res_dim_v(phi::vectorize(bd_dim));
+  std::vector<const phi::DenseTensor*> res_indices_v(x.dims().size(), nullptr);
+  std::vector<DenseTensor> tmp_res_indices_v;
+  std::vector<DenseTensor> tmp_value_v;
+  std::vector<DenseTensor> range_tensor_v;
+  const DenseTensor* ptr_value = nullptr;
+
+  for (int i = indices.size(); i < x.dims().size(); ++i) {
+    range_tensor_v.emplace_back(funcs::GetRangeTensor<int64_t, Context>(
+        dev_ctx, x.dims()[i], phi::DataType::INT64));
+  }
+
+  funcs::DealWithIndices<T, Context>(dev_ctx,
+                                     x,
+                                     int_indices_v,
+                                     &res_indices_v,
+                                     &tmp_res_indices_v,
+                                     range_tensor_v,
+                                     bd_dim,
+                                     &res_dim_v);
+  if (value.numel() != 1) {
+    tmp_value_v.emplace_back(
+        DenseTensor(value.dtype()).Resize(phi::make_ddim(res_dim_v)));
+    ExpandKernel<T, Context>(
+        dev_ctx, value, IntArray(res_dim_v), &tmp_value_v[0]);
+    ptr_value = &tmp_value_v[0];
+  } else {
+    ptr_value = &value;
+  }
+
+  LaunchIndexPutKernel<T, Context>(
+      dev_ctx, x, res_indices_v, *ptr_value, accumulate, out);
+}
+}  // namespace phi
+
+PD_REGISTER_KERNEL(index_put,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::IndexPutKernel,
+                   float,
+                   double,
+                   int,
+                   int64_t,
+                   bool) {}
diff --git a/paddle/phi/kernels/funcs/index_put_utils.h b/paddle/phi/kernels/funcs/index_put_utils.h
new file mode 100644
index 0000000000000..51e918c852347
--- /dev/null
+++ b/paddle/phi/kernels/funcs/index_put_utils.h
@@ -0,0 +1,348 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <vector>
+#include "paddle/phi/common/int_array.h"
+#include "paddle/phi/common/memory_utils.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/utils/array.h"
+#include "paddle/phi/kernels/cast_kernel.h"
+#include "paddle/phi/kernels/expand_kernel.h"
+#include "paddle/phi/kernels/nonzero_kernel.h"
+#include "paddle/phi/kernels/reshape_kernel.h"
+#include "paddle/phi/kernels/split_kernel.h"
+
+#if defined(__NVCC__) || defined(__HIPCC__)
+#ifdef __NVCC__
+#include <cuda.h>
+#include <cuda_runtime.h>
+#elif defined(__HIPCC__)
+#include <hip/hip_runtime.h>
+#endif
+#endif
+
+namespace phi {
+
+namespace funcs {
+
+template <typename T, typename Context>
+phi::DenseTensor GetReshapeAndExpandTensor(const Context& dev_ctx,
+                                           const phi::DenseTensor& tensor,
+                                           const phi::DDim& res_dim,
+                                           const phi::DDim& bd_dim,
+                                           int index) {
+  std::vector<int64_t> before_dims = phi::vectorize(tensor.dims());
+  std::vector<int64_t> mid_dims(res_dim.size(), 1);
+
+  if (index == 0) {
+    for (size_t i = 0; i < before_dims.size(); ++i) {
+      mid_dims[bd_dim.size() - i - 1] = before_dims[before_dims.size() - i - 1];
+    }
+  } else {
+    mid_dims[index] = before_dims[0];
+  }
+
+  phi::DenseTensor mid_tensor(tensor.dtype());
+  mid_tensor.Resize(phi::make_ddim(mid_dims));
+  ReshapeInferKernel<Context>(dev_ctx, tensor, IntArray(mid_dims), &mid_tensor);
+
+  phi::DenseTensor res_tensor(tensor.dtype());
+  res_tensor.Resize(res_dim);
+  ExpandKernel<T, Context>(
+      dev_ctx, mid_tensor, IntArray(phi::vectorize(res_dim)), &res_tensor);
+  return res_tensor;
+}
+
+template <typename T, typename Context>
+std::vector<const phi::DenseTensor*> DealWithBoolIndices(
+    const Context& dev_ctx,
+    const std::vector<const phi::DenseTensor*>& indices_v,
+    std::vector<phi::DenseTensor>* tmp_indices_v) {
+  std::vector<const phi::DenseTensor*> res(indices_v.begin(), indices_v.end());
+  bool contains_bool_tensor = false;
+  for (size_t i = 0; i < indices_v.size(); ++i) {
+    if (indices_v[i]->dtype() == phi::DataType::BOOL) {
+      contains_bool_tensor = true;
+    } else if ((indices_v[i]->dtype() == phi::DataType::INT64) ||
+               (indices_v[i]->dtype() == phi::DataType::INT32)) {
+      PADDLE_ENFORCE_EQ(
+          contains_bool_tensor,
+          false,
+          phi::errors::InvalidArgument(
+              "indices contains bool tensor and int32/int64 tensor at the same "
+              "time"));
+    } else {
+      PADDLE_THROW(phi::errors::InvalidArgument(
+          "data type of tensor in indices must be int32, int64 or bool"));
+    }
+  }
+
+  if (contains_bool_tensor) {
+    if (indices_v.size() != 1) {
+      PADDLE_THROW(phi::errors::InvalidArgument(
+          "the size of indices must be 1 when it containts bool tensor"));
+    }
+    int rank = indices_v[0]->dims().size();
+    PADDLE_ENFORCE_GE(
+        rank,
+        1UL,
+        phi::errors::InvalidArgument("the only bool tensor in indices should "
+                                     "have number of dimension at least 1"));
+    phi::DenseTensor nonzero_indices(phi::DataType::INT64);
+    nonzero_indices.Resize(phi::make_ddim({-1, rank}));
+    NonZeroKernel<bool, Context>(dev_ctx, *indices_v[0], &nonzero_indices);
+
+    std::vector<phi::DenseTensor*> integer_indices(rank, nullptr);
+    for (int i = 0; i < rank; ++i) {
+      tmp_indices_v->emplace_back(
+          DenseTensor(phi::DataType::INT64)
+              .Resize(phi::make_ddim({nonzero_indices.dims()[0]})));
+    }
+    for (int i = 0; i < rank; ++i) {
+      integer_indices[i] = &((*tmp_indices_v)[i]);
+    }
+    SplitWithNumKernel<int64_t, Context>(
+        dev_ctx, nonzero_indices, rank, 1, integer_indices);
+
+    std::vector<const phi::DenseTensor*> res_tmp(integer_indices.size(),
+                                                 nullptr);
+    for (int i = 0; i < rank; ++i) {
+      res_tmp[i] = &((*tmp_indices_v)[i]);
+    }
+    res.swap(res_tmp);
+  }
+  return res;
+}
+
+static phi::DDim BroadCastTensorsDims(
+    const std::vector<const phi::DenseTensor*>& tensors) {
+  int target_rank = 0;
+  for (const auto& tensor : tensors) {
+    target_rank = std::max(target_rank, tensor->dims().size());
+  }
+
+  PADDLE_ENFORCE_GT(target_rank,
+                    0,
+                    errors::InvalidArgument("BroadCastTensorsDims requires at "
+                                            "least one input tensor to have "
+                                            "rank greater than zero"));
+
+  std::vector<int64_t> target_dims(target_rank, 0);
+  for (int index = 0; index < target_rank; index++) {
+    int target_dim_size = 1;
+    for (const auto& tensor : tensors) {
+      auto input_ddim = tensor->dims();
+      int axis = static_cast<int>(input_ddim.size()) - index - 1;
+      int dim_size = 1;
+      if (axis >= 0) {
+        dim_size = input_ddim[axis];
+      }
+
+      if (target_dim_size != 1 && dim_size != 1 &&
+          target_dim_size != dim_size) {
+        PADDLE_THROW(errors::InvalidArgument(
+            "BroadCastTensorsDims inputs does not satisfy bcast semantics, "
+            "please check axis = %d in reverse order",
+            index));
+      }
+
+      target_dim_size = dim_size == 1 ? target_dim_size : dim_size;
+    }
+    target_dims[target_rank - index - 1] = target_dim_size;
+  }
+  return phi::make_ddim(target_dims);
+}
+
+template <typename T, typename Context>
+T** GetDevicePointerArray(const Context& ctx,
+                          const std::vector<const DenseTensor*>& indices_v) {
+  std::vector<const T*> h_indices_v(indices_v.size());
+  for (int i = 0; i < indices_v.size(); ++i) {
+    h_indices_v[i] = indices_v[i]->data<T>();
+  }
+  auto d_indices_data = phi::memory_utils::Alloc(
+      ctx.GetPlace(),
+      h_indices_v.size() * sizeof(T*),
+      phi::Stream(reinterpret_cast<phi::StreamId>(ctx.stream())));
+  phi::memory_utils::Copy(ctx.GetPlace(),
+                          d_indices_data->ptr(),
+                          phi::CPUPlace(),
+                          reinterpret_cast<void*>(h_indices_v.data()),
+                          h_indices_v.size() * sizeof(T*),
+                          ctx.stream());
+  return reinterpret_cast<T**>(d_indices_data->ptr());
+}
+
+template <typename T, typename Context>
+void DealWithIndices(const Context& dev_ctx,
+                     const DenseTensor& x,
+                     const std::vector<const phi::DenseTensor*>& int_indices_v,
+                     std::vector<const phi::DenseTensor*>* res_indices_v,
+                     std::vector<DenseTensor>* tmp_res_indices_v,
+                     const std::vector<DenseTensor>& range_tensor_v,
+                     const phi::DDim& bd_dim,
+                     std::vector<int64_t>* res_dim_v) {
+  size_t total_dims = x.dims().size();
+  if (int_indices_v.size() < total_dims) {
+    std::vector<int64_t> tmp_x_dims = phi::vectorize(x.dims());
+    int len_bd_dim = bd_dim.size();
+    res_dim_v->insert(res_dim_v->end(),
+                      tmp_x_dims.begin() + int_indices_v.size(),
+                      tmp_x_dims.end());
+
+    std::vector<DenseTensor> reshaped_indices_v;
+    for (size_t i = 0; i < int_indices_v.size(); ++i) {
+      if (int_indices_v[i]->dtype() == phi::DataType::INT32) {
+        reshaped_indices_v.emplace_back(phi::Cast<int, Context>(
+            dev_ctx, *int_indices_v[i], phi::DataType::INT64));
+      } else {
+        reshaped_indices_v.emplace_back(*int_indices_v[i]);
+      }
+    }
+    reshaped_indices_v.insert(
+        reshaped_indices_v.end(), range_tensor_v.begin(), range_tensor_v.end());
+
+    phi::DDim res_dim = phi::make_ddim(*res_dim_v);
+
+    for (size_t i = 0; i < reshaped_indices_v.size(); ++i) {
+      tmp_res_indices_v->emplace_back(
+          GetReshapeAndExpandTensor<int64_t, Context>(
+              dev_ctx,
+              reshaped_indices_v[i],
+              res_dim,
+              bd_dim,
+              ((i < int_indices_v.size())
+                   ? 0
+                   : i - int_indices_v.size() + len_bd_dim)));
+    }
+    for (size_t i = 0; i < res_indices_v->size(); ++i) {
+      (*res_indices_v)[i] = &(*tmp_res_indices_v)[i];
+    }
+
+  } else {
+    std::vector<DenseTensor> int_indices_v_tmp;
+
+    for (size_t i = 0; i < int_indices_v.size(); ++i) {
+      if (int_indices_v[i]->dtype() == phi::DataType::INT32) {
+        int_indices_v_tmp.emplace_back(phi::Cast<int, Context>(
+            dev_ctx, *int_indices_v[i], phi::DataType::INT64));
+      } else {
+        int_indices_v_tmp.emplace_back(*int_indices_v[i]);
+      }
+    }
+
+    for (size_t i = 0; i < int_indices_v.size(); ++i) {
+      if (bd_dim != int_indices_v[i]->dims()) {
+        tmp_res_indices_v->emplace_back(
+            DenseTensor(phi::DataType::INT64).Resize(bd_dim));
+        ExpandKernel<int64_t, Context>(
+            dev_ctx,
+            int_indices_v_tmp[i],
+            IntArray(phi::vectorize<int64_t>(bd_dim)),
+            &(*tmp_res_indices_v)[i]);
+      } else {
+        tmp_res_indices_v->emplace_back(int_indices_v_tmp[i]);
+      }
+    }
+
+    for (size_t i = 0; i < res_indices_v->size(); ++i) {
+      (*res_indices_v)[i] = &(*tmp_res_indices_v)[i];
+    }
+  }
+}
+
+static void CalCompressedDimsWith1AndWithout1(
+    std::vector<int64_t>* after_dims,
+    std::vector<int64_t>* before_dims,
+    std::vector<int64_t>* compress_dims,
+    std::vector<int64_t>* dims_without_1) {
+  int i = static_cast<int>(after_dims->size()) - 1;
+  int j = static_cast<int>(before_dims->size()) - 1;
+  if (i < j) {
+    PADDLE_THROW(phi::errors::InvalidArgument(
+        "shape of value can't not be broadcast to shape of x[indices]"));
+  }
+
+  while ((i >= 0) && (j >= 0)) {
+    if ((*after_dims)[i] == (*before_dims)[j]) {
+      dims_without_1->push_back((*before_dims)[j]);
+      i--;
+      j--;
+      continue;
+    } else if ((*before_dims)[j] == 1) {
+      compress_dims->push_back(i);
+      i--;
+      j--;
+    } else {
+      PADDLE_THROW(phi::errors::InvalidArgument(
+          "shape of value can't not be broadcast to shape of x[indices]"));
+    }
+  }
+  while (i >= 0) {
+    compress_dims->push_back(i);
+    i--;
+  }
+}
+
+#if defined(__NVCC__) || defined(__HIPCC__)
+template <typename T>
+__global__ void range_cuda_kernel(int64_t N, T* out) {
+  int64_t idx = threadIdx.x + blockDim.x * blockIdx.x;
+
+  if (idx >= N) {
+    return;
+  }
+  out[idx] = idx;
+}
+
+template <typename T, typename Context>
+phi::DenseTensor GetRangeCudaTensor(const Context& dev_ctx,
+                                    int64_t N,
+                                    phi::DataType dtype) {
+  phi::DenseTensor res(dtype);
+  res.Resize(phi::make_ddim({N}));
+  DenseTensor* p_res = &res;
+  T* out = dev_ctx.template Alloc<T>(p_res);
+  auto config = phi::backends::gpu::GetGpuLaunchConfig1D(dev_ctx, N);
+  range_cuda_kernel<T>
+      <<<config.block_per_grid, config.thread_per_block, 0, dev_ctx.stream()>>>(
+          N, out);
+  return res;
+}
+#endif
+
+template <typename T>
+void range_kernel(int64_t N, T* out) {
+  for (int64_t idx = 0; idx < N; ++idx) {
+    out[idx] = idx;
+  }
+}
+
+template <typename T, typename Context>
+phi::DenseTensor GetRangeTensor(const Context& dev_ctx,
+                                int64_t N,
+                                phi::DataType dtype) {
+  phi::DenseTensor res(dtype);
+  res.Resize(phi::make_ddim({N}));
+  DenseTensor* p_res = &res;
+  T* out = dev_ctx.template Alloc<T>(p_res);
+  range_kernel<T>(N, out);
+  return res;
+}
+
+}  // namespace funcs
+}  // namespace phi
diff --git a/paddle/phi/kernels/gpu/index_put_grad_kernel.cu b/paddle/phi/kernels/gpu/index_put_grad_kernel.cu
new file mode 100644
index 0000000000000..7ae1e42c067cb
--- /dev/null
+++ b/paddle/phi/kernels/gpu/index_put_grad_kernel.cu
@@ -0,0 +1,287 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/index_put_grad_kernel.h"
+#include <numeric>
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/backends/gpu/gpu_launch_config.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/cast_kernel.h"
+#include "paddle/phi/kernels/funcs/index_put_utils.h"
+#include "paddle/phi/kernels/reduce_sum_kernel.h"
+
+namespace phi {
+
+template <typename T, size_t Rank>
+__global__ void set_zero_cuda_kernel(const int64_t N,
+                                     int64_t** indices,
+                                     phi::Array<int64_t, Rank> stride,
+                                     phi::Array<int64_t, Rank> shape,
+                                     T* out) {
+  int64_t idx = threadIdx.x + blockDim.x * blockIdx.x;
+  int64_t cur_ix = 0;
+
+  if (idx >= N) {
+    return;
+  }
+  int64_t offset = 0;
+  for (int i = 0; i < Rank; ++i) {
+    cur_ix = (static_cast<int64_t>(*(indices[i] + idx)));
+    if (cur_ix < 0) {
+      cur_ix += shape[i];
+    }
+    offset += stride[i] * cur_ix;
+  }
+
+  *(out + offset) = 0;
+}
+
+template <typename T, size_t Rank>
+__global__ void index_put_grad_cuda_kernel(const int64_t N,
+                                           const T* out_grad,
+                                           int64_t** indices,
+                                           phi::Array<int64_t, Rank> stride,
+                                           phi::Array<int64_t, Rank> shape,
+                                           T* value_grad) {
+  int64_t idx = threadIdx.x + blockDim.x * blockIdx.x;
+  int64_t cur_ix = 0;
+
+  if (idx >= N) {
+    return;
+  }
+  int64_t offset = 0;
+  for (int i = 0; i < Rank; ++i) {
+    cur_ix = (static_cast<int64_t>(*(indices[i] + idx)));
+    if (cur_ix < 0) {
+      cur_ix += shape[i];
+    }
+    offset += stride[i] * cur_ix;
+  }
+
+  *(value_grad + idx) = *(out_grad + offset);
+}
+
+template <typename T, typename Context, size_t Rank>
+void LaunchIndexPutGradCudaKernel(
+    const Context& dev_ctx,
+    const std::vector<const DenseTensor*>& indices,
+    const DenseTensor& out_grad,
+    bool accumulate,
+    DenseTensor* value_grad,
+    DenseTensor* x_grad) {
+  if (x_grad) {
+    phi::Copy(dev_ctx, out_grad, dev_ctx.GetPlace(), false, x_grad);
+    if (!accumulate) {
+      T* x_grad_data = x_grad->data<T>();
+
+      auto x_grad_dims = x_grad->dims();
+      const int64_t numel = indices[0]->numel();
+      auto config = phi::backends::gpu::GetGpuLaunchConfig1D(dev_ctx, numel);
+      auto x_grad_stride = phi::stride(x_grad_dims);
+
+      phi::Array<int64_t, Rank> stride_a;
+      phi::Array<int64_t, Rank> shape_a;
+
+      for (size_t idx = 0; idx < Rank; ++idx) {
+        stride_a[idx] = x_grad_stride[idx];
+        shape_a[idx] = x_grad_dims[idx];
+      }
+
+      auto pd_indices =
+          funcs::GetDevicePointerArray<int64_t, Context>(dev_ctx, indices);
+      set_zero_cuda_kernel<T, Rank><<<config.block_per_grid,
+                                      config.thread_per_block,
+                                      0,
+                                      dev_ctx.stream()>>>(
+          numel, pd_indices, stride_a, shape_a, x_grad_data);
+    }
+  }
+
+  auto out_grad_dims = out_grad.dims();
+  const int64_t numel = indices[0]->numel();
+  auto config = phi::backends::gpu::GetGpuLaunchConfig1D(dev_ctx, numel);
+  auto out_grad_stride = phi::stride(out_grad_dims);
+
+  phi::Array<int64_t, Rank> stride_a;
+  phi::Array<int64_t, Rank> shape_a;
+
+  for (size_t idx = 0; idx < Rank; ++idx) {
+    stride_a[idx] = out_grad_stride[idx];
+    shape_a[idx] = out_grad_dims[idx];
+  }
+
+  auto pd_indices =
+      funcs::GetDevicePointerArray<int64_t, Context>(dev_ctx, indices);
+
+  if (value_grad) {
+    if (value_grad->numel() == 1) {
+      DenseTensor tmp_value_grad(value_grad->dtype());
+      tmp_value_grad.Resize(indices[0]->dims());
+
+      T* tmp_value_grad_data = dev_ctx.template Alloc<T>(&tmp_value_grad);
+      auto out_grad_data = out_grad.data<T>();
+
+      index_put_grad_cuda_kernel<T, Rank>
+          <<<config.block_per_grid,
+             config.thread_per_block,
+             0,
+             dev_ctx.stream()>>>(numel,
+                                 out_grad_data,
+                                 pd_indices,
+                                 stride_a,
+                                 shape_a,
+                                 tmp_value_grad_data);
+
+      std::vector<int> v_dims(tmp_value_grad.dims().size());
+      std::iota(v_dims.begin(), v_dims.end(), 0);
+      IntArray v_axis(v_dims);
+      SumKernel<T, Context>(dev_ctx,
+                            tmp_value_grad,
+                            v_axis,
+                            value_grad->dtype(),
+                            false,
+                            value_grad);
+    } else if (value_grad->numel() == indices[0]->numel()) {
+      T* value_grad_data = dev_ctx.template Alloc<T>(value_grad);
+      auto out_grad_data = out_grad.data<T>();
+
+      index_put_grad_cuda_kernel<T, Rank><<<config.block_per_grid,
+                                            config.thread_per_block,
+                                            0,
+                                            dev_ctx.stream()>>>(
+          numel, out_grad_data, pd_indices, stride_a, shape_a, value_grad_data);
+    } else {
+      DenseTensor tmp_value_grad(value_grad->dtype());
+      tmp_value_grad.Resize(indices[0]->dims());
+
+      T* tmp_value_grad_data = dev_ctx.template Alloc<T>(&tmp_value_grad);
+      auto out_grad_data = out_grad.data<T>();
+
+      index_put_grad_cuda_kernel<T, Rank>
+          <<<config.block_per_grid,
+             config.thread_per_block,
+             0,
+             dev_ctx.stream()>>>(numel,
+                                 out_grad_data,
+                                 pd_indices,
+                                 stride_a,
+                                 shape_a,
+                                 tmp_value_grad_data);
+
+      std::vector<int64_t> after_dims = phi::vectorize(tmp_value_grad.dims());
+      std::vector<int64_t> before_dims = phi::vectorize(value_grad->dims());
+      std::vector<int64_t> compress_dims;
+      std::vector<int64_t> dims_without_1;
+
+      funcs::CalCompressedDimsWith1AndWithout1(
+          &after_dims, &before_dims, &compress_dims, &dims_without_1);
+
+      auto pre_dims = value_grad->dims();
+      value_grad->Resize(phi::make_ddim(dims_without_1));
+      IntArray v_axis(compress_dims);
+      SumKernel<T, Context>(dev_ctx,
+                            tmp_value_grad,
+                            v_axis,
+                            value_grad->dtype(),
+                            false,
+                            value_grad);
+      value_grad->Resize(pre_dims);
+    }
+  }
+}
+
+template <typename T, typename Context>
+void IndexPutGradKernel(const Context& dev_ctx,
+                        const DenseTensor& x,
+                        const std::vector<const DenseTensor*>& indices,
+                        const DenseTensor& value,
+                        const DenseTensor& out_grad,
+                        bool accumulate,
+                        DenseTensor* x_grad,
+                        DenseTensor* value_grad) {
+  PADDLE_ENFORCE_EQ(
+      x.dtype(),
+      value.dtype(),
+      phi::errors::InvalidArgument(
+          "The data type of tensor in indices must be same to the data type "
+          "of tensor x."));
+  std::vector<DenseTensor> tmp_args;
+  std::vector<const phi::DenseTensor*> int_indices_v =
+      funcs::DealWithBoolIndices<T, Context>(dev_ctx, indices, &tmp_args);
+  const size_t total_dims = x.dims().size();
+  auto bd_dim = funcs::BroadCastTensorsDims(int_indices_v);
+
+  std::vector<int64_t> res_dim_v(phi::vectorize(bd_dim));
+  std::vector<const phi::DenseTensor*> res_indices_v(x.dims().size(), nullptr);
+  std::vector<DenseTensor> tmp_res_indices_v;
+  std::vector<DenseTensor> range_tensor_v;
+
+  for (int i = indices.size(); i < x.dims().size(); ++i) {
+    range_tensor_v.emplace_back(funcs::GetRangeCudaTensor<int64_t, Context>(
+        dev_ctx, x.dims()[i], phi::DataType::INT64));
+  }
+
+  funcs::DealWithIndices<T, Context>(dev_ctx,
+                                     x,
+                                     int_indices_v,
+                                     &res_indices_v,
+                                     &tmp_res_indices_v,
+                                     range_tensor_v,
+                                     bd_dim,
+                                     &res_dim_v);
+
+  switch (total_dims) {
+    case 1:
+      LaunchIndexPutGradCudaKernel<T, Context, 1>(
+          dev_ctx, res_indices_v, out_grad, accumulate, value_grad, x_grad);
+      break;
+    case 2:
+      LaunchIndexPutGradCudaKernel<T, Context, 2>(
+          dev_ctx, res_indices_v, out_grad, accumulate, value_grad, x_grad);
+      break;
+    case 3:
+      LaunchIndexPutGradCudaKernel<T, Context, 3>(
+          dev_ctx, res_indices_v, out_grad, accumulate, value_grad, x_grad);
+      break;
+    case 4:
+      LaunchIndexPutGradCudaKernel<T, Context, 4>(
+          dev_ctx, res_indices_v, out_grad, accumulate, value_grad, x_grad);
+      break;
+    case 5:
+      LaunchIndexPutGradCudaKernel<T, Context, 5>(
+          dev_ctx, res_indices_v, out_grad, accumulate, value_grad, x_grad);
+      break;
+    case 6:
+      LaunchIndexPutGradCudaKernel<T, Context, 6>(
+          dev_ctx, res_indices_v, out_grad, accumulate, value_grad, x_grad);
+      break;
+    default:
+      PADDLE_THROW(phi::errors::InvalidArgument(
+          "dims of input tensor should be less than 7, But received"
+          "%d",
+          x.dims().size()));
+  }
+}
+}  // namespace phi
+
+PD_REGISTER_KERNEL(index_put_grad,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::IndexPutGradKernel,
+                   float,
+                   double,
+                   int,
+                   int64_t,
+                   bool,
+                   phi::dtype::float16) {}
diff --git a/paddle/phi/kernels/gpu/index_put_kernel.cu b/paddle/phi/kernels/gpu/index_put_kernel.cu
new file mode 100644
index 0000000000000..ad27993c35244
--- /dev/null
+++ b/paddle/phi/kernels/gpu/index_put_kernel.cu
@@ -0,0 +1,198 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/index_put_kernel.h"
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/backends/gpu/gpu_launch_config.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/cast_kernel.h"
+#include "paddle/phi/kernels/funcs/index_put_utils.h"
+
+namespace phi {
+
+template <typename T, size_t Rank>
+__global__ void index_put_cuda_kernel(const int64_t N,
+                                      const T* x,
+                                      const T* vals,
+                                      int64_t** indices,
+                                      phi::Array<int64_t, Rank> stride,
+                                      phi::Array<int64_t, Rank> shape,
+                                      int64_t is_single_val_tensor,
+                                      bool accumulate,
+                                      T* out) {
+  int64_t idx = threadIdx.x + blockDim.x * blockIdx.x;
+  int64_t cur_ix = 0;
+
+  if (idx >= N) {
+    return;
+  }
+  int64_t offset = 0;
+  for (int i = 0; i < Rank; ++i) {
+    cur_ix = (static_cast<int64_t>(*(indices[i] + idx)));
+    if (cur_ix < 0) {
+      cur_ix += shape[i];
+    }
+    offset += stride[i] * cur_ix;
+  }
+
+  if (accumulate) {
+    *(out + offset) += *(vals + (idx & is_single_val_tensor));
+  } else {
+    *(out + offset) = *(vals + (idx & is_single_val_tensor));
+  }
+}
+
+template <typename T, typename Context, size_t Rank>
+void LaunchIndexPutCudaKernel(const Context& dev_ctx,
+                              const DenseTensor& x,
+                              const std::vector<const DenseTensor*>& indices,
+                              const DenseTensor& value,
+                              bool accumulate,
+                              DenseTensor* out) {
+  auto* x_data = x.data<T>();
+  auto* val_data = value.data<T>();
+  bool is_initialized = out->initialized();
+  T* out_data = dev_ctx.template Alloc<T>(out);
+
+  if (!is_initialized) {
+    phi::Copy(dev_ctx, x, dev_ctx.GetPlace(), false, out);
+  }
+
+  auto x_dims = x.dims();
+  const int64_t numel = indices[0]->numel();
+  auto config = phi::backends::gpu::GetGpuLaunchConfig1D(dev_ctx, numel);
+  auto x_stride = phi::stride(x_dims);
+
+  phi::Array<int64_t, Rank> stride_a;
+  phi::Array<int64_t, Rank> shape_a;
+
+  for (size_t idx = 0; idx < Rank; ++idx) {
+    stride_a[idx] = x_stride[idx];
+    shape_a[idx] = x_dims[idx];
+  }
+
+  int64_t is_single_val_tensor = (value.numel() == 1) ? 0 : INT64_MAX;
+
+  auto pd_indices =
+      funcs::GetDevicePointerArray<int64_t, Context>(dev_ctx, indices);
+  index_put_cuda_kernel<T, Rank>
+      <<<config.block_per_grid, config.thread_per_block, 0, dev_ctx.stream()>>>(
+          numel,
+          x_data,
+          val_data,
+          pd_indices,
+          stride_a,
+          shape_a,
+          is_single_val_tensor,
+          accumulate,
+          out_data);
+}
+
+template <typename T, typename Context>
+void IndexPutKernel(const Context& dev_ctx,
+                    const DenseTensor& x,
+                    const std::vector<const DenseTensor*>& indices,
+                    const DenseTensor& value,
+                    bool accumulate,
+                    DenseTensor* out) {
+  PADDLE_ENFORCE_EQ(
+      x.dtype(),
+      value.dtype(),
+      phi::errors::InvalidArgument(
+          "The data type of tensor in indices must be same to the data type "
+          "of tensor x."));
+  PADDLE_ENFORCE_EQ(indices.empty(),
+                    false,
+                    phi::errors::InvalidArgument("Indices cannot be empty."));
+  std::vector<DenseTensor> tmp_args;
+  std::vector<const phi::DenseTensor*> int_indices_v =
+      funcs::DealWithBoolIndices<T, Context>(dev_ctx, indices, &tmp_args);
+  const size_t total_dims = x.dims().size();
+  auto bd_dim = funcs::BroadCastTensorsDims(int_indices_v);
+
+  std::vector<int64_t> res_dim_v(phi::vectorize(bd_dim));
+  std::vector<const phi::DenseTensor*> res_indices_v(x.dims().size(), nullptr);
+  std::vector<DenseTensor> tmp_res_indices_v;
+  std::vector<DenseTensor> tmp_value_v;
+  std::vector<DenseTensor> range_tensor_v;
+  const DenseTensor* ptr_value = nullptr;
+
+  for (int i = indices.size(); i < x.dims().size(); ++i) {
+    range_tensor_v.emplace_back(funcs::GetRangeCudaTensor<int64_t, Context>(
+        dev_ctx, x.dims()[i], phi::DataType::INT64));
+  }
+
+  funcs::DealWithIndices<T, Context>(dev_ctx,
+                                     x,
+                                     int_indices_v,
+                                     &res_indices_v,
+                                     &tmp_res_indices_v,
+                                     range_tensor_v,
+                                     bd_dim,
+                                     &res_dim_v);
+
+  if (value.numel() != 1) {
+    tmp_value_v.emplace_back(
+        DenseTensor(value.dtype()).Resize(phi::make_ddim(res_dim_v)));
+    ExpandKernel<T, Context>(
+        dev_ctx, value, IntArray(res_dim_v), &tmp_value_v[0]);
+    ptr_value = &tmp_value_v[0];
+  } else {
+    ptr_value = &value;
+  }
+
+  switch (total_dims) {
+    case 1:
+      LaunchIndexPutCudaKernel<T, Context, 1>(
+          dev_ctx, x, res_indices_v, *ptr_value, accumulate, out);
+      break;
+    case 2:
+      LaunchIndexPutCudaKernel<T, Context, 2>(
+          dev_ctx, x, res_indices_v, *ptr_value, accumulate, out);
+      break;
+    case 3:
+      LaunchIndexPutCudaKernel<T, Context, 3>(
+          dev_ctx, x, res_indices_v, *ptr_value, accumulate, out);
+      break;
+    case 4:
+      LaunchIndexPutCudaKernel<T, Context, 4>(
+          dev_ctx, x, res_indices_v, *ptr_value, accumulate, out);
+      break;
+    case 5:
+      LaunchIndexPutCudaKernel<T, Context, 5>(
+          dev_ctx, x, res_indices_v, *ptr_value, accumulate, out);
+      break;
+    case 6:
+      LaunchIndexPutCudaKernel<T, Context, 6>(
+          dev_ctx, x, res_indices_v, *ptr_value, accumulate, out);
+      break;
+    default:
+      PADDLE_THROW(phi::errors::InvalidArgument(
+          "dims of input tensor should be less than 7, But received"
+          "%d",
+          x.dims().size()));
+  }
+}
+}  // namespace phi
+
+PD_REGISTER_KERNEL(index_put,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::IndexPutKernel,
+                   float,
+                   double,
+                   int,
+                   int64_t,
+                   bool,
+                   phi::dtype::float16) {}
diff --git a/paddle/phi/kernels/index_put_grad_kernel.h b/paddle/phi/kernels/index_put_grad_kernel.h
new file mode 100644
index 0000000000000..575b7df5f2739
--- /dev/null
+++ b/paddle/phi/kernels/index_put_grad_kernel.h
@@ -0,0 +1,30 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <vector>
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+template <typename T, typename Context>
+void IndexPutGradKernel(const Context& dev_ctx,
+                        const DenseTensor& x,
+                        const std::vector<const DenseTensor*>& indices_v,
+                        const DenseTensor& value,
+                        const DenseTensor& out_grad,
+                        bool accumulate,
+                        DenseTensor* x_grad,
+                        DenseTensor* value_grad);
+}  // namespace phi
diff --git a/paddle/phi/kernels/index_put_kernel.h b/paddle/phi/kernels/index_put_kernel.h
new file mode 100644
index 0000000000000..4410a50824457
--- /dev/null
+++ b/paddle/phi/kernels/index_put_kernel.h
@@ -0,0 +1,29 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <vector>
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+template <typename T, typename Context>
+void IndexPutKernel(const Context& dev_ctx,
+                    const DenseTensor& x,
+                    const std::vector<const DenseTensor*>& indices_v,
+                    const DenseTensor& value,
+                    bool accumulate,
+                    DenseTensor* out);
+
+}  // namespace phi
diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py
index 673a855c79540..8804cfff6de19 100644
--- a/python/paddle/__init__.py
+++ b/python/paddle/__init__.py
@@ -198,6 +198,8 @@
 from .tensor.manipulation import repeat_interleave  # noqa: F401
 from .tensor.manipulation import index_add  # noqa: F401
 from .tensor.manipulation import index_add_  # noqa: F401
+from .tensor.manipulation import index_put  # noqa: F401
+from .tensor.manipulation import index_put_  # noqa: F401
 from .tensor.manipulation import unflatten  # noqa: F401
 from .tensor.math import abs  # noqa: F401
 from .tensor.math import acos  # noqa: F401
@@ -684,6 +686,8 @@
     'tril_indices',
     'index_add',
     "index_add_",
+    "index_put",
+    "index_put_",
     'sgn',
     'triu_indices',
     'take',
diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt
index e7d11ed8b16d6..8ca0bc01ab31b 100755
--- a/python/paddle/fluid/tests/unittests/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt
@@ -916,6 +916,7 @@ set_tests_properties(test_imperative_selected_rows_to_lod_tensor
                      PROPERTIES TIMEOUT 200)
 set_tests_properties(test_index_select_op PROPERTIES TIMEOUT 120)
 set_tests_properties(test_index_add_op PROPERTIES TIMEOUT 120)
+set_tests_properties(test_index_put_op PROPERTIES TIMEOUT 120)
 set_tests_properties(test_tensordot PROPERTIES TIMEOUT 200)
 set_tests_properties(test_partial_eager_deletion_transformer PROPERTIES TIMEOUT
                                                                         120)
diff --git a/python/paddle/fluid/tests/unittests/test_index_put_op.py b/python/paddle/fluid/tests/unittests/test_index_put_op.py
new file mode 100644
index 0000000000000..5f0257f25535c
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_index_put_op.py
@@ -0,0 +1,826 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import unittest
+
+import numpy as np
+
+import paddle
+from paddle.fluid import Program
+
+
+def compute_index_put_ref(x_np, indices_np, value_np, accumulate=False):
+    if accumulate:
+        x_np[indices_np] += value_np
+        return x_np
+    else:
+        x_np[indices_np] = value_np
+        return x_np
+
+
+def raw_index_put(x, indices, value, accummulate):
+    return paddle.index_put(x, indices, value, accummulate)
+
+
+def has_duplicate_index(indices, shapes):
+    bd_shape = np.broadcast_shapes(*shapes)
+    bd_indices = [
+        list(np.broadcast_to(indice, bd_shape).flatten()) for indice in indices
+    ]
+
+    zip_res = list(zip(*bd_indices))
+    if len(zip_res) == len(set(zip_res)):
+        return False
+    else:
+        return True
+
+
+def gen_indices_np(x_shape, indices_shapes, index_type):
+    indices = []
+    if index_type == np.bool_:
+        indice = np.zeros(indices_shapes[0], dtype=np.bool_)
+        indice.flatten()
+        for i in range(len(indice)):
+            indice[i] = (i & 1) == 0
+        indice = indice.reshape(indices_shapes[0])
+        indices.append(indice)
+    else:
+        while True:
+            indices = []
+            for i in range(len(indices_shapes)):
+                np.random.seed()
+                index_np = np.random.randint(
+                    low=0,
+                    high=x_shape[i],
+                    size=indices_shapes[i],
+                    dtype=index_type,
+                )
+                indices.append(index_np)
+            if not has_duplicate_index(
+                copy.deepcopy(indices), copy.deepcopy(indices_shapes)
+            ):
+                break
+    return tuple(indices)
+
+
+class TestIndexPutAPIBase(unittest.TestCase):
+    def setUp(self):
+        self.init_dtype_type()
+        self.setPlace()
+        self.x_np = np.random.random(self.x_shape).astype(self.dtype_np)
+        self.value_np = np.random.random(self.value_shape).astype(self.dtype_np)
+        self.indices_np = gen_indices_np(
+            self.x_shape, self.indices_shapes, self.index_type_np
+        )
+
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int64
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int64
+        self.accumulate = False
+
+    def setPlace(self):
+        self.place = ['cpu']
+        if self.dtype_np is np.float16:
+            self.place = []
+        if paddle.is_compiled_with_cuda():
+            self.place.append('gpu')
+
+    def test_dygraph_forward(self):
+        paddle.disable_static()
+        for place in self.place:
+            paddle.device.set_device(place)
+            self.x_pd = paddle.to_tensor(self.x_np, dtype=self.dtype_pd)
+            self.value_pd = paddle.to_tensor(self.value_np, dtype=self.dtype_pd)
+            self.indices_pd = [
+                paddle.to_tensor(indice, dtype=self.index_type_pd)
+                for indice in self.indices_np
+            ]
+            self.indices_pd = tuple(self.indices_pd)
+            ref_res = compute_index_put_ref(
+                self.x_np, self.indices_np, self.value_np, self.accumulate
+            )
+            pd_res = paddle.index_put(
+                self.x_pd, self.indices_pd, self.value_pd, self.accumulate
+            )
+            np.testing.assert_allclose(ref_res, pd_res.numpy(), atol=1e-7)
+
+    def test_static_forward(self):
+        paddle.enable_static()
+        for place in self.place:
+            with paddle.static.program_guard(Program()):
+                x = paddle.static.data(
+                    name="x", shape=self.x_shape, dtype=self.dtype_pd
+                )
+                indices = tuple(
+                    [
+                        paddle.static.data(
+                            name="indice" + str(i),
+                            shape=self.indices_shapes[i],
+                            dtype=self.index_type_pd,
+                        )
+                        for i in range(len(self.indices_shapes))
+                    ]
+                )
+                value = paddle.static.data(
+                    name="value", shape=self.value_shape, dtype=self.dtype_pd
+                )
+
+                out = paddle.index_put(x, indices, value, self.accumulate)
+                exe = paddle.static.Executor(place=place)
+                feed_list = {}
+                feed_list.update({"x": self.x_np})
+                for i in range(len(indices)):
+                    feed_list.update({"indice" + str(i): self.indices_np[i]})
+                feed_list.update({"value": self.value_np})
+                pd_res = exe.run(
+                    paddle.static.default_main_program(),
+                    feed=feed_list,
+                    fetch_list=[out],
+                )[0]
+                ref_res = compute_index_put_ref(
+                    self.x_np, self.indices_np, self.value_np, self.accumulate
+                )
+                np.testing.assert_allclose(ref_res, pd_res, atol=1e-7)
+
+
+class TestIndexPutAPI0(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int64
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int64
+        self.accumulate = True
+
+
+class TestIndexPutAPI1(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int64
+        self.x_shape = (110, 42, 56, 56)
+        self.indices_shapes = ((16, 16), (16, 16), (1, 16), (1, 16))
+        self.value_shape = (16, 16)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int64
+        self.accumulate = False
+
+
+class TestIndexPutAPI2(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int64
+        self.x_shape = (110, 42, 56, 56)
+        self.indices_shapes = ((16, 16), (16, 16), (1, 16), (1, 16))
+        self.value_shape = (16, 16)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int64
+        self.accumulate = True
+
+
+class TestIndexPutAPI3(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.bool_
+        self.x_shape = (110, 94)
+        self.indices_shapes = [(110, 94)]
+        self.value_shape = (5170,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.bool
+        self.accumulate = False
+
+
+class TestIndexPutAPI4(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.bool_
+        self.x_shape = (110, 94)
+        self.indices_shapes = [(110, 94)]
+        self.value_shape = (5170,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.bool
+        self.accumulate = True
+
+
+class TestIndexPutAPI5(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int64
+        self.x_shape = (110, 42, 56, 56)
+        self.indices_shapes = ((16, 16), (16, 16), (1, 16))
+        self.value_shape = (16, 16, 56)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int64
+        self.accumulate = False
+
+
+class TestIndexPutAPI6(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int64
+        self.x_shape = (110, 42, 56, 56)
+        self.indices_shapes = ((16, 16), (16, 16), (1, 16))
+        self.value_shape = (16, 16, 56)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int64
+        self.accumulate = True
+
+
+class TestIndexPutAPI7(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.bool_
+        self.x_shape = (110, 94)
+        self.indices_shapes = [(110,)]
+        self.value_shape = (55, 94)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.bool
+        self.accumulate = False
+
+
+class TestIndexPutAPI8(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.bool_
+        self.x_shape = (110, 94)
+        self.indices_shapes = [(110,)]
+        self.value_shape = (55, 94)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.bool
+        self.accumulate = True
+
+
+class TestIndexPutAPI9(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int64
+        self.x_shape = (110, 42, 56, 56)
+        self.indices_shapes = ((16, 16), (16, 16), (1, 16))
+        self.value_shape = (56,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int64
+        self.accumulate = False
+
+
+class TestIndexPutAPI10(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int64
+        self.x_shape = (110, 42, 56, 56)
+        self.indices_shapes = ((16, 16), (16, 16), (1, 16))
+        self.value_shape = (56,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int64
+        self.accumulate = True
+
+
+class TestIndexPutAPI11(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int64
+        self.x_shape = (110, 42, 56, 56)
+        self.indices_shapes = ((16, 16), (16, 16), (1, 16))
+        self.value_shape = (1,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int64
+        self.accumulate = False
+
+
+class TestIndexPutAPI12(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int64
+        self.x_shape = (110, 42, 56, 56)
+        self.indices_shapes = ((16, 16), (16, 16), (1, 16))
+        self.value_shape = (1,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int64
+        self.accumulate = True
+
+
+class TestIndexPutAPI13(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.bool_
+        self.x_shape = (44, 94)
+        self.indices_shapes = [(44,)]
+        self.value_shape = (94,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.bool
+        self.accumulate = False
+
+
+class TestIndexPutAPI14(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.bool_
+        self.x_shape = (44, 94)
+        self.indices_shapes = [(44,)]
+        self.value_shape = (94,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.bool
+        self.accumulate = True
+
+
+class TestIndexPutAPI15(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.bool_
+        self.x_shape = (44, 94)
+        self.indices_shapes = [(44,)]
+        self.value_shape = (1,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.bool
+        self.accumulate = False
+
+
+class TestIndexPutAPI16(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.bool_
+        self.x_shape = (44, 94)
+        self.indices_shapes = [(44,)]
+        self.value_shape = (1,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.bool
+        self.accumulate = True
+
+
+class TestIndexPutAPI17(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int32
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int32
+        self.accumulate = False
+
+
+class TestIndexPutAPI18(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int32
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int32
+        self.accumulate = True
+
+
+class TestIndexPutAPI19(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float32
+        self.index_type_np = np.int32
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.float32
+        self.index_type_pd = paddle.int32
+        self.accumulate = False
+
+
+class TestIndexPutAPI20(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float32
+        self.index_type_np = np.int32
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.float32
+        self.index_type_pd = paddle.int32
+        self.accumulate = True
+
+
+class TestIndexPutAPI21(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float16
+        self.index_type_np = np.int32
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.float16
+        self.index_type_pd = paddle.int32
+        self.accumulate = False
+
+
+class TestIndexPutAPI22(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float16
+        self.index_type_np = np.int32
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.float16
+        self.index_type_pd = paddle.int32
+        self.accumulate = True
+
+
+class TestIndexPutAPI23(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.int32
+        self.index_type_np = np.int32
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.int32
+        self.index_type_pd = paddle.int32
+        self.accumulate = False
+
+
+class TestIndexPutAPI24(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.int32
+        self.index_type_np = np.int32
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.int32
+        self.index_type_pd = paddle.int32
+        self.accumulate = True
+
+
+class TestIndexPutAPI25(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.int64
+        self.index_type_np = np.int32
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.int64
+        self.index_type_pd = paddle.int32
+        self.accumulate = False
+
+
+class TestIndexPutAPI26(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.int64
+        self.index_type_np = np.int32
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.int64
+        self.index_type_pd = paddle.int32
+        self.accumulate = True
+
+
+class TestIndexPutAPI27(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.bool_
+        self.index_type_np = np.int32
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.bool
+        self.index_type_pd = paddle.int32
+        self.accumulate = False
+
+
+class TestIndexPutAPI28(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.bool_
+        self.index_type_np = np.int32
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.bool
+        self.index_type_pd = paddle.int32
+        self.accumulate = True
+
+
+class TestIndexPutAPI29(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int32
+        self.x_shape = (110, 42, 56, 56)
+        self.indices_shapes = ((16, 16), (16, 16), (1, 16))
+        self.value_shape = (16, 16, 56)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int32
+        self.accumulate = False
+
+
+class TestIndexPutAPI30(TestIndexPutAPIBase):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int32
+        self.x_shape = (110, 42, 56, 56)
+        self.indices_shapes = ((16, 16), (16, 16), (1, 16))
+        self.value_shape = (16, 16, 56)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int32
+        self.accumulate = True
+
+
+class TestIndexPutInplaceAPI(unittest.TestCase):
+    def setUp(self):
+        self.init_dtype_type()
+        self.setPlace()
+        self.x_np = np.random.random(self.x_shape).astype(self.dtype_np)
+        self.value_np = np.random.random(self.value_shape).astype(self.dtype_np)
+        self.indices_np = gen_indices_np(
+            self.x_shape, self.indices_shapes, self.index_type_np
+        )
+
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int64
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int64
+        self.accumulate = False
+
+    def setPlace(self):
+        self.place = ['cpu']
+        if paddle.is_compiled_with_cuda():
+            self.place.append('gpu')
+
+    def test_dygraph_forward(self):
+        paddle.disable_static()
+        for place in self.place:
+            paddle.device.set_device(place)
+            self.x_pd = paddle.to_tensor(self.x_np, dtype=self.dtype_pd)
+            self.value_pd = paddle.to_tensor(self.value_np, dtype=self.dtype_pd)
+            self.indices_pd = [
+                paddle.to_tensor(indice, dtype=self.index_type_pd)
+                for indice in self.indices_np
+            ]
+            self.indices_pd = tuple(self.indices_pd)
+            ref_res = compute_index_put_ref(
+                self.x_np, self.indices_np, self.value_np, self.accumulate
+            )
+            x_pd_bk = self.x_pd.clone()
+            pd_res = paddle.index_put_(
+                x_pd_bk, self.indices_pd, self.value_pd, self.accumulate
+            )
+            np.testing.assert_allclose(ref_res, pd_res.numpy(), atol=1e-7)
+            np.testing.assert_allclose(ref_res, x_pd_bk.numpy(), atol=1e-7)
+
+
+class TestIndexPutInplaceAPI1(TestIndexPutInplaceAPI):
+    def init_dtype_type(self):
+        self.dtype_np = np.float64
+        self.index_type_np = np.int64
+        self.x_shape = (100, 110)
+        self.indices_shapes = [(21,), (21,)]
+        self.value_shape = (21,)
+        self.dtype_pd = paddle.float64
+        self.index_type_pd = paddle.int64
+        self.accumulate = True
+
+
+class TestIndexPutAPIBackward(unittest.TestCase):
+    def setUp(self):
+        self.setPlace()
+
+    def setPlace(self):
+        self.place = ['cpu']
+        if paddle.is_compiled_with_cuda():
+            self.place.append('gpu')
+
+    def test_backward(self):
+        paddle.disable_static()
+        for place in self.place:
+            paddle.device.set_device(place)
+            value = paddle.ones(shape=[4], dtype=paddle.float64)
+            x = paddle.ones(shape=[16, 21], dtype=paddle.float64)
+            ix1 = paddle.to_tensor([0, 1, 2, 3], dtype=paddle.int64)
+            ix2 = paddle.to_tensor([0, 1, 2, 3], dtype=paddle.int64)
+            value.stop_gradient = False
+            x.stop_gradient = False
+            out = paddle.index_put(x, (ix1, ix2), value, False)
+
+            dx, dvalue = paddle.grad(
+                outputs=[out],
+                inputs=[x, value],
+                create_graph=False,
+                retain_graph=True,
+            )
+            ref_dx = np.ones(shape=[16, 21], dtype=np.float64)
+            ref_dx[ix1, ix2] = 0
+
+            np.testing.assert_allclose(ref_dx, dx.numpy(), atol=1e-7)
+            np.testing.assert_allclose(
+                np.array([1.0, 1.0, 1.0, 1.0], dtype=np.float64),
+                dvalue.numpy(),
+                atol=1e-7,
+            )
+
+            out = paddle.index_put(x, (ix1, ix2), value, True)
+
+            dx, dvalue = paddle.grad(
+                outputs=[out],
+                inputs=[x, value],
+                create_graph=False,
+                retain_graph=True,
+            )
+            ref_dx = np.ones(shape=[16, 21], dtype=np.float64)
+
+            np.testing.assert_allclose(ref_dx, dx.numpy(), atol=1e-7)
+            np.testing.assert_allclose(
+                np.array([1.0, 1.0, 1.0, 1.0], dtype=np.float64),
+                dvalue.numpy(),
+                atol=1e-7,
+            )
+
+    def test_backwardScalarVal(self):
+        paddle.disable_static()
+        for place in self.place:
+            paddle.device.set_device(place)
+            value = paddle.ones(shape=[1], dtype=paddle.float64)
+            x = paddle.ones(shape=[16, 21], dtype=paddle.float64)
+            ix1 = paddle.to_tensor([0, 1, 2, 3], dtype=paddle.int64)
+            ix2 = paddle.to_tensor([0, 1, 2, 3], dtype=paddle.int64)
+            value.stop_gradient = False
+            x.stop_gradient = False
+            out = paddle.index_put(x, (ix1, ix2), value, False)
+
+            dx, dvalue = paddle.grad(
+                outputs=[out],
+                inputs=[x, value],
+                create_graph=False,
+                retain_graph=True,
+            )
+            ref_dx = np.ones(shape=[16, 21], dtype=np.float64)
+            ref_dx[ix1, ix2] = 0
+
+            np.testing.assert_allclose(ref_dx, dx.numpy(), atol=1e-7)
+            np.testing.assert_allclose(
+                np.array([4.0], dtype=np.float64), dvalue.numpy(), atol=1e-7
+            )
+
+            out = paddle.index_put(x, (ix1, ix2), value, True)
+
+            dx, dvalue = paddle.grad(
+                outputs=[out],
+                inputs=[x, value],
+                create_graph=False,
+                retain_graph=True,
+            )
+            ref_dx = np.ones(shape=[16, 21], dtype=np.float64)
+
+            np.testing.assert_allclose(ref_dx, dx.numpy(), atol=1e-7)
+            np.testing.assert_allclose(
+                np.array([4.0], dtype=np.float64), dvalue.numpy(), atol=1e-7
+            )
+
+    def test_backwardBroadCastValue(self):
+        paddle.disable_static()
+        for place in self.place:
+            paddle.device.set_device(place)
+            value = paddle.ones(shape=[2], dtype=paddle.float64)
+            x = paddle.ones(shape=[16, 21], dtype=paddle.float64)
+            ix1 = paddle.to_tensor([[0, 1], [2, 3]], dtype=paddle.int64)
+            ix2 = paddle.to_tensor([[0, 1], [2, 3]], dtype=paddle.int64)
+            value.stop_gradient = False
+            x.stop_gradient = False
+            out = paddle.index_put(x, (ix1, ix2), value, False)
+
+            dx, dvalue = paddle.grad(
+                outputs=[out],
+                inputs=[x, value],
+                create_graph=False,
+                retain_graph=True,
+            )
+            ref_dx = np.ones(shape=[16, 21], dtype=np.float64)
+            ref_dx[ix1, ix2] = 0
+
+            np.testing.assert_allclose(ref_dx, dx.numpy(), atol=1e-7)
+            np.testing.assert_allclose(
+                np.array([2.0, 2.0], dtype=np.float64),
+                dvalue.numpy(),
+                atol=1e-7,
+            )
+
+            out = paddle.index_put(x, (ix1, ix2), value, True)
+
+            dx, dvalue = paddle.grad(
+                outputs=[out],
+                inputs=[x, value],
+                create_graph=False,
+                retain_graph=True,
+            )
+            ref_dx = np.ones(shape=[16, 21], dtype=np.float64)
+
+            np.testing.assert_allclose(ref_dx, dx.numpy(), atol=1e-7)
+            np.testing.assert_allclose(
+                np.array([2.0, 2.0], dtype=np.float64),
+                dvalue.numpy(),
+                atol=1e-7,
+            )
+
+    def test_backwardBroadCastValue1(self):
+        paddle.disable_static()
+        for place in self.place:
+            paddle.device.set_device(place)
+            value = paddle.ones(shape=[1, 2], dtype=paddle.float64)
+            x = paddle.ones(shape=[16, 21], dtype=paddle.float64)
+            ix1 = paddle.to_tensor([[0, 1], [2, 3]], dtype=paddle.int64)
+            ix2 = paddle.to_tensor([[0, 1], [2, 3]], dtype=paddle.int64)
+            value.stop_gradient = False
+            x.stop_gradient = False
+            out = paddle.index_put(x, (ix1, ix2), value, False)
+
+            dx, dvalue = paddle.grad(
+                outputs=[out],
+                inputs=[x, value],
+                create_graph=False,
+                retain_graph=True,
+            )
+            ref_dx = np.ones(shape=[16, 21], dtype=np.float64)
+            ref_dx[ix1, ix2] = 0
+
+            np.testing.assert_allclose(ref_dx, dx.numpy(), atol=1e-7)
+            np.testing.assert_allclose(
+                np.array([[2.0, 2.0]], dtype=np.float64),
+                dvalue.numpy(),
+                atol=1e-7,
+            )
+
+            out = paddle.index_put(x, (ix1, ix2), value, True)
+
+            dx, dvalue = paddle.grad(
+                outputs=[out],
+                inputs=[x, value],
+                create_graph=False,
+                retain_graph=True,
+            )
+            ref_dx = np.ones(shape=[16, 21], dtype=np.float64)
+
+            np.testing.assert_allclose(ref_dx, dx.numpy(), atol=1e-7)
+            np.testing.assert_allclose(
+                np.array([[2.0, 2.0]], dtype=np.float64),
+                dvalue.numpy(),
+                atol=1e-7,
+            )
+
+    def test_backwardBroadCastValue2(self):
+        paddle.disable_static()
+        for place in self.place:
+            paddle.device.set_device(place)
+            value = paddle.ones(shape=[2, 1], dtype=paddle.float64)
+            x = paddle.ones(shape=[16, 21], dtype=paddle.float64)
+            ix1 = paddle.to_tensor([[0, 1], [2, 3]], dtype=paddle.int64)
+            ix2 = paddle.to_tensor([[0, 1], [2, 3]], dtype=paddle.int64)
+            value.stop_gradient = False
+            x.stop_gradient = False
+            out = paddle.index_put(x, (ix1, ix2), value, False)
+
+            dx, dvalue = paddle.grad(
+                outputs=[out],
+                inputs=[x, value],
+                create_graph=False,
+                retain_graph=True,
+            )
+            ref_dx = np.ones(shape=[16, 21], dtype=np.float64)
+            ref_dx[ix1, ix2] = 0
+
+            np.testing.assert_allclose(ref_dx, dx.numpy(), atol=1e-7)
+            np.testing.assert_allclose(
+                np.array([[2.0], [2.0]], dtype=np.float64),
+                dvalue.numpy(),
+                atol=1e-7,
+            )
+
+            out = paddle.index_put(x, (ix1, ix2), value, True)
+
+            dx, dvalue = paddle.grad(
+                outputs=[out],
+                inputs=[x, value],
+                create_graph=False,
+                retain_graph=True,
+            )
+            ref_dx = np.ones(shape=[16, 21], dtype=np.float64)
+
+            np.testing.assert_allclose(ref_dx, dx.numpy(), atol=1e-7)
+            np.testing.assert_allclose(
+                np.array([[2.0], [2.0]], dtype=np.float64),
+                dvalue.numpy(),
+                atol=1e-7,
+            )
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/python/paddle/tensor/__init__.py b/python/paddle/tensor/__init__.py
index f8d75c9651d28..a399fad5a8fb4 100644
--- a/python/paddle/tensor/__init__.py
+++ b/python/paddle/tensor/__init__.py
@@ -135,6 +135,8 @@
 from .manipulation import repeat_interleave  # noqa: F401
 from .manipulation import index_add  # noqa: F401
 from .manipulation import index_add_  # noqa: F401
+from .manipulation import index_put  # noqa: F401
+from .manipulation import index_put_  # noqa: F401
 from .manipulation import unflatten  # noqa: F401
 from .math import abs  # noqa: F401
 from .math import acos  # noqa: F401
@@ -534,6 +536,8 @@
     'heaviside',
     'index_add',
     "index_add_",
+    'index_put',
+    'index_put_',
     'take',
     'bucketize',
     'sgn',
diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py
index cc48d8a83dcbe..7fe717383cd3b 100644
--- a/python/paddle/tensor/manipulation.py
+++ b/python/paddle/tensor/manipulation.py
@@ -4795,6 +4795,108 @@ def index_add_(x, index, axis, value, name=None):
     return _C_ops.index_add_(x, index, value, axis)
 
 
+@inplace_apis_in_dygraph_only
+def index_put_(x, indices, value, accumulate=False, name=None):
+    """
+    Puts values from the tensor values into the tensor x using the indices specified in indices (which is a tuple of Tensors).
+    The expression paddle.index_put_(x, indices, values) is equivalent to tensor[indices] = values. Returns x.
+    If accumulate is True, the elements in values are added to x. If accumulate is False, the behavior is undefined if indices contain duplicate elements.
+
+    Args:
+        x (Tensor) : The Source Tensor. Supported data types are int32, int64, float16, float32, float64, bool.
+        indices (Tuple of Tensor): The tuple of Tensor containing the indices to index.
+            The data type of ``tensor in indices`` must be int32, int64 or bool
+        value (Tensor): The tensor used to be assigned to x.
+        accummulate (Bool, optional): Whether the elements in values are added to x. Default: False.
+        name(str, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
+
+    Returns:
+        Tensor, same dimention and dtype with x.
+
+    Examples:
+        .. code-block:: python
+            import paddle
+
+            x = paddle.zeros([3, 3])
+            value = paddle.ones([3])
+            ix1 = paddle.to_tensor([0,1,2])
+            ix2 = paddle.to_tensor([1,2,1])
+            indices=(ix1,ix2)
+
+            out = paddle.index_put_(x,indices,value)
+            print(x)
+            # Tensor(shape=[3, 3], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        [[0., 1., 0.],
+            #         [0., 0., 1.],
+            #         [0., 1., 0.]])
+            print(out)
+            # Tensor(shape=[3, 3], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        [[0., 1., 0.],
+            #         [0., 0., 1.],
+            #         [0., 1., 0.]])
+    """
+    return _C_ops.index_put_(x, indices, value, accumulate)
+
+
+def index_put(x, indices, value, accumulate=False, name=None):
+    """
+    Outplace version of ``index_put_`` API, the output Tensor will be inplaced with input ``x``.
+    Please refer to :ref:`api_paddle_index_put`.
+
+    Examples:
+        .. code-block:: python
+            import paddle
+
+            x = paddle.zeros([3, 3])
+            value = paddle.ones([3])
+            ix1 = paddle.to_tensor([0,1,2])
+            ix2 = paddle.to_tensor([1,2,1])
+            indices=(ix1,ix2)
+
+            out = paddle.index_put(x,indices,value)
+            print(x)
+            # Tensor(shape=[3, 3], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        [[0., 0., 0.],
+            #         [0., 0., 0.],
+            #         [0., 0., 0.]])
+            print(out)
+            # Tensor(shape=[3, 3], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        [[0., 1., 0.],
+            #         [0., 0., 1.],
+            #         [0., 1., 0.]])
+    """
+    if in_dygraph_mode():
+        return _C_ops.index_put(x, indices, value, accumulate)
+
+    helper = LayerHelper("index_put", **locals())
+    check_variable_and_dtype(
+        x,
+        'x',
+        ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'],
+        'paddle.tensor.manipulation.index_put',
+    )
+    check_variable_and_dtype(
+        value,
+        'value',
+        ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'],
+        'paddle.tensor.manipulation.index_put',
+    )
+
+    out = helper.create_variable_for_type_inference(x.dtype)
+
+    helper.append_op(
+        type='index_put',
+        inputs={
+            'x': x,
+            'indices': indices,
+            'value': value,
+        },
+        outputs={'out': out},
+        attrs={'accumulate': accumulate},
+    )
+    return out
+
+
 def unflatten(x, axis, shape, name=None):
     """
     Expand a certain dimension of the input x Tensor into a desired shape.
@@ -4840,7 +4942,6 @@ def unflatten(x, axis, shape, name=None):
 
     # determine whether the input axis is valid.
     axis = non_negative_axis(x, axis)
-
     if isinstance(shape, (list, tuple)):
         new_shape = (
             list(x.shape[:axis]) + list(shape) + list(x.shape[axis + 1 :])

From fb8ea98c3cc202ff967c607ab123a0a3d2aa59cd Mon Sep 17 00:00:00 2001
From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com>
Date: Thu, 11 May 2023 09:15:19 +0800
Subject: [PATCH 263/405] =?UTF-8?q?=E3=80=90prim=E3=80=91add=20dygraph=20e?=
 =?UTF-8?q?rror=20code=20when=20close=20prim=20flag=20for=20op=20who=20has?=
 =?UTF-8?q?=20composite=20implement=20but=20no=20grad=20kernel=20(#53610)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add no prim no gradOp error code

* delete prim_white_list throw error

* delete invoke_forward_api  throw error

* delete invoke_forward_api  throw error

* review

* review
---
 .../generator/eager_gen.py                    | 22 +++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
index f69b5944c5603..ed21e1171c17c 100644
--- a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
+++ b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
@@ -1893,12 +1893,29 @@ def GenerateHigherOrderNodeCreationCode(self):
             False if self.composite_func_info == {} else True
         )
 
-        if is_composite_grad_api and next_grad_node_creation_str != '':
-            next_grad_node_creation_str = f"""
+        if is_composite_grad_api:
+            if next_grad_node_creation_str != '':
+                next_grad_node_creation_str = f"""
  if (!paddle::prim::PrimCommonUtils::IsEagerPrimEnabled()) {{
     {next_grad_node_creation_str}
  }}
   """
+            else:
+                if not (
+                    self.grad_api_contents["backward_op"] in prim_white_list
+                    or is_invoke_forward_api
+                ):
+
+                    next_grad_node_creation_str = f"""
+ if (!paddle::prim::PrimCommonUtils::IsEagerPrimEnabled()) {{
+    if(trace_backward) {{
+    PADDLE_THROW(phi::errors::Unavailable(
+    \"The Op {self.backward_api_name} doesn't have any grad\"
+    \"op. If you don't intend calculating higher order\"
+    \"derivatives, please set `create_graph`to False.\"));
+  }}
+ }}
+  """
 
         if next_node_generator is not None:
             has_higher_order_node = True
@@ -1918,6 +1935,7 @@ def GenerateHigherOrderNodeCreationCode(self):
     \"op. If you don't intend calculating higher order\"
     \"derivatives, please set `create_graph`to False.\"));
   }}"""
+
         return (
             has_higher_order_node,
             is_invoke_forward_api,

From 00ded2eadaf37b4088b193bd3877ca177a2bd55b Mon Sep 17 00:00:00 2001
From: WangZhen <23097963+0x45f@users.noreply.github.com>
Date: Thu, 11 May 2023 10:49:45 +0800
Subject: [PATCH 264/405] Fix div error when dtype is int64 in static mode
 (#53705)

* Fix div error when dtype is int64 in static mode

* Fix out dtype
---
 python/paddle/fluid/layers/math_op_patch.py   | 10 +++++++++-
 test/dygraph_to_static/test_tensor_methods.py | 19 +++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/python/paddle/fluid/layers/math_op_patch.py b/python/paddle/fluid/layers/math_op_patch.py
index 01426a0c79282..41299a624c12b 100644
--- a/python/paddle/fluid/layers/math_op_patch.py
+++ b/python/paddle/fluid/layers/math_op_patch.py
@@ -409,11 +409,19 @@ def __impl__(self, other_var):
                 self = other_var
                 other_var = tmp
 
+            if (
+                op_type == "divide" or op_type == "elementwise_div"
+            ) and self.dtype in _supported_int_dtype_:
+                self = astype(self, 'float32')
+                other_var = astype(other_var, 'float32')
+
             # NOTE(zhiqiu): the output of compare operator should be bool.
             if method_name in compare_ops:
                 out = create_new_tmp_var(current_block(self), dtype="bool")
             else:
-                out = create_new_tmp_var(current_block(self), dtype=lhs_dtype)
+                out = create_new_tmp_var(
+                    current_block(self), dtype=safe_get_dtype(self)
+                )
 
             axis = -1
             if other_var.ndim > 0 and other_var.shape[0] == -1:
diff --git a/test/dygraph_to_static/test_tensor_methods.py b/test/dygraph_to_static/test_tensor_methods.py
index b1a512c08fddd..fff5476167d03 100644
--- a/test/dygraph_to_static/test_tensor_methods.py
+++ b/test/dygraph_to_static/test_tensor_methods.py
@@ -101,5 +101,24 @@ def test_tensor_clone(self):
         np.testing.assert_allclose(dygraph_res, static_res, rtol=1e-5)
 
 
+@paddle.jit.to_static
+def true_div(x, y):
+    z = x / y
+    return z
+
+
+class TestTrueDiv(unittest.TestCase):
+    def _run(self, to_static):
+        paddle.jit.enable_to_static(to_static)
+        x = paddle.to_tensor([3], dtype='int64')
+        y = paddle.to_tensor([4], dtype='int64')
+        return true_div(x, y).numpy()
+
+    def test_ture_div(self):
+        dygraph_res = self._run(to_static=False)
+        static_res = self._run(to_static=True)
+        np.testing.assert_allclose(dygraph_res, static_res, rtol=1e-5)
+
+
 if __name__ == '__main__':
     unittest.main()

From 0d45ac739a6f85afbb00cd83500d10a013bd0477 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?=
 <83450930+Liyulingyue@users.noreply.github.com>
Date: Thu, 11 May 2023 10:57:59 +0800
Subject: [PATCH 265/405] =?UTF-8?q?=E6=98=87=E8=85=BE=E5=92=8C=E5=AF=92?=
 =?UTF-8?q?=E6=AD=A6=E7=BA=AA=E7=9B=B8=E5=85=B3=E4=BB=A3=E7=A0=81=E9=80=80?=
 =?UTF-8?q?=E5=9C=BA=20npu=E7=9B=B8=E5=85=B3=E4=BB=A3=E7=A0=81=E9=80=80?=
 =?UTF-8?q?=E5=9C=BA2=20(#53568)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../distributed/fleet_executor/message_bus.cc | 10 +-
 paddle/fluid/framework/dlpack_tensor.cc       |  5 -
 paddle/fluid/framework/executor_cache.cc      |  4 -
 .../new_executor/interpreter/data_transfer.cc |  4 +-
 .../interpreter/stream_analyzer.cc            |  2 +-
 paddle/fluid/framework/parallel_executor.cc   |  4 +-
 paddle/fluid/imperative/amp_auto_cast.cc      | 14 +--
 paddle/fluid/inference/analysis/argument.h    |  4 -
 paddle/fluid/inference/api/analysis_config.cc | 26 -----
 .../fluid/inference/api/analysis_predictor.cc |  7 +-
 .../inference/api/details/zero_copy_tensor.cc |  6 +-
 .../inference/api/paddle_analysis_config.h    | 16 ----
 paddle/fluid/inference/api/paddle_api.h       |  1 -
 .../fluid/inference/api/paddle_pass_builder.h | 20 ----
 paddle/fluid/inference/capi_exp/pd_config.cc  |  9 --
 paddle/fluid/inference/capi_exp/pd_config.h   | 16 ----
 paddle/fluid/inference/goapi/config.go        | 18 ----
 paddle/fluid/platform/monitor.h               | 10 --
 paddle/fluid/platform/place.h                 |  6 --
 paddle/fluid/pybind/inference_api.cc          |  3 -
 paddle/fluid/pybind/place.cc                  |  1 -
 paddle/phi/backends/cpu/cpu_info.cc           | 17 ----
 paddle/phi/backends/cpu/cpu_info.h            |  9 --
 paddle/phi/common/place.cc                    |  3 -
 paddle/phi/common/place.h                     | 10 --
 paddle/phi/kernels/funcs/math_function.cc     |  6 --
 .../distributed/fleet/layers/mpu/mp_ops.py    |  7 +-
 .../meta_optimizers/sharding_optimizer.py     |  8 +-
 python/paddle/fluid/executor.py               | 22 ++---
 python/paddle/fluid/framework.py              | 19 +---
 python/paddle/fluid/optimizer.py              | 16 +---
 python/paddle/nn/functional/conv.py           | 15 ---
 python/paddle/nn/functional/loss.py           | 94 +++----------------
 python/paddle/static/amp/decorator.py         | 39 ++------
 python/paddle/static/amp/fp16_lists.py        |  2 -
 python/paddle/static/nn/common.py             |  7 --
 tools/timeline.py                             |  6 --
 37 files changed, 55 insertions(+), 411 deletions(-)

diff --git a/paddle/fluid/distributed/fleet_executor/message_bus.cc b/paddle/fluid/distributed/fleet_executor/message_bus.cc
index 75278334245fa..099be64f66746 100644
--- a/paddle/fluid/distributed/fleet_executor/message_bus.cc
+++ b/paddle/fluid/distributed/fleet_executor/message_bus.cc
@@ -111,8 +111,7 @@ bool MessageBus::Send(int64_t dst_rank,
 #else
   PADDLE_THROW(platform::errors::Unavailable(
       "Fleet executor does not support sending message between different "
-      "ranks when Paddle is compiled with npu or "
-      "isn't compiled with distributed for now."));
+      "ranks when Paddle isn't compiled with distributed for now."));
 #endif
   return true;
 }
@@ -202,10 +201,9 @@ void MessageBus::ListenPort() {
   }
   LOG(INFO) << "Message bus's listen port thread starts successful.";
 #else
-  LOG(WARNING)
-      << "Fleet executor's ListenPort() is a fake function when Paddle is "
-         "compiled with npu or Paddle isn't compiled "
-         "with distributed for now.";
+  LOG(WARNING) << "Fleet executor's ListenPort() is a fake function when "
+                  "Paddle isn't compiled "
+                  "with distributed for now.";
 #endif
 }
 
diff --git a/paddle/fluid/framework/dlpack_tensor.cc b/paddle/fluid/framework/dlpack_tensor.cc
index f80f2359cf701..6b2ab9b3b8462 100644
--- a/paddle/fluid/framework/dlpack_tensor.cc
+++ b/paddle/fluid/framework/dlpack_tensor.cc
@@ -89,11 +89,6 @@ struct DLDeviceVisitor
         platform::errors::Unimplemented("platform::XPUPlace is not supported"));
   }
 
-  inline ::DLDevice operator()(const platform::NPUPinnedPlace &place) const {
-    PADDLE_THROW(platform::errors::Unimplemented(
-        "platform::NPUPinnedPlace is not supported"));
-  }
-
   inline ::DLDevice operator()(const platform::CustomPlace &place) const {
     PADDLE_THROW(platform::errors::Unimplemented(
         "platform::CustomPlace is not supported"));
diff --git a/paddle/fluid/framework/executor_cache.cc b/paddle/fluid/framework/executor_cache.cc
index 5df7c49bc9ac7..d63921977e954 100644
--- a/paddle/fluid/framework/executor_cache.cc
+++ b/paddle/fluid/framework/executor_cache.cc
@@ -50,10 +50,6 @@ static ExecutionStrategy GetExecutionStrategy(const platform::Place &place) {
       execution_strategy.num_threads_ = 1;
       break;
     }
-    case platform::DeviceType::NPU: {
-      execution_strategy.num_threads_ = 1;
-      break;
-    }
     case platform::DeviceType::CUSTOM_DEVICE: {
       execution_strategy.num_threads_ = 1;
       break;
diff --git a/paddle/fluid/framework/new_executor/interpreter/data_transfer.cc b/paddle/fluid/framework/new_executor/interpreter/data_transfer.cc
index 6d0615a4b7666..48aeefc553c07 100644
--- a/paddle/fluid/framework/new_executor/interpreter/data_transfer.cc
+++ b/paddle/fluid/framework/new_executor/interpreter/data_transfer.cc
@@ -196,7 +196,7 @@ void DataTranferHelper::RunAndConstructOpFuncNode(
                                  ? OpFuncType::kGpuSync
                                  : OpFuncType::kGpuAsync;
   } else {
-    // Memcpy in npu and custom devices is asynchronous
+    // Memcpy in custom devices is asynchronous
     new_op_func_node.type_ = OpFuncType::kGpuAsync;
   }
 
@@ -225,7 +225,7 @@ void DataTranferHelper::RunAndConstructOpFuncNode(
     }
   }
 
-  // NOTE(winter-wang): in npu and custom device, D2H kernel is asynchronous.
+  // NOTE(winter-wang): in custom device, D2H kernel is asynchronous.
   // need to explicit synchronization.
   if ((platform::is_custom_place(place)) && op_type == kMemcpyD2H) {
     dev_ctx->Wait();
diff --git a/paddle/fluid/framework/new_executor/interpreter/stream_analyzer.cc b/paddle/fluid/framework/new_executor/interpreter/stream_analyzer.cc
index 37d0c41953f96..38c9ce3d8091e 100644
--- a/paddle/fluid/framework/new_executor/interpreter/stream_analyzer.cc
+++ b/paddle/fluid/framework/new_executor/interpreter/stream_analyzer.cc
@@ -150,7 +150,7 @@ DeviceContext* StreamAnalyzer::ParseDeviceContext(
 
   DeviceContext* dev_ctx = nullptr;
 
-  // only gpu/npu need update. xpu not need, because xpu memcpy op kernel is
+  // only gpu needs update. xpu not need, because xpu memcpy op kernel is
   // synchronous.
   if (platform::is_gpu_place(place_) || platform::is_custom_place(place_)) {
     VLOG(6) << "Parse DeviceContext for " << op_type
diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index 8937d58f8465e..806b8570108b9 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -1331,8 +1331,8 @@ void ParallelExecutor::InitExecutorPrivateMemberInfo(
     device_name = "XPU";
   } else {
     PADDLE_THROW(
-        platform::errors::Unavailable("Only CPU/CUDA/NPU/XPU is supportted. "
-                                      "please use CPU/CUDA/NPU/XPU backend."));
+        platform::errors::Unavailable("Only CPU/CUDA/XPU is supportted. "
+                                      "please use CPU/CUDA/XPU backend."));
   }
 
   VLOG(1) << string::Sprintf(
diff --git a/paddle/fluid/imperative/amp_auto_cast.cc b/paddle/fluid/imperative/amp_auto_cast.cc
index 16ad89d99ac15..85633f1b9cc3a 100644
--- a/paddle/fluid/imperative/amp_auto_cast.cc
+++ b/paddle/fluid/imperative/amp_auto_cast.cc
@@ -52,12 +52,12 @@ OpSupportedInfos(const std::string& place,
       {"CPU", &platform::is_cpu_place},
       {"XPU", &platform::is_xpu_place},
   };
-  PADDLE_ENFORCE_NE(is_target_place.count(query_place),
-                    0,
-                    platform::errors::InvalidArgument(
-                        "The argument `place` should be 'GPU', 'CPU', 'XPU', "
-                        "'NPU', but got '%s'.",
-                        place));
+  PADDLE_ENFORCE_NE(
+      is_target_place.count(query_place),
+      0,
+      platform::errors::InvalidArgument(
+          "The argument `place` should be 'GPU', 'CPU', 'XPU', but got '%s'.",
+          place));
 
   std::unordered_set<std::string> all_ops;
   const auto& op_info = framework::OpInfoMap::Instance().map();
@@ -147,7 +147,7 @@ AmpOperators::AmpOperators()
       OpSupportedInfos("GPU", paddle::framework::proto::VarType::BF16));
   unsupported_bf16_ops_->insert(unsupported_ops_gpu_bf16.begin(),
                                 unsupported_ops_gpu_bf16.end());
-// NOTE: GPU/NPU/XPU is compiled seperatly.
+// NOTE: GPU/XPU is compiled seperatly.
 #elif defined(PADDLE_WITH_XPU)
   auto unsupported_ops_xpu_fp16 = std::get<2>(
       OpSupportedInfos("XPU", paddle::framework::proto::VarType::FP16));
diff --git a/paddle/fluid/inference/analysis/argument.h b/paddle/fluid/inference/analysis/argument.h
index 8a43229af7971..f44c877b93965 100644
--- a/paddle/fluid/inference/analysis/argument.h
+++ b/paddle/fluid/inference/analysis/argument.h
@@ -364,10 +364,6 @@ struct Argument {
                       IpuEnableModelRuntimeExecutor,
                       bool);
 
-  // npu related
-  DECL_ARGUMENT_FIELD(use_npu, UseNpu, bool);
-  DECL_ARGUMENT_FIELD(npu_device_id, NPUDeviceId, int);
-
   // mixed precision related
   DECL_ARGUMENT_FIELD(model_precision, ModelPrecision, int);
   DECL_ARGUMENT_FIELD(mixed_black_list,
diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc
index 4f73fb23c6a5a..af7b8574b863d 100644
--- a/paddle/fluid/inference/api/analysis_config.cc
+++ b/paddle/fluid/inference/api/analysis_config.cc
@@ -56,8 +56,6 @@ PassStrategy *AnalysisConfig::pass_builder() const {
       pass_builder_.reset(new GpuPassStrategy);
     } else if (use_xpu_) {
       pass_builder_.reset(new XpuPassStrategy);
-    } else if (use_npu_) {
-      pass_builder_.reset(new NpuPassStrategy);
     } else if (use_ipu_) {
       LOG(INFO) << "Create IPU IR passes";
       pass_builder_.reset(new IpuPassStrategy);
@@ -506,8 +504,6 @@ AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) {
   CP_MEMBER(use_opencl_);
 
   // NPU related.
-  CP_MEMBER(use_npu_);
-  CP_MEMBER(npu_device_id_);
   CP_MEMBER(nnadapter_config_);
 
   // profile related.
@@ -574,9 +570,6 @@ AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) {
   } else if (use_custom_device_) {
     pass_builder_.reset(new CustomDevicePassStrategy(
         *static_cast<CustomDevicePassStrategy *>(other.pass_builder())));
-  } else if (use_npu_) {
-    pass_builder_.reset(new NpuPassStrategy(
-        *static_cast<NpuPassStrategy *>(other.pass_builder())));
   } else {
     pass_builder_.reset(new CpuPassStrategy(
         *static_cast<CpuPassStrategy *>(other.pass_builder())));
@@ -827,7 +820,6 @@ void AnalysisConfig::Update() {
   // Transfer pass_builder and copy the existing compatible passes.
   if (!pass_builder_ || ((use_gpu() ^ pass_builder_->use_gpu())) ||
       ((use_xpu() ^ pass_builder_->use_xpu())) ||
-      ((use_npu() ^ pass_builder_->use_npu())) ||
       ((use_ipu() ^ pass_builder_->use_ipu())) ||
       ((use_custom_device() ^ pass_builder_->use_custom_device()))) {
     if (use_gpu()) {
@@ -841,13 +833,6 @@ void AnalysisConfig::Update() {
           platform::errors::InvalidArgument(
               "Only one choice can be made between CPU and XPU."));
       pass_builder_.reset(new XpuPassStrategy);
-    } else if (use_npu()) {
-      PADDLE_ENFORCE_EQ(
-          use_gpu(),
-          false,
-          platform::errors::InvalidArgument(
-              "Only one choice can be made between GPU and NPU."));
-      pass_builder_.reset(new NpuPassStrategy);
     } else if (use_custom_device()) {
       PADDLE_ENFORCE_EQ(
           use_gpu(),
@@ -875,14 +860,6 @@ void AnalysisConfig::Update() {
               "Only one choice can be made between CPU and XPU."));
       pass_builder_.reset(new XpuPassStrategy(
           *static_cast<XpuPassStrategy *>(pass_builder_.get())));
-    } else if (use_npu()) {
-      PADDLE_ENFORCE_EQ(
-          use_gpu(),
-          false,
-          platform::errors::InvalidArgument(
-              "Only one choice can be made between GPU and NPU."));
-      pass_builder_.reset(new NpuPassStrategy(
-          *static_cast<NpuPassStrategy *>(pass_builder_.get())));
     } else if (use_custom_device()) {
       PADDLE_ENFORCE_EQ(
           use_gpu(),
@@ -1114,9 +1091,6 @@ std::string AnalysisConfig::SerializeInfoCache() {
     ss << op_type;
   }
 
-  ss << use_npu_;
-  ss << npu_device_id_;
-
   ss << thread_local_stream_;
 
   ss << use_ipu_;
diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc
index 5495f929e8895..a2debb7b43cd3 100644
--- a/paddle/fluid/inference/api/analysis_predictor.cc
+++ b/paddle/fluid/inference/api/analysis_predictor.cc
@@ -148,8 +148,8 @@ phi::Backend ConvertBackend(paddle_infer::PlaceType backend) {
       return phi::Backend::CUSTOM;
     default:
       PADDLE_THROW(paddle::platform::errors::InvalidArgument(
-          "Paddle Inference not support backend, we now only support GPU, XPU, "
-          "NPU and CPU."));
+          "Paddle Inference not support backend, we now only support GPU, XPU "
+          "and CPU."));
       return phi::Backend::CPU;
   }
 }
@@ -1432,9 +1432,6 @@ void AnalysisPredictor::PrepareArgument() {
   argument_->SetIpuCustomPatterns(config_.ipu_custom_patterns_);
 #endif
 
-  argument_->SetUseNpu(config_.use_npu_);
-  argument_->SetNPUDeviceId(config_.npu_device_id());
-
   if (config_.use_mkldnn_) {
     LOG(INFO) << "MKLDNN is enabled";
     argument_->SetMKLDNNEnabledOpTypes(config_.mkldnn_enabled_op_types_);
diff --git a/paddle/fluid/inference/api/details/zero_copy_tensor.cc b/paddle/fluid/inference/api/details/zero_copy_tensor.cc
index f5a9f4f02f8bc..37ee2b4df643d 100644
--- a/paddle/fluid/inference/api/details/zero_copy_tensor.cc
+++ b/paddle/fluid/inference/api/details/zero_copy_tensor.cc
@@ -130,7 +130,7 @@ T *Tensor::mutable_data(PlaceType place) {
     }
     default:
       PADDLE_THROW(paddle::platform::errors::Unavailable(
-          "Only CPU / CUDA / XPU / NPU places is supported. The place `%d` is "
+          "Only CPU / CUDA / XPU places is supported. The place `%d` is "
           "not supported.",
           static_cast<int>(place)));
       break;
@@ -261,7 +261,7 @@ void Tensor::CopyFromCpu(const T *data) {
                          dev_ctx->stream());
 #else
     PADDLE_THROW(paddle::platform::errors::InvalidArgument(
-        "The analysis predictor supports CPU, GPU, NPU and XPU now."));
+        "The analysis predictor supports CPU, GPU and XPU now."));
 #endif
   }
 }
@@ -468,7 +468,7 @@ void Tensor::CopyToCpuImpl(T *data,
     dev_ctx->GetStream()->Synchronize();
 #else
     PADDLE_THROW(paddle::platform::errors::InvalidArgument(
-        "The analysis predictor supports CPU, GPU, NPU and XPU now."));
+        "The analysis predictor supports CPU, GPU and XPU now."));
 #endif
   }
 }
diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h
index d810442810af7..b30838c3680e9 100644
--- a/paddle/fluid/inference/api/paddle_analysis_config.h
+++ b/paddle/fluid/inference/api/paddle_analysis_config.h
@@ -414,12 +414,6 @@ struct PD_INFER_DECL AnalysisConfig {
   /// \return bool Whether the XPU is turned on.
   ///
   bool use_xpu() const { return use_xpu_; }
-  ///
-  /// \brief A boolean state telling whether the NPU is turned on.
-  ///
-  /// \return bool Whether the NPU is turned on.
-  ///
-  bool use_npu() const { return use_npu_; }
   /// \brief A boolean state telling whether the IPU is turned on.
   ///
   /// \return bool Whether the IPU is turned on.
@@ -461,12 +455,6 @@ struct PD_INFER_DECL AnalysisConfig {
   /// \return int The XPU device id.
   ///
   int xpu_device_id() const { return xpu_device_id_; }
-  ///
-  /// \brief Get the NPU device id.
-  ///
-  /// \return int The NPU device id.
-  ///
-  int npu_device_id() const { return npu_device_id_; }
   /// \brief Get the number of IPU device .
   ///
   /// \return int The number of IPU device.
@@ -1083,10 +1071,6 @@ struct PD_INFER_DECL AnalysisConfig {
   bool use_external_stream_{false};
   void* exec_stream_{nullptr};
 
-  // NPU related
-  bool use_npu_{false};
-  int npu_device_id_{0};
-
   // CustomDevice related
   bool use_custom_device_{false};
   int custom_device_id_{0};
diff --git a/paddle/fluid/inference/api/paddle_api.h b/paddle/fluid/inference/api/paddle_api.h
index ee15468c9b81e..de9108aef7755 100644
--- a/paddle/fluid/inference/api/paddle_api.h
+++ b/paddle/fluid/inference/api/paddle_api.h
@@ -360,7 +360,6 @@ struct PD_INFER_DECL NativeConfig : public PaddlePredictor::Config {
   /// GPU related fields.
   bool use_xpu{false};
   bool use_gpu{false};
-  bool use_npu{false};
   int device{0};
   float fraction_of_gpu_memory{
       -1.f};  ///< Change to a float in (0,1] if needed.
diff --git a/paddle/fluid/inference/api/paddle_pass_builder.h b/paddle/fluid/inference/api/paddle_pass_builder.h
index 021b758239eee..db0048cc363db 100644
--- a/paddle/fluid/inference/api/paddle_pass_builder.h
+++ b/paddle/fluid/inference/api/paddle_pass_builder.h
@@ -162,10 +162,6 @@ class PD_INFER_DECL PassStrategy : public PaddlePassBuilder {
   /// \return A bool variable implying whether we are in xpu mode.
   bool use_xpu() const { return use_xpu_; }
 
-  /// \brief Check if we are using npu.
-  /// \return A bool variable implying whether we are in npu mode.
-  bool use_npu() const { return use_npu_; }
-
   /// \brief Check if we are using ipu.
   /// \return A bool variable implying whether we are in ipu mode.
   bool use_ipu() const { return use_ipu_; }
@@ -181,7 +177,6 @@ class PD_INFER_DECL PassStrategy : public PaddlePassBuilder {
   /// \cond Protected
   bool use_xpu_{false};
   bool use_gpu_{false};
-  bool use_npu_{false};
   bool use_ipu_{false};
   bool use_mkldnn_{false};
   bool use_custom_device_{false};
@@ -293,21 +288,6 @@ class PD_INFER_DECL XpuPassStrategy final : public PassStrategy {
   XpuPassStrategy();
 };
 
-/// \class NpuPassStrategy
-/// \brief The NPU passes controller, it is used in AnalysisPredictor with NPU
-/// mode.
-class PD_INFER_DECL NpuPassStrategy final : public PassStrategy {
- public:
-  NpuPassStrategy() : PassStrategy({}) { use_npu_ = true; }
-
-  /// \brief Construct by copying another NpuPassStrategy object.
-  /// \param[in] other The NpuPassStrategy object we want to copy.
-  explicit NpuPassStrategy(const NpuPassStrategy &other)
-      : PassStrategy(other.AllPasses()) {
-    use_npu_ = true;
-  }
-};
-
 /// \class CustomDevicePassStrategy
 /// \brief The CustomDevice passes controller, it is used in AnalysisPredictor
 /// with CustomDevice
diff --git a/paddle/fluid/inference/capi_exp/pd_config.cc b/paddle/fluid/inference/capi_exp/pd_config.cc
index a36573016faf2..ed91e0721c48b 100644
--- a/paddle/fluid/inference/capi_exp/pd_config.cc
+++ b/paddle/fluid/inference/capi_exp/pd_config.cc
@@ -176,11 +176,6 @@ PD_Bool PD_ConfigUseXpu(__pd_keep PD_Config* pd_config) {
   return config->use_xpu();
 }
 
-PD_Bool PD_ConfigUseNpu(__pd_keep PD_Config* pd_config) {
-  CHECK_AND_CONVERT_PD_CONFIG;
-  return config->use_npu();
-}
-
 int32_t PD_ConfigGpuDeviceId(__pd_keep PD_Config* pd_config) {
   CHECK_AND_CONVERT_PD_CONFIG;
   return config->gpu_device_id();
@@ -189,10 +184,6 @@ int32_t PD_ConfigXpuDeviceId(__pd_keep PD_Config* pd_config) {
   CHECK_AND_CONVERT_PD_CONFIG;
   return config->xpu_device_id();
 }
-int32_t PD_ConfigNpuDeviceId(__pd_keep PD_Config* pd_config) {
-  CHECK_AND_CONVERT_PD_CONFIG;
-  return config->npu_device_id();
-}
 
 void PD_ConfigEnableCustomDevice(__pd_keep PD_Config* pd_config,
                                  char* device_type,
diff --git a/paddle/fluid/inference/capi_exp/pd_config.h b/paddle/fluid/inference/capi_exp/pd_config.h
index 458973d2a830a..0a07993bccedc 100644
--- a/paddle/fluid/inference/capi_exp/pd_config.h
+++ b/paddle/fluid/inference/capi_exp/pd_config.h
@@ -222,14 +222,6 @@ PADDLE_CAPI_EXPORT extern void PD_ConfigEnableXpu(
 PADDLE_CAPI_EXPORT extern PD_Bool PD_ConfigUseXpu(
     __pd_keep PD_Config* pd_config);
 ///
-/// \brief A boolean state telling whether the NPU is turned on.
-///
-/// \param[in] pd_onfig config
-/// \return Whether the NPU is turned on.
-///
-PADDLE_CAPI_EXPORT extern PD_Bool PD_ConfigUseNpu(
-    __pd_keep PD_Config* pd_config);
-///
 /// \brief Get the GPU device id.
 ///
 /// \param[in] pd_onfig config
@@ -246,14 +238,6 @@ PADDLE_CAPI_EXPORT extern int32_t PD_ConfigGpuDeviceId(
 PADDLE_CAPI_EXPORT extern int32_t PD_ConfigXpuDeviceId(
     __pd_keep PD_Config* pd_config);
 ///
-/// \brief Get the NPU device id.
-///
-/// \param[in] pd_onfig config
-/// \return The NPU device id.
-///
-PADDLE_CAPI_EXPORT extern int32_t PD_ConfigNpuDeviceId(
-    __pd_keep PD_Config* pd_config);
-///
 /// \brief Turn on custome device.
 ///
 /// \param[in] pd_config config
diff --git a/paddle/fluid/inference/goapi/config.go b/paddle/fluid/inference/goapi/config.go
index c24b941e33e3c..3ed39e94a23ec 100644
--- a/paddle/fluid/inference/goapi/config.go
+++ b/paddle/fluid/inference/goapi/config.go
@@ -230,15 +230,6 @@ func (config *Config) UseXpu() bool {
 	return cvtPDBoolToGo(C.PD_ConfigUseXpu(config.c))
 }
 
-///
-/// \brief A boolean state telling whether the NPU is turned on.
-///
-/// \return bool Whether the NPU is turned on.
-///
-func (config *Config) UseNpu() bool {
-	return cvtPDBoolToGo(C.PD_ConfigUseNpu(config.c))
-}
-
 ///
 /// \brief Get the GPU device id.
 ///
@@ -257,15 +248,6 @@ func (config *Config) XpuDeviceId() int32 {
 	return int32(C.PD_ConfigXpuDeviceId(config.c))
 }
 
-///
-/// \brief Get the NPU device id.
-///
-/// \return int32 The NPU device id.
-///
-func (config *Config) NpuDeviceId() int32 {
-	return int32(C.PD_ConfigNpuDeviceId(config.c))
-}
-
 ///
 /// \brief Get the initial size in MB of the GPU memory pool.
 ///
diff --git a/paddle/fluid/platform/monitor.h b/paddle/fluid/platform/monitor.h
index 08c1f3383bc8d..bfbbcf3db77be 100644
--- a/paddle/fluid/platform/monitor.h
+++ b/paddle/fluid/platform/monitor.h
@@ -190,13 +190,3 @@ class StatRegistry {
   USE_INT_STAT(STAT_gpu13_mem_size); \
   USE_INT_STAT(STAT_gpu14_mem_size); \
   USE_INT_STAT(STAT_gpu15_mem_size)
-
-#define USE_NPU_MEM_STAT            \
-  USE_INT_STAT(STAT_npu0_mem_size); \
-  USE_INT_STAT(STAT_npu1_mem_size); \
-  USE_INT_STAT(STAT_npu2_mem_size); \
-  USE_INT_STAT(STAT_npu3_mem_size); \
-  USE_INT_STAT(STAT_npu4_mem_size); \
-  USE_INT_STAT(STAT_npu5_mem_size); \
-  USE_INT_STAT(STAT_npu6_mem_size); \
-  USE_INT_STAT(STAT_npu7_mem_size)
diff --git a/paddle/fluid/platform/place.h b/paddle/fluid/platform/place.h
index fe20429ae829e..959379260419d 100644
--- a/paddle/fluid/platform/place.h
+++ b/paddle/fluid/platform/place.h
@@ -28,7 +28,6 @@ using Place = phi::Place;
 using CPUPlace = phi::CPUPlace;
 using CUDAPlace = phi::GPUPlace;
 using CUDAPinnedPlace = phi::GPUPinnedPlace;
-using NPUPinnedPlace = phi::NPUPinnedPlace;
 using XPUPlace = phi::XPUPlace;
 using IPUPlace = phi::IPUPlace;
 using CustomPlace = phi::CustomPlace;
@@ -87,11 +86,6 @@ typename Visitor::result_type VisitPlace(const Place &place,
       return typename Visitor::result_type();
 #endif
     }
-    case phi::AllocationType::NPUPINNED: {
-      PADDLE_THROW(platform::errors::Unavailable(
-          "Paddle is not compiled with NPU. Cannot visit npu_pinned"));
-      return typename Visitor::result_type();
-    }
     case phi::AllocationType::IPU: {
 #ifdef PADDLE_WITH_IPU
       platform::IPUPlace p(place.GetDeviceId());
diff --git a/paddle/fluid/pybind/inference_api.cc b/paddle/fluid/pybind/inference_api.cc
index e861c5b5bbe90..09e1e42cfbdd9 100644
--- a/paddle/fluid/pybind/inference_api.cc
+++ b/paddle/fluid/pybind/inference_api.cc
@@ -673,7 +673,6 @@ void BindNativeConfig(py::module *m) {
       .def(py::init<>())
       .def_readwrite("use_gpu", &NativeConfig::use_gpu)
       .def_readwrite("use_xpu", &NativeConfig::use_xpu)
-      .def_readwrite("use_npu", &NativeConfig::use_npu)
       .def_readwrite("device", &NativeConfig::device)
       .def_readwrite("fraction_of_gpu_memory",
                      &NativeConfig::fraction_of_gpu_memory)
@@ -805,10 +804,8 @@ void BindAnalysisConfig(py::module *m) {
       .def("enable_ort_optimization", &AnalysisConfig::EnableORTOptimization)
       .def("use_gpu", &AnalysisConfig::use_gpu)
       .def("use_xpu", &AnalysisConfig::use_xpu)
-      .def("use_npu", &AnalysisConfig::use_npu)
       .def("gpu_device_id", &AnalysisConfig::gpu_device_id)
       .def("xpu_device_id", &AnalysisConfig::xpu_device_id)
-      .def("npu_device_id", &AnalysisConfig::npu_device_id)
       .def("memory_pool_init_size_mb",
            &AnalysisConfig::memory_pool_init_size_mb)
       .def("fraction_of_gpu_memory_for_pool",
diff --git a/paddle/fluid/pybind/place.cc b/paddle/fluid/pybind/place.cc
index ae25b00c93465..477dfef6c7227 100644
--- a/paddle/fluid/pybind/place.cc
+++ b/paddle/fluid/pybind/place.cc
@@ -629,7 +629,6 @@ void BindPlace(pybind11::module &m) {  // NOLINT
           [](platform::Place &self) { return platform::is_custom_place(self); })
       .def("gpu_device_id", [](platform::Place &self) { return self.device; })
       .def("xpu_device_id", [](platform::Place &self) { return self.device; })
-      .def("npu_device_id", [](platform::Place &self) { return self.device; })
       .def("ipu_device_id", [](platform::Place &self) { return self.device; })
       .def("custom_device_id",
            [](platform::Place &self) { return self.device; })
diff --git a/paddle/phi/backends/cpu/cpu_info.cc b/paddle/phi/backends/cpu/cpu_info.cc
index 0e91898fc3b93..4a96a947eb364 100644
--- a/paddle/phi/backends/cpu/cpu_info.cc
+++ b/paddle/phi/backends/cpu/cpu_info.cc
@@ -110,23 +110,6 @@ size_t CUDAPinnedMaxChunkSize() {
   return CUDAPinnedMaxAllocSize() / 256;
 }
 
-size_t NPUPinnedMaxAllocSize() {
-  // For distributed systems, it requires configuring and limiting
-  // the fraction of memory to use.
-  return FLAGS_fraction_of_cuda_pinned_memory_to_use * CpuTotalPhysicalMemory();
-}
-
-size_t NPUPinnedMinChunkSize() {
-  // Allow to allocate the minimum chunk size is 64 KB.
-  return 1 << 16;
-}
-
-size_t NPUPinnedMaxChunkSize() {
-  // Allow to allocate the maximum chunk size is roughly 1/256 of NPU_PINNED
-  // memory.
-  return NPUPinnedMaxAllocSize() / 256;
-}
-
 #ifdef PADDLE_WITH_XBYAK
 static Xbyak::util::Cpu cpu;
 bool MayIUse(const cpu_isa_t cpu_isa) {
diff --git a/paddle/phi/backends/cpu/cpu_info.h b/paddle/phi/backends/cpu/cpu_info.h
index 3bc8551cd95d2..eb3175541afee 100644
--- a/paddle/phi/backends/cpu/cpu_info.h
+++ b/paddle/phi/backends/cpu/cpu_info.h
@@ -75,15 +75,6 @@ size_t CUDAPinnedMinChunkSize();
 //! Get the maximum chunk size for buddy allocator.
 size_t CUDAPinnedMaxChunkSize();
 
-//! Get the maximum allocation size for a machine.
-size_t NPUPinnedMaxAllocSize();
-
-//! Get the minimum chunk size for buddy allocator.
-size_t NPUPinnedMinChunkSize();
-
-//! Get the maximum chunk size for buddy allocator.
-size_t NPUPinnedMaxChunkSize();
-
 typedef enum {
   isa_any,
   sse42,
diff --git a/paddle/phi/common/place.cc b/paddle/phi/common/place.cc
index d97f3f4fd7598..fe15be4b2b909 100644
--- a/paddle/phi/common/place.cc
+++ b/paddle/phi/common/place.cc
@@ -35,8 +35,6 @@ const char *AllocationTypeStr(AllocationType type) {
       return "gpu_pinned";
     case AllocationType::XPU:
       return "xpu";
-    case AllocationType::NPUPINNED:
-      return "npu_pinned";
     case AllocationType::IPU:
       return "ipu";
     default:
@@ -55,7 +53,6 @@ std::string Place::DebugString() const {
     os << AllocationTypeStr(alloc_type_);
   }
   if (alloc_type_ == AllocationType::GPUPINNED ||
-      alloc_type_ == AllocationType::NPUPINNED ||
       alloc_type_ == AllocationType::CPU) {
     os << ")";
   } else {
diff --git a/paddle/phi/common/place.h b/paddle/phi/common/place.h
index fcd9634a1892d..d8709c73d5ad6 100644
--- a/paddle/phi/common/place.h
+++ b/paddle/phi/common/place.h
@@ -32,7 +32,6 @@ enum class AllocationType : int8_t {
   GPUPINNED = 3,
   XPU = 4,
   NPU = 5,
-  NPUPINNED = 6,
   IPU = 7,
   CUSTOM = 9,
 };
@@ -163,15 +162,6 @@ class XPUPlace : public Place {
       : Place(AllocationType::XPU, place.GetDeviceId()) {}
 };
 
-class NPUPinnedPlace : public Place {
- public:
-  NPUPinnedPlace() : Place(AllocationType::NPUPINNED) {}
-
-  NPUPinnedPlace(const NPUPinnedPlace&) = default;
-  NPUPinnedPlace(const Place& place UNUSED)  // NOLINT
-      : Place(AllocationType::NPUPINNED) {}
-};
-
 class IPUPlace : public Place {
  public:
   IPUPlace() : Place(AllocationType::IPU, 0) {}
diff --git a/paddle/phi/kernels/funcs/math_function.cc b/paddle/phi/kernels/funcs/math_function.cc
index 1a61f48579188..6a907b211afce 100644
--- a/paddle/phi/kernels/funcs/math_function.cc
+++ b/paddle/phi/kernels/funcs/math_function.cc
@@ -161,12 +161,6 @@ void set_constant_with_place<phi::XPUPlace>(const phi::DeviceContext& context,
 #endif
 }
 
-template <>
-void set_constant_with_place<phi::NPUPinnedPlace>(
-    const phi::DeviceContext& context, phi::DenseTensor* tensor, float value) {
-  PADDLE_THROW(phi::errors::Unimplemented("NPUPinnedPlace is not supported"));
-}
-
 template <>
 void set_constant_with_place<phi::IPUPlace>(const phi::DeviceContext& context,
                                             phi::DenseTensor* tensor,
diff --git a/python/paddle/distributed/fleet/layers/mpu/mp_ops.py b/python/paddle/distributed/fleet/layers/mpu/mp_ops.py
index 9ad7b29ec7e33..b435795adedc2 100644
--- a/python/paddle/distributed/fleet/layers/mpu/mp_ops.py
+++ b/python/paddle/distributed/fleet/layers/mpu/mp_ops.py
@@ -15,7 +15,6 @@
 import paddle
 from paddle import _legacy_C_ops
 from paddle.distributed import collective
-from paddle.fluid import core
 from paddle.fluid.data_feeder import check_dtype, check_variable_and_dtype
 from paddle.framework import LayerHelper, _create_tensor, in_dygraph_mode
 from paddle.nn import Layer
@@ -551,11 +550,7 @@ def _parallel_linear(
     )
 
     # NOTE: npu linear function use matmul_v2 but linear use matmul
-    linear_function = (
-        _linear
-        if core.is_compiled_with_custom_device('npu')
-        else paddle.nn.functional.linear
-    )
+    linear_function = paddle.nn.functional.linear
     linear_out = linear_function(
         x,
         linear.weight,
diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py
index 88ad5cbfdc32c..29be8a87fa7d8 100755
--- a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py
+++ b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py
@@ -595,9 +595,6 @@ def _adapt_amp_clip_without_sharding(self):
         # amp inf_var & clip global_norm_var
 
         rings = [self.mp_ring_id, self.pp_ring_id]
-        # FIXME(wangxi): some problem with NPU found_finite, need sync with DP
-        if core.is_compiled_with_custom_device('npu'):
-            rings += [self.dp_ring_id]
         FP16Utils.sync_amp_check_nan_inf(main_block, rings)
 
         gradientclip_helper = GradientClipHelper(None)
@@ -719,10 +716,7 @@ def minimize_impl(
         self._recreate_not_persist_param_as_var()
 
         self._dump_program_for_debug()
-
-        # GPU need to wait server ready, GPU and NPU is Layered connection
-        if not core.is_compiled_with_custom_device('npu'):
-            self._wait()
+        self._wait()
         return optimize_ops, params_grads
 
     def _init_pair_comm(self, pair, ring_id):
diff --git a/python/paddle/fluid/executor.py b/python/paddle/fluid/executor.py
index da1521044c94d..21354647f1382 100755
--- a/python/paddle/fluid/executor.py
+++ b/python/paddle/fluid/executor.py
@@ -1988,14 +1988,9 @@ def _run_from_dataset(
             for var in program.global_block().vars.values():
                 if var.is_data:
                     data_vars.append(var)
-            if core.is_compiled_with_custom_device('npu'):
-                dataset = paddle.fluid.DatasetFactory().create_dataset(
-                    'InMemoryDataset'
-                )
-            else:
-                dataset = paddle.fluid.DatasetFactory().create_dataset(
-                    'FileInstantDataset'
-                )
+            dataset = paddle.fluid.DatasetFactory().create_dataset(
+                'FileInstantDataset'
+            )
             dataset.set_batch_size(1)
             dataset.set_thread(1)
             dataset.set_filelist(['None'])
@@ -2165,14 +2160,9 @@ def _get_dataset():
             for var in program.global_block().vars.values():
                 if var.is_data:
                     data_vars.append(var)
-            if core.is_compiled_with_custom_device('npu'):
-                dataset = paddle.fluid.DatasetFactory().create_dataset(
-                    'InMemoryDataset'
-                )
-            else:
-                dataset = paddle.fluid.DatasetFactory().create_dataset(
-                    'FileInstantDataset'
-                )
+            dataset = paddle.fluid.DatasetFactory().create_dataset(
+                'FileInstantDataset'
+            )
             dataset.set_batch_size(1)
             dataset.set_thread(1)
             dataset.set_filelist(['None'])
diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py
index 184d6cb326e5b..88354f8226f1d 100644
--- a/python/paddle/fluid/framework.py
+++ b/python/paddle/fluid/framework.py
@@ -597,21 +597,6 @@ def _current_expected_place():
                     "You are using XPU version Paddle, but your XPU device is not set properly. CPU device will be used by default."
                 )
                 _global_expected_place_ = core.CPUPlace()
-        elif core.is_compiled_with_custom_device("npu"):
-            # TODO(duanyanhui): Optimize DeviceManager and Return all expected places when device registered in DeviceManager is greater than 1.
-            try:
-                device_count = core.get_custom_device_count("npu")
-            except Exception as e:
-                device_count = 0
-            if device_count > 0:
-                _global_expected_place_ = core.CustomPlace(
-                    "npu", _custom_device_ids("npu")[0]
-                )
-            else:
-                warnings.warn(
-                    "You are using NPU version Paddle, but your NPU device is not set properly. CPU device will be used by default."
-                )
-                _global_expected_place_ = core.CPUPlace()
         else:
             _global_expected_place_ = core.CPUPlace()
 
@@ -7454,9 +7439,9 @@ def device_guard(device=None):
         device, index = device.split(':')
         if device == 'cpu':
             raise ValueError("Should not set device id for cpu.")
-    if device not in ['cpu', 'gpu', 'xpu', 'npu', '', None]:
+    if device not in ['cpu', 'gpu', 'xpu', '', None]:
         raise ValueError(
-            "The Attr(device) should be 'cpu' 'npu' or 'gpu', and it can also be empty string or None "
+            "The Attr(device) should be 'cpu' or 'gpu', and it can also be empty string or None "
             "when there is no need to specify device. But received %s" % device
         )
     if index:
diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py
index f6bd5dbd37c33..173084fdc8d00 100755
--- a/python/paddle/fluid/optimizer.py
+++ b/python/paddle/fluid/optimizer.py
@@ -4554,9 +4554,7 @@ def train_reader():
 
     def __init__(self, optimizer, num_microbatches=1, start_cpu_core_id=0):
         self._device = 'cpu'
-        if core.is_compiled_with_custom_device('npu'):
-            self._device = "npu"
-        elif core.is_compiled_with_cuda():
+        if core.is_compiled_with_cuda():
             self._device = "gpu"
         if in_dygraph_mode():
             raise Exception("In dygraph, don't support PipelineOptimizer.")
@@ -4945,8 +4943,8 @@ def _get_op_device_attr(self, op):
             else None
         )
         if device:
-            assert device[0:3] == 'gpu' or device[0:3] == 'npu', (
-                "Now, only gpu and npu devices are "
+            assert device[0:3] == 'gpu', (
+                "Now, only gpu devices are "
                 "supported in pipeline parallemism."
             )
         return device
@@ -5148,8 +5146,8 @@ def _check_validation(self, block):
                 continue
 
             dev_type = device.split(':')[0]
-            assert dev_type == "gpu" or dev_type == 'npu', (
-                "Now only gpu and npu devices are supported "
+            assert dev_type == "gpu", (
+                "Now only gpu devices are supported "
                 "for pipeline parallelism."
             )
 
@@ -6388,8 +6386,6 @@ def device_cmp(device1, device2):
             dev_index = int(dev.split(":")[1])
             if core.is_compiled_with_cuda():
                 place_list.append(core.CUDAPlace(dev_index % 1))
-            elif paddle.is_compiled_with_custom_device('npu'):
-                place_list.append(paddle.CustomPlace('npu', dev_index % 1))
 
         # Step6: Split startup program
         new_startup_program = self._split_startup_program(
@@ -6412,8 +6408,6 @@ def device_cmp(device1, device2):
 
         if core.is_compiled_with_cuda():
             place_id = int(os.getenv("FLAGS_selected_gpus", "0"))
-        elif core.is_compiled_with_custom_device('npu'):
-            place_id = int(os.getenv("FLAGS_selected_npus", "0"))
         # A pass to move the recv op to the beginning of
         # the forward/backward phase
         self._mv_head_recv(program_list[self.local_rank])
diff --git a/python/paddle/nn/functional/conv.py b/python/paddle/nn/functional/conv.py
index bd3100a644c0d..e44847b0a9bab 100644
--- a/python/paddle/nn/functional/conv.py
+++ b/python/paddle/nn/functional/conv.py
@@ -16,7 +16,6 @@
 from paddle.device import (
     get_all_custom_device_type,
     is_compiled_with_cuda,
-    is_compiled_with_custom_device,
     is_compiled_with_rocm,
 )
 from paddle.fluid.framework import _global_flags, in_dygraph_mode
@@ -465,13 +464,6 @@ def conv1d(
         l_type = 'depthwise_conv2d'
         use_cudnn = False
 
-    # NPU only supports depthwise_conv2d when  "input_channel = output_channel = groups"
-    if is_compiled_with_custom_device('npu'):
-        if num_channels == groups and num_channels == num_filters:
-            l_type = 'depthwise_conv2d'
-        else:
-            l_type = 'conv2d'
-
     squeeze_aixs = -3 if channel_last else -2
     x = unsqueeze(x, axis=[squeeze_aixs])
 
@@ -755,13 +747,6 @@ def conv2d(
 
     use_mkldnn = _global_flags()["FLAGS_use_mkldnn"]
 
-    # NPU only supports depthwise_conv2d when  "input_channel = output_channel = groups"
-    if is_compiled_with_custom_device('npu'):
-        if num_channels == groups and num_channels == num_filters:
-            l_type = 'depthwise_conv2d'
-        else:
-            l_type = 'conv2d'
-
     if (
         is_compiled_with_cuda()
         and get_flags("FLAGS_conv2d_disable_cudnn")[
diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py
index ae7b8813a0e3e..f27fa360609af 100644
--- a/python/paddle/nn/functional/loss.py
+++ b/python/paddle/nn/functional/loss.py
@@ -16,7 +16,7 @@
 
 # TODO: define loss functions of neural network
 import paddle
-from paddle import _C_ops, _legacy_C_ops, fluid, in_dynamic_mode
+from paddle import _C_ops, fluid, in_dynamic_mode
 from paddle.framework import core
 from paddle.static.nn.control_flow import Assert
 from paddle.utils import deprecated
@@ -269,51 +269,15 @@ def fluid_softmax_with_cross_entropy(
     if input_dims - 1 == label_dims:
         label = paddle.unsqueeze(label, axis=axis)
     if in_dygraph_mode():
-        if core.is_compiled_with_custom_device("npu"):
-            if not soft_label:
-                valid_label = (
-                    paddle.cast(label != ignore_index, dtype=label.dtype)
-                    * label
-                )
-                softmax, loss = _legacy_C_ops.softmax_with_cross_entropy(
-                    logits,
-                    valid_label,
-                    'soft_label',
-                    soft_label,
-                    'ignore_index',
-                    ignore_index,
-                    'numeric_stable_mode',
-                    numeric_stable_mode,
-                    'axis',
-                    axis,
-                    'use_softmax',
-                    True,
-                )
-            else:
-                softmax, loss = _legacy_C_ops.softmax_with_cross_entropy(
-                    logits,
-                    label,
-                    'soft_label',
-                    soft_label,
-                    'ignore_index',
-                    ignore_index,
-                    'numeric_stable_mode',
-                    numeric_stable_mode,
-                    'axis',
-                    axis,
-                    'use_softmax',
-                    True,
-                )
-        else:
-            softmax, loss = _C_ops.cross_entropy_with_softmax(
-                logits,
-                label,
-                soft_label,
-                True,
-                numeric_stable_mode,
-                ignore_index,
-                axis,
-            )
+        softmax, loss = _C_ops.cross_entropy_with_softmax(
+            logits,
+            label,
+            soft_label,
+            True,
+            numeric_stable_mode,
+            ignore_index,
+            axis,
+        )
         if not return_softmax:
             return loss
         else:
@@ -2734,41 +2698,9 @@ def cross_entropy(
             valid_label = (
                 paddle.cast(label != ignore_index, dtype=label.dtype) * label
             )
-        if core.is_compiled_with_custom_device("npu"):
-            if not soft_label:
-                _, out = _legacy_C_ops.softmax_with_cross_entropy(
-                    input,
-                    valid_label,
-                    'soft_label',
-                    soft_label,
-                    'ignore_index',
-                    ignore_index,
-                    'numeric_stable_mode',
-                    True,
-                    'axis',
-                    axis,
-                    'use_softmax',
-                    use_softmax,
-                )
-            else:
-                _, out = _legacy_C_ops.softmax_with_cross_entropy(
-                    input,
-                    label,
-                    'soft_label',
-                    soft_label,
-                    'ignore_index',
-                    ignore_index,
-                    'numeric_stable_mode',
-                    True,
-                    'axis',
-                    axis,
-                    'use_softmax',
-                    use_softmax,
-                )
-        else:
-            _, out = _C_ops.cross_entropy_with_softmax(
-                input, label, soft_label, use_softmax, True, ignore_index, axis
-            )
+        _, out = _C_ops.cross_entropy_with_softmax(
+            input, label, soft_label, use_softmax, True, ignore_index, axis
+        )
 
         if weight is not None:
 
diff --git a/python/paddle/static/amp/decorator.py b/python/paddle/static/amp/decorator.py
index fc0aaac92bf46..43dc849925a9a 100644
--- a/python/paddle/static/amp/decorator.py
+++ b/python/paddle/static/amp/decorator.py
@@ -220,24 +220,7 @@ def backward(
         """
         train_program = loss.block.program
         self._train_program = train_program
-
-        # NOTE(zhiqiu): _float_status is only used for NPU.
-        if core.is_compiled_with_custom_device('npu'):
-            float_status = paddle.static.data(
-                name="float_status", shape=[8], dtype='float32'
-            )
-            self._train_program.global_block().append_op(
-                type="alloc_float_status",
-                outputs={"FloatStatus": float_status},
-            )
-            self._train_program.global_block().append_op(
-                type="clear_float_status",
-                inputs={"FloatStatus": float_status},
-                outputs={"FloatStatusOut": float_status},
-            )
-            self._float_status = float_status
-        else:
-            self._float_status = None
+        self._float_status = None
 
         with program_guard(self._train_program, startup_program):
             self._init_amp_var()
@@ -476,27 +459,17 @@ def _check_finite_and_unscale(self, params_grads):
         if self._is_distributed:
             # if distributed, split check_finite_and_unscale to overlap
             # unscale with communication
-            if core.is_compiled_with_custom_device('npu'):
-                with self._train_program._optimized_guard(grads):
+            for p, g in params_grads:
+                with self._train_program._optimized_guard([p, g]):
                     _, found_inf = check_finite_and_unscale(
-                        grads,
+                        [
+                            g,
+                        ],
                         self._loss_scaling,
                         name="find_infinite_scale",
                         float_status=self._float_status,
                     )
                     found_infs.append(found_inf)
-            else:
-                for p, g in params_grads:
-                    with self._train_program._optimized_guard([p, g]):
-                        _, found_inf = check_finite_and_unscale(
-                            [
-                                g,
-                            ],
-                            self._loss_scaling,
-                            name="find_infinite_scale",
-                            float_status=self._float_status,
-                        )
-                        found_infs.append(found_inf)
         elif self._use_pure_fp16:
             if fp32_grads:
                 with self._train_program._optimized_guard(fp32_grads):
diff --git a/python/paddle/static/amp/fp16_lists.py b/python/paddle/static/amp/fp16_lists.py
index 23da3f1c7469a..dcf13c0847d6e 100644
--- a/python/paddle/static/amp/fp16_lists.py
+++ b/python/paddle/static/amp/fp16_lists.py
@@ -97,8 +97,6 @@ def _get_sys_unsupported_list(dtype):
     device = None
     if core.is_compiled_with_xpu():
         device = 'XPU'
-    elif core.is_compiled_with_custom_device('npu'):
-        device = 'NPU'
     else:
         device = 'GPU'
     _, _, sys_unsupported_list = core.op_supported_infos(device, var_type)
diff --git a/python/paddle/static/nn/common.py b/python/paddle/static/nn/common.py
index 061909ceac456..cd0b96c04e8c3 100644
--- a/python/paddle/static/nn/common.py
+++ b/python/paddle/static/nn/common.py
@@ -948,13 +948,6 @@ def conv2d(
     ):
         l_type = 'depthwise_conv2d'
 
-    # NPU only supports depthwise_conv2d when  "input_channel = output_channel = groups"
-    if core.is_compiled_with_custom_device('npu'):
-        if num_channels == groups and num_channels == num_filters:
-            l_type = 'depthwise_conv2d'
-        else:
-            l_type = 'conv2d'
-
     helper = LayerHelper(l_type, **locals())
     dtype = helper.input_dtype()
 
diff --git a/tools/timeline.py b/tools/timeline.py
index 0ef4a886f4623..c5521bdfb107f 100644
--- a/tools/timeline.py
+++ b/tools/timeline.py
@@ -212,12 +212,6 @@ def _allocate_pids(self):
                     self._chrome_trace.emit_pid(
                         "memory usage on %s:cudapinnedplace:%d" % (k, 0), pid
                     )
-                if (k, 0, "NPU") not in self._mem_devices:
-                    pid = self._allocate_pid()
-                    self._mem_devices[(k, 0, "NPU")] = pid
-                    self._chrome_trace.emit_pid(
-                        "memory usage on %s:npu:%d" % (k, 0), pid
-                    )
 
     def _allocate_events(self):
         for k, profile_pb in self._profile_dict.items():

From 7ff9f5eae29386dca6217dbc8725b05881994849 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Thu, 11 May 2023 11:39:56 +0800
Subject: [PATCH 266/405] Try to crop the flash-attention lib.

---
 cmake/external/flashattn.cmake                      |  4 ++--
 paddle/fluid/operators/fused/fused_gate_attention.h | 12 ++----------
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index 21267f3f023fc..8d09f7823a4da 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -19,8 +19,8 @@ add_definitions(-DPADDLE_WITH_FLASHATTN)
 set(FLASHATTN_PREFIX_DIR ${THIRD_PARTY_PATH}/flashattn)
 set(FLASHATTN_SOURCE_SUBDIR csrc/flash_attn)
 set(FLASHATTN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flashattn)
-set(FLASHATTN_REPOSITORY ${GIT_URL}/PaddlePaddle/flash-attention.git)
-set(FLASHATTN_TAG 209f02b3852f1f6672b01e00729f20f0e52633a9)
+set(FLASHATTN_REPOSITORY ${GIT_URL}/Xreki/flash-attention.git)
+set(FLASHATTN_TAG reduce_lib)
 
 set(FLASHATTN_INCLUDE_DIR
     "${FLASHATTN_INSTALL_DIR}/include"
diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 1aa22751d3e38..b8a0ae53cc3a2 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -200,16 +200,8 @@ struct GateAttentionConfig {
       return false;
     }
 
-    if (merge_qkv) {
-      switch (head_dim) {
-        case 16:
-        case 32:
-        case 64:
-        case 128:
-          return use_flash_attn;
-        default:
-          return false;
-      }
+    if (merge_qkv && (head_dim % 8 == 0) && (head_dim <= 128)) {
+      return use_flash_attn;
     } else {
       return false;
     }

From d67d74cc172c339e721deeef99287d7bce0ba29e Mon Sep 17 00:00:00 2001
From: houj04 <35131887+houj04@users.noreply.github.com>
Date: Thu, 11 May 2023 13:22:39 +0800
Subject: [PATCH 267/405] [XPU] update log for bkcl function calls. (#53609)

* [XPU] update log for bkcl function calls.

* minor update

* revert unnecessary modifications.
---
 .../collective/process_group_bkcl.cc          | 108 +++++++++++++++---
 .../fused/fused_gemm_epilogue_op_xpu.cc       |   1 -
 .../fluid/platform/device/xpu/bkcl_helper.h   |  12 ++
 3 files changed, 106 insertions(+), 15 deletions(-)

diff --git a/paddle/fluid/distributed/collective/process_group_bkcl.cc b/paddle/fluid/distributed/collective/process_group_bkcl.cc
index 47dd2241c2cde..a3c3e085c6df5 100644
--- a/paddle/fluid/distributed/collective/process_group_bkcl.cc
+++ b/paddle/fluid/distributed/collective/process_group_bkcl.cc
@@ -115,7 +115,13 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::Recv(
           const phi::DenseTensor& input,
           BKCLContext_t comm,
           const XPUStream& stream) {
-        VLOG(3) << "bkcl_recv";
+        VLOG(3) << "calling bkcl_recv"
+                << ", rank_id: " << platform::GetBKCLRankID(comm)
+                << ", dev_id: " << platform::GetBKCLDevID(comm)
+                << ", nranks: " << platform::GetBKCLNRanks(comm)
+                << ", src_rank: " << src_rank << ", numel: " << output->numel()
+                << ", dtype: " << output->type() << ", sync_op: " << sync_op
+                << ", use_calc_stream: " << use_calc_stream;
         int r = bkcl_recv(comm,
                           output->data(),
                           output->numel(),
@@ -148,7 +154,14 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::Send(
           const phi::DenseTensor& input,
           BKCLContext_t comm,
           const XPUStream& stream) {
-        VLOG(3) << "bkcl_send";
+        VLOG(3) << "calling bkcl_send"
+                << ", rank_id: " << platform::GetBKCLRankID(comm)
+                << ", dev_id: " << platform::GetBKCLDevID(comm)
+                << ", nranks: " << platform::GetBKCLNRanks(comm)
+                << ", dst_rank: " << dst_rank
+                << ", input numel: " << input.numel()
+                << ", dtype: " << input.type() << ", sync_op: " << sync_op
+                << ", use_calc_stream: " << use_calc_stream;
         int r = bkcl_send(comm,
                           input.data(),
                           input.numel(),
@@ -276,7 +289,14 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::AllReduce(
           const phi::DenseTensor& input,
           BKCLContext_t comm,
           const XPUStream& stream) {
-        VLOG(3) << "bkcl_all_reduce";
+        VLOG(3) << "calling bkcl_all_reduce"
+                << ", rank_id: " << platform::GetBKCLRankID(comm)
+                << ", dev_id: " << platform::GetBKCLDevID(comm)
+                << ", nranks: " << platform::GetBKCLNRanks(comm)
+                << ", numel: " << input.numel() << ", dtype: " << input.type()
+                << ", reduce_type: " << ToBKCLRedType(opts.reduce_op)
+                << ", sync_op: " << sync_op
+                << ", use_calc_stream: " << use_calc_stream;
         int r =
             bkcl_all_reduce(comm,
                             input.data(),
@@ -307,7 +327,13 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::Broadcast(
           BKCLContext_t comm,
           const XPUStream& stream) {
         int root = opts.source_rank + opts.source_root;
-        VLOG(3) << "bkcl_broadcast";
+        VLOG(3) << "calling bkcl_broadcast"
+                << ", rank_id: " << platform::GetBKCLRankID(comm)
+                << ", dev_id: " << platform::GetBKCLDevID(comm)
+                << ", nranks: " << platform::GetBKCLNRanks(comm)
+                << ", root: " << root << ", numel: " << input.numel()
+                << ", dtype: " << input.type() << ", sync_op: " << sync_op
+                << ", use_calc_stream: " << use_calc_stream;
         int r =
             bkcl_broadcast(comm,
                            input.data(),
@@ -346,7 +372,13 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::AllGather(
           const phi::DenseTensor& input,
           BKCLContext_t comm,
           const XPUStream& stream) {
-        VLOG(3) << "bkcl_all_gather";
+        VLOG(3) << "calling bkcl_all_gather"
+                << ", rank_id: " << platform::GetBKCLRankID(comm)
+                << ", dev_id: " << platform::GetBKCLDevID(comm)
+                << ", nranks: " << platform::GetBKCLNRanks(comm)
+                << ", numel: " << in_tensor_maybe_partial.numel()
+                << ", dtype: " << input.type() << ", sync_op: " << sync_op
+                << ", use_calc_stream: " << use_calc_stream;
         int r =
             bkcl_all_gather(comm,
                             in_tensor_maybe_partial.data(),
@@ -375,7 +407,15 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::Reduce(
           const phi::DenseTensor& input,
           BKCLContext_t comm,
           const XPUStream& stream) {
-        VLOG(3) << "bkcl_reduce";
+        VLOG(3) << "calling bkcl_reduce"
+                << ", rank_id: " << platform::GetBKCLRankID(comm)
+                << ", dev_id: " << platform::GetBKCLDevID(comm)
+                << ", nranks: " << platform::GetBKCLNRanks(comm)
+                << ", root: " << opts.root_rank << ", numel: " << input.numel()
+                << ", dtype: " << input.type()
+                << ", reduce_type: " << ToBKCLRedType(opts.reduce_op)
+                << ", sync_op: " << sync_op
+                << ", use_calc_stream: " << use_calc_stream;
         int r = bkcl_reduce(comm,
                             input.data(),
                             output->data(),
@@ -405,7 +445,14 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::ReduceScatter(
           const phi::DenseTensor& input,
           BKCLContext_t comm,
           const XPUStream& stream) {
-        VLOG(3) << "bkcl_reduce_scatter";
+        VLOG(3) << "calling bkcl_reduce_scatter"
+                << ", rank_id: " << platform::GetBKCLRankID(comm)
+                << ", dev_id: " << platform::GetBKCLDevID(comm)
+                << ", nranks: " << platform::GetBKCLNRanks(comm)
+                << ", numel: " << output->numel() << ", dtype: " << input.type()
+                << ", reduce_type: " << ToBKCLRedType(opts.reduce_op)
+                << ", sync_op: " << sync_op
+                << ", use_calc_stream: " << use_calc_stream;
         int r = bkcl_reduce_scatter(
             comm,
             input.data(),
@@ -491,8 +538,13 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::AllReduce(
           const phi::DenseTensor& input,
           BKCLContext_t comm,
           const XPUStream& stream) {
-        VLOG(3) << "bkcl_all_reduce";
-
+        VLOG(3) << "calling bkcl_all_reduce"
+                << ", rank_id: " << platform::GetBKCLRankID(comm)
+                << ", dev_id: " << platform::GetBKCLDevID(comm)
+                << ", nranks: " << platform::GetBKCLNRanks(comm)
+                << ", numel: " << input.numel() << ", dtype: " << input.type()
+                << ", reduce_type: " << ToBKCLRedType(opts.reduce_op)
+                << ", sync_op: " << true << ", use_calc_stream: " << false;
         int r =
             bkcl_all_reduce(comm,
                             input.data(),
@@ -535,7 +587,13 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::AllReduce(
           const phi::DenseTensor& input,
           BKCLContext_t comm,
           const XPUStream& stream) {
-        VLOG(3) << "bkcl_all_reduce";
+        VLOG(3) << "calling bkcl_all_reduce"
+                << ", rank_id: " << platform::GetBKCLRankID(comm)
+                << ", dev_id: " << platform::GetBKCLDevID(comm)
+                << ", nranks: " << platform::GetBKCLNRanks(comm)
+                << ", numel: " << input.numel() << ", dtype: " << input.type()
+                << ", reduce_type: " << ToBKCLRedType(opts.reduce_op)
+                << ", sync_op: " << sync_op << ", use_calc_stream: " << false;
         int r =
             bkcl_all_reduce(comm,
                             input.data(),
@@ -580,7 +638,13 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::Broadcast(
           const XPUStream& stream) {
         const auto root =
             opts.source_rank * in_tensors.size() + opts.source_root;
-        VLOG(3) << "bkcl_broadcast";
+        VLOG(3) << "calling bkcl_broadcast"
+                << ", rank_id: " << platform::GetBKCLRankID(comm)
+                << ", dev_id: " << platform::GetBKCLDevID(comm)
+                << ", nranks: " << platform::GetBKCLNRanks(comm)
+                << ", root: " << root << ", numel: " << input.numel()
+                << ", dtype: " << input.type() << ", sync_op: " << true
+                << ", use_calc_stream: " << false;
         int r =
             bkcl_broadcast(comm,
                            input.data(),
@@ -626,7 +690,13 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::Broadcast(
           const XPUStream& stream) {
         const auto root =
             opts.source_rank * in_tensors.size() + opts.source_root;
-        VLOG(3) << "bkcl_broadcast";
+        VLOG(3) << "calling bkcl_broadcast"
+                << ", rank_id: " << platform::GetBKCLRankID(comm)
+                << ", dev_id: " << platform::GetBKCLDevID(comm)
+                << ", nranks: " << platform::GetBKCLNRanks(comm)
+                << ", root: " << root << ", numel: " << input.numel()
+                << ", dtype: " << input.type() << ", sync_op: " << sync_op
+                << ", use_calc_stream: " << false;
         int r =
             bkcl_broadcast(comm,
                            input.data(),
@@ -671,7 +741,12 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::AllGather(
           const phi::DenseTensor& input,
           BKCLContext_t comm,
           const XPUStream& stream) {
-        VLOG(3) << "bkcl_all_gather";
+        VLOG(3) << "calling bkcl_all_gather"
+                << ", rank_id: " << platform::GetBKCLRankID(comm)
+                << ", dev_id: " << platform::GetBKCLDevID(comm)
+                << ", nranks: " << platform::GetBKCLNRanks(comm)
+                << ", numel: " << input.numel() << ", dtype: " << input.type()
+                << ", sync_op: " << true << ", use_calc_stream: " << false;
         int r =
             bkcl_all_gather(comm,
                             input.data(),
@@ -712,7 +787,12 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::AllGather(
           const phi::DenseTensor& input,
           BKCLContext_t comm,
           const XPUStream& stream) {
-        VLOG(3) << "bkcl_all_gather";
+        VLOG(3) << "calling bkcl_all_gather"
+                << ", rank_id: " << platform::GetBKCLRankID(comm)
+                << ", dev_id: " << platform::GetBKCLDevID(comm)
+                << ", nranks: " << platform::GetBKCLNRanks(comm)
+                << ", numel: " << input.numel() << ", dtype: " << input.type()
+                << ", sync_op: " << sync_op << ", use_calc_stream: " << false;
         int r =
             bkcl_all_gather(comm,
                             input.data(),
diff --git a/paddle/fluid/operators/fused/fused_gemm_epilogue_op_xpu.cc b/paddle/fluid/operators/fused/fused_gemm_epilogue_op_xpu.cc
index 6594df2f5164f..58d81ebf8be06 100644
--- a/paddle/fluid/operators/fused/fused_gemm_epilogue_op_xpu.cc
+++ b/paddle/fluid/operators/fused/fused_gemm_epilogue_op_xpu.cc
@@ -66,7 +66,6 @@ class FusedGemmEpilogueXPUKernel : public framework::OpKernel<T> {
     phi::XpuFcInfo fc_info;
 
     phi::GetFCInfo(x_mat_dims, y->dims(), trans_x, trans_y, &fc_info);
-    VLOG(0) << "FusedGemmEpilogueXPUKernel 000";
     xpu::Context* xpu_ctx = dev_ctx.x_context();
 
     const XPUType* x_ptr = reinterpret_cast<const XPUType*>(x->data<T>());
diff --git a/paddle/fluid/platform/device/xpu/bkcl_helper.h b/paddle/fluid/platform/device/xpu/bkcl_helper.h
index 1f44bc0a8c98b..dcb17bfd2b932 100644
--- a/paddle/fluid/platform/device/xpu/bkcl_helper.h
+++ b/paddle/fluid/platform/device/xpu/bkcl_helper.h
@@ -62,6 +62,18 @@ inline BKCLDataType ToBKCLDataType(framework::proto::VarType::Type type) {
   }
 }
 
+inline int GetBKCLRankID(BKCLContext_t comm) {
+  return reinterpret_cast<int *>(comm)[0];
+}
+
+inline int GetBKCLDevID(BKCLContext_t comm) {
+  return reinterpret_cast<int *>(comm)[1];
+}
+
+inline int GetBKCLNRanks(BKCLContext_t comm) {
+  return reinterpret_cast<int *>(comm)[2];
+}
+
 class BKCLGroupGuard {
  public:
   static std::mutex &BKCLMutex() {

From 44aebd456e11b33c08ac99433251c5b6912cd53c Mon Sep 17 00:00:00 2001
From: houj04 <35131887+houj04@users.noreply.github.com>
Date: Thu, 11 May 2023 13:23:08 +0800
Subject: [PATCH 268/405] [XPU] update dependency for xccl. (#53697)

* [XPU] update dependency for xccl.

* remove unnecessary codes.
---
 cmake/external/xpu.cmake          |  27 ++++---
 tools/xpu/check_xpu_dependence.sh | 120 ++++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+), 12 deletions(-)
 create mode 100644 tools/xpu/check_xpu_dependence.sh

diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake
index 6202d4edf0496..c2be9fab7ee78 100644
--- a/cmake/external/xpu.cmake
+++ b/cmake/external/xpu.cmake
@@ -9,7 +9,7 @@ set(XPU_RT_LIB_NAME "libxpurt.so")
 set(XPU_XFT_LIB_NAME "libxft.so")
 
 set(XPU_BASE_DATE "20230427")
-set(XPU_XCCL_BASE_VERSION "1.0.13")
+set(XPU_XCCL_BASE_VERSION "1.0.49.2")
 set(XPU_XFT_BASE_VERSION "latest")
 
 if(NOT DEFINED XPU_BASE_URL)
@@ -30,35 +30,41 @@ if(NOT XPU_XFT_BASE_URL)
   )
 endif()
 
+if(WITH_XCCL_RDMA)
+  set(XPU_XCCL_PREFIX "xccl_rdma")
+else()
+  set(XPU_XCCL_PREFIX "xccl_socket")
+endif()
+
 if(WITH_AARCH64)
   set(XPU_XRE_DIR_NAME "xre-kylin_aarch64")
   set(XPU_XDNN_DIR_NAME "xdnn-kylin_aarch64")
-  set(XPU_XCCL_DIR_NAME "xccl-kylin_aarch64")
+  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}-kylin_aarch64")
   set(XPU_XFT_DIR_NAME "") # TODO: xft has no kylin output at now.
 elseif(WITH_SUNWAY)
   set(XPU_XRE_DIR_NAME "xre-deepin_sw6_64")
   set(XPU_XDNN_DIR_NAME "xdnn-deepin_sw6_64")
-  set(XPU_XCCL_DIR_NAME "xccl-deepin_sw6_64")
+  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}-deepin_sw6_64")
   set(XPU_XFT_DIR_NAME "") # TODO: xft has no deepin output at now.
 elseif(WITH_BDCENTOS)
   set(XPU_XRE_DIR_NAME "xre-bdcentos_x86_64")
   set(XPU_XDNN_DIR_NAME "xdnn-bdcentos_x86_64")
-  set(XPU_XCCL_DIR_NAME "xccl-bdcentos_x86_64")
+  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}-bdcentos_x86_64")
   set(XPU_XFT_DIR_NAME "xft_bdcentos6u3_x86_64_gcc82")
 elseif(WITH_UBUNTU)
   set(XPU_XRE_DIR_NAME "xre-ubuntu_x86_64")
   set(XPU_XDNN_DIR_NAME "xdnn-ubuntu_x86_64")
-  set(XPU_XCCL_DIR_NAME "xccl-ubuntu_x86_64")
+  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}-ubuntu_x86_64")
   set(XPU_XFT_DIR_NAME "xft_ubuntu1604_x86_64")
 elseif(WITH_CENTOS)
   set(XPU_XRE_DIR_NAME "xre-centos7_x86_64")
   set(XPU_XDNN_DIR_NAME "xdnn-centos7_x86_64")
-  set(XPU_XCCL_DIR_NAME "xccl-bdcentos_x86_64")
+  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}-bdcentos_x86_64")
   set(XPU_XFT_DIR_NAME "xft_bdcentos6u3_x86_64_gcc82")
 else()
   set(XPU_XRE_DIR_NAME "xre-ubuntu_x86_64")
   set(XPU_XDNN_DIR_NAME "xdnn-ubuntu_x86_64")
-  set(XPU_XCCL_DIR_NAME "xccl-ubuntu_x86_64")
+  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}-ubuntu_x86_64")
   set(XPU_XFT_DIR_NAME "xft_ubuntu1604_x86_64")
 endif()
 
@@ -75,9 +81,6 @@ set(XPU_XFT_URL "${XPU_XFT_BASE_URL}/${XPU_XFT_DIR_NAME}.tar.gz")
 set(XPU_PACK_DEPENCE_URL
     "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/pack_paddle_depence.sh"
     CACHE STRING "" FORCE)
-set(XPU_CHECK_DEPENCE_URL
-    "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/check_xpu_dependence.sh"
-    CACHE STRING "" FORCE)
 set(XPU_XFT_GET_DEPENCE_URL
     "https://baidu-kunlun-public.su.bcebos.com/paddle_depence/get_xft_dependence.sh"
     CACHE STRING "" FORCE)
@@ -115,8 +118,8 @@ ExternalProject_Add(
   PREFIX ${SNAPPY_PREFIX_DIR}
   DOWNLOAD_DIR ${XPU_DOWNLOAD_DIR}
   DOWNLOAD_COMMAND
-    wget ${XPU_CHECK_DEPENCE_URL} && bash check_xpu_dependence.sh
-    ${XPU_BASE_URL} ${XPU_XCCL_BASE_URL} && wget ${XPU_PACK_DEPENCE_URL} && bash
+    bash ${CMAKE_SOURCE_DIR}/tools/xpu/check_xpu_dependence.sh ${XPU_BASE_URL}
+    ${XPU_XCCL_BASE_URL} && wget ${XPU_PACK_DEPENCE_URL} && bash
     pack_paddle_depence.sh ${XPU_XRE_URL} ${XPU_XRE_DIR_NAME} ${XPU_XDNN_URL}
     ${XPU_XDNN_DIR_NAME} ${XPU_XCCL_URL} ${XPU_XCCL_DIR_NAME} && wget
     ${XPU_XFT_GET_DEPENCE_URL} && bash get_xft_dependence.sh ${XPU_XFT_URL}
diff --git a/tools/xpu/check_xpu_dependence.sh b/tools/xpu/check_xpu_dependence.sh
new file mode 100644
index 0000000000000..abfea14330819
--- /dev/null
+++ b/tools/xpu/check_xpu_dependence.sh
@@ -0,0 +1,120 @@
+#!/bin/bash
+
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -u
+
+if [[ $# -ne 2 ]]; then
+    echo "usage: ./check_xpu_dependence.sh XPU_BASE_URL XPU_XCCL_BASE_URL"
+    exit 1
+fi
+
+xpu_base_url=$1
+xccl_base_url=$2
+
+echo "xpu_base_url: $xpu_base_url"
+echo "xccl_base_url: $xccl_base_url"
+
+function check_files() {
+    local url="$1"
+    local local_dir="$2"
+    echo "local dir: $local_dir"
+    local local_file_name="${local_dir}.tar.gz"
+    echo "local file name: $local_file_name"
+
+    shift
+    shift
+    local files=("$@")
+
+    # start to download
+    echo "downloading: $url"
+    rm -f ./$local_file_name
+    wget -q $url -O ${local_file_name}
+    if [[ $? -ne 0 ]]; then
+        echo "downloading failed: $url"
+        return 1
+    else
+        echo "downloading ok: $url"
+    fi
+
+    # remove local dir and de-compress
+    rm -rf ./$local_dir
+    tar xf $local_file_name
+    if [[ $? -ne 0 ]]; then
+        echo "de-compress failed: $local_file_name"
+        return 1
+    fi
+
+    for i in "${files[@]}";
+    do
+        echo "checking $local_dir/$i"
+        if [[ ! -f $local_dir/$i ]]; then
+            echo "checking failed: $local_dir/$i"
+            return 1
+        else
+            echo "checking ok: $local_dir/$i"
+        fi
+    done
+
+    # clean
+    rm -f ./$local_file_name
+    rm -rf ./$local_dir
+}
+
+# XRE
+xre_tar_file_names=("xre-kylin_aarch64" "xre-bdcentos_x86_64" "xre-ubuntu_x86_64" "xre-centos7_x86_64")
+xre_inner_file_names=("include/xpu/runtime.h" "so/libxpurt.so")
+for name in ${xre_tar_file_names[@]}; do
+    url="${xpu_base_url}/${name}.tar.gz"
+    check_files $url $name "${xre_inner_file_names[@]}"
+    if [[ $? -ne 0 ]]; then
+        echo "XRE check failed, name: $name"
+        exit 1
+    else
+        echo "XRE check ok, name: $name"
+    fi
+done
+
+# XDNN
+xdnn_tar_file_names=("xdnn-kylin_aarch64" "xdnn-bdcentos_x86_64" "xdnn-ubuntu_x86_64" "xdnn-centos7_x86_64")
+xdnn_inner_file_names=("include/xpu/xdnn.h" "so/libxpuapi.so")
+for name in ${xdnn_tar_file_names[@]}; do
+    url="${xpu_base_url}/${name}.tar.gz"
+    check_files $url $name "${xdnn_inner_file_names[@]}"
+    if [[ $? -ne 0 ]]; then
+        echo "XDNN check failed, name: $name"
+        exit 1
+    else
+        echo "XDNN check ok, name: $name"
+    fi
+done
+
+# XCCL
+xccl_tar_file_names=("xccl_rdma-bdcentos_x86_64" "xccl_rdma-ubuntu_x86_64" "xccl_socket-bdcentos_x86_64" "xccl_socket-deepin_sw6_64" "xccl_socket-kylin_aarch64" "xccl_socket-ubuntu_x86_64")
+xccl_inner_file_names=("include/bkcl.h" "so/libbkcl.so")
+for name in ${xccl_tar_file_names[@]}; do
+    url="${xccl_base_url}/${name}.tar.gz"
+    check_files $url $name "${xccl_inner_file_names[@]}"
+    if [[ $? -ne 0 ]]; then
+        echo "XCCL check failed, name: $name"
+        exit 1
+    else
+        echo "XCCL check ok, name: $name"
+    fi
+done
+
+echo "ALL CHECKS PASSED"
+
+exit 0

From 49de9de53ff3f3a78f9bfc2072cc38876b941025 Mon Sep 17 00:00:00 2001
From: kangguangli <kangguangli@hotmail.com>
Date: Thu, 11 May 2023 14:28:40 +0800
Subject: [PATCH 269/405] [Doc] remove execution_strategy doc (#53668)

* remove execution_strategy docstring

* remove doc of num_iteration_per_run; test=document_fix
---
 paddle/fluid/pybind/parallel_executor.cc | 109 ++---------------------
 1 file changed, 6 insertions(+), 103 deletions(-)

diff --git a/paddle/fluid/pybind/parallel_executor.cc b/paddle/fluid/pybind/parallel_executor.cc
index 553701906f59e..3f20a2498f840 100644
--- a/paddle/fluid/pybind/parallel_executor.cc
+++ b/paddle/fluid/pybind/parallel_executor.cc
@@ -188,39 +188,7 @@ using namespace paddle::framework;                // NOLINT
 void BindParallelExecutor(pybind11::module &m) {  // NOLINT
   // -- python binds for parallel executor.
   py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
-  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
-    ExecutionStrategy allows the user to more preciously control how to run
-    the program in ParallelExecutor by setting the property.
-
-    Returns:
-        ExecutionStrategy: An ExecutionStrategy object.
-
-    Examples:
-        .. code-block:: python
-
-          import paddle
-          import paddle.static as static
-          import paddle.nn.functional as F
-
-          paddle.enable_static()
-
-          x = static.data(name='x', shape=[None, 13], dtype='float32')
-          y = static.data(name='y', shape=[None, 1], dtype='float32')
-          y_predict = static.nn.fc(input=x, size=1, act=None)
-
-          cost = F.square_error_cost(input=y_predict, label=y)
-          avg_loss = paddle.mean(cost)
-
-          sgd_optimizer = paddle.optimizer.SGD(learning_rate=0.001)
-          sgd_optimizer.minimize(avg_loss)
-
-          exec_strategy = static.ExecutionStrategy()
-          exec_strategy.num_threads = 4
-
-          train_exe = static.ParallelExecutor(use_cuda=False,
-                                              loss_name=avg_loss.name,
-                                              exec_strategy=exec_strategy)
-        )DOC");
+  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy");
 
   py::enum_<paddle::platform::DeviceType>(m, "DeviceType", py::arithmetic())
       .value("CPU", paddle::platform::DeviceType::CPU)
@@ -233,29 +201,7 @@ void BindParallelExecutor(pybind11::module &m) {  // NOLINT
           [](const ExecutionStrategy &self) { return self.num_threads_; },
           [](ExecutionStrategy &self, size_t num_threads) {
             self.num_threads_ = num_threads;
-          },
-          R"DOC(
-            The type is INT, num_threads represents the size of thread pool that
-            used to run the operators of the current program in ParallelExecutor.
-            If :math:`num\_threads=1`, all the operators will execute one by one,
-            but the order maybe difference between iterations.
-            If it is not set, it will be set in ParallelExecutor according to the
-            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
-            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
-            if it is not set, ParallelExecutor will get the cpu count by calling
-            `multiprocessing.cpu_count()`. Default 0.
-
-            Examples:
-                .. code-block:: python
-
-                    import paddle
-                    import paddle.static as static
-
-                    paddle.enable_static()
-
-                    exec_strategy = static.ExecutionStrategy()
-                    exec_strategy.num_threads = 4
-            )DOC")
+          })
       .def_property(
           "_use_device",
           [](const ExecutionStrategy &self) { return self.use_device_; },
@@ -268,11 +214,7 @@ void BindParallelExecutor(pybind11::module &m) {  // NOLINT
           [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
           [](ExecutionStrategy &self, bool allow_op_delay) {
             self.allow_op_delay_ = allow_op_delay;
-          },
-          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
-                communication operators to run, it may make the execution faster.
-                Note that this option is invalid now, and it will be removed in
-                next version. Default False.)DOC")
+          })
       .def_property(
           "num_iteration_per_drop_scope",
           [](const ExecutionStrategy &self) {
@@ -280,30 +222,7 @@ void BindParallelExecutor(pybind11::module &m) {  // NOLINT
           },
           [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
             self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
-          },
-          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
-                many iterations to clean up the temp variables which
-                is generated during execution. It may make the execution faster,
-                because the temp variable's shape maybe the same between two iterations.
-                Default 100.
-
-                .. note::
-                    1. If you fetch data when calling the 'run', the ParallelExecutor
-                    will clean up the temp variables at the end of the current iteration.
-                    2. In some NLP model, it may cause the GPU memory is insufficient,
-                    in this case, you should reduce `num_iteration_per_drop_scope`.
-
-                Examples:
-                    .. code-block:: python
-
-                        import paddle
-                        import paddle.static as static
-
-                        paddle.enable_static()
-
-                        exec_strategy = static.ExecutionStrategy()
-                        exec_strategy.num_iteration_per_drop_scope = 10
-              )DOC")
+          })
       .def_property(
           "num_iteration_per_run",
           [](const ExecutionStrategy &self) {
@@ -311,29 +230,13 @@ void BindParallelExecutor(pybind11::module &m) {  // NOLINT
           },
           [](ExecutionStrategy &self, size_t num_iteration_per_run) {
             self.num_iteration_per_run_ = num_iteration_per_run;
-          },
-          R"DOC(This config that how many iteration the executor will run when
-                user call exe.run() in python。Default: 1.
-
-                Examples:
-                    .. code-block:: python
-
-                        import paddle
-                        import paddle.static as static
-
-                        paddle.enable_static()
-
-                        exec_strategy = static.ExecutionStrategy()
-                        exec_strategy.num_iteration_per_run = 10
-              )DOC")
+          })
       .def_property(
           "use_thread_barrier",
           [](const ExecutionStrategy &self) { return self.thread_barrier_; },
           [](ExecutionStrategy &self, bool use_thread_barrier) {
             self.thread_barrier_ = use_thread_barrier;
-          },
-          R"DOC(This config that the this is distributed training with parameter server
-              )DOC")
+          })
       .def_property(
           "_dry_run",
           [](const ExecutionStrategy &self) { return self.dry_run_; },

From 6ec8d8567b8ea78a6456052c238ac1a46e1aac54 Mon Sep 17 00:00:00 2001
From: JYChen <zoooo0820@qq.com>
Date: Thu, 11 May 2023 14:35:25 +0800
Subject: [PATCH 270/405] up index warning level (#53691)

* up warning level

* numpy still vlog-0
---
 paddle/fluid/pybind/eager_method.cc | 2 +-
 paddle/fluid/pybind/imperative.cc   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc
index 2caf978db6f23..7f32c14c493af 100644
--- a/paddle/fluid/pybind/eager_method.cc
+++ b/paddle/fluid/pybind/eager_method.cc
@@ -931,7 +931,7 @@ static PyObject* tensor__getitem_index_not_tensor(TensorObject* self,
     // with FLAGS_set_to_1d=True. In this case, one `None` should be pop out,
     // otherwise the output shape will be not correct.
     if (static_cast<int>(decrease_axis.size()) == tensor->dims().size()) {
-      VLOG(0)
+      VLOG(1)
           << "Warning: In Tensor '__getitem__', if the number of scalar "
              "elements "
              "in the index is equal to the rank of the Tensor, the output "
diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc
index aa1ee4724925e..65eac1e3dc6fd 100644
--- a/paddle/fluid/pybind/imperative.cc
+++ b/paddle/fluid/pybind/imperative.cc
@@ -1060,7 +1060,7 @@ void BindImperative(py::module *m_ptr) {
                // not correct.
                if (static_cast<int>(decrease_axis.size()) ==
                    tensor->dims().size()) {
-                 VLOG(0) << "Warning: In Tensor '__getitem__', if the number "
+                 VLOG(1) << "Warning: In Tensor '__getitem__', if the number "
                             "of scalar "
                             "elements "
                             "in the index is equal to the rank of the Tensor, "

From 08b6f5d625640ab6f3ebdffa75358cac2aa2bbea Mon Sep 17 00:00:00 2001
From: SaltFish11 <108038822+SaltFish11@users.noreply.github.com>
Date: Thu, 11 May 2023 14:36:34 +0800
Subject: [PATCH 271/405] [XPU] add depthwise_conv2d_transpose (#53680)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add_depthwise_conv2d_transpose

* Update test_depthwise_conv2d_transpose_op_xpu.py

删除print语句
---
 paddle/phi/backends/xpu/xpu2_op_list.cc       |   3 +
 .../kernels/xpu/conv_transpose_grad_kernel.cc |  35 ++
 .../phi/kernels/xpu/conv_transpose_kernel.cc  |  31 ++
 .../test_depthwise_conv2d_transpose_op_xpu.py | 298 ++++++++++++++++++
 4 files changed, 367 insertions(+)
 create mode 100644 test/xpu/test_depthwise_conv2d_transpose_op_xpu.py

diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc
index 4c3688b0badfa..ed19835b6469d 100644
--- a/paddle/phi/backends/xpu/xpu2_op_list.cc
+++ b/paddle/phi/backends/xpu/xpu2_op_list.cc
@@ -185,6 +185,9 @@ XPUOpMap& get_kl2_ops() {
       {"deformable_conv_v1", XPUKernelSet({phi::DataType::FLOAT32})},
       {"depthwise_conv2d_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"depthwise_conv2d", XPUKernelSet({phi::DataType::FLOAT32})},
+      {"depthwise_conv2d_transpose_grad",
+       XPUKernelSet({phi::DataType::FLOAT32})},
+      {"depthwise_conv2d_transpose", XPUKernelSet({phi::DataType::FLOAT32})},
       {"diag_v2",
        XPUKernelSet({phi::DataType::FLOAT32,
                      phi::DataType::FLOAT16,
diff --git a/paddle/phi/kernels/xpu/conv_transpose_grad_kernel.cc b/paddle/phi/kernels/xpu/conv_transpose_grad_kernel.cc
index 9db36ace02e4d..4b61b61b5e254 100644
--- a/paddle/phi/kernels/xpu/conv_transpose_grad_kernel.cc
+++ b/paddle/phi/kernels/xpu/conv_transpose_grad_kernel.cc
@@ -97,6 +97,36 @@ void Conv2dTransposeGradKernel(const Context& ctx,
   PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d_transpose_grad");
 }
 
+template <typename T, typename Context>
+void DepthwiseConv2dTransposeGradKernel(const Context& ctx,
+                                        const DenseTensor& x,
+                                        const DenseTensor& filter,
+                                        const DenseTensor& dout,
+                                        const std::vector<int>& strides,
+                                        const std::vector<int>& paddings,
+                                        const std::vector<int>& output_padding,
+                                        const IntArray& output_size,
+                                        const std::string& padding_algorithm,
+                                        int groups,
+                                        const std::vector<int>& dilations,
+                                        const std::string& data_format,
+                                        DenseTensor* dx,
+                                        DenseTensor* dfilter) {
+  Conv2dTransposeGradKernel<T, Context>(ctx,
+                                        x,
+                                        filter,
+                                        dout,
+                                        strides,
+                                        paddings,
+                                        output_padding,
+                                        output_size,
+                                        padding_algorithm,
+                                        groups,
+                                        dilations,
+                                        data_format,
+                                        dx,
+                                        dfilter);
+}
 }  // namespace phi
 
 PD_REGISTER_KERNEL(conv2d_transpose_grad,
@@ -104,3 +134,8 @@ PD_REGISTER_KERNEL(conv2d_transpose_grad,
                    ALL_LAYOUT,
                    phi::Conv2dTransposeGradKernel,
                    float) {}
+PD_REGISTER_KERNEL(depthwise_conv2d_transpose_grad,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::DepthwiseConv2dTransposeGradKernel,
+                   float) {}
diff --git a/paddle/phi/kernels/xpu/conv_transpose_kernel.cc b/paddle/phi/kernels/xpu/conv_transpose_kernel.cc
index 1b3c31f665c7c..f658f06a9908d 100644
--- a/paddle/phi/kernels/xpu/conv_transpose_kernel.cc
+++ b/paddle/phi/kernels/xpu/conv_transpose_kernel.cc
@@ -145,8 +145,39 @@ void Conv2dTransposeKernel(const Context& ctx,
     PADDLE_ENFORCE_XDNN_SUCCESS(r, "conv2d_transpose_v2");
   }
 }
+template <typename T, typename Context>
+void DepthwiseConv2dTransposeKernel(const Context& ctx,
+                                    const DenseTensor& x,
+                                    const DenseTensor& filter,
+                                    const std::vector<int>& strides,
+                                    const std::vector<int>& paddings,
+                                    const std::vector<int>& output_padding,
+                                    const IntArray& output_size,
+                                    const std::string& padding_algorithm,
+                                    int groups,
+                                    const std::vector<int>& dilations,
+                                    const std::string& data_format,
+                                    DenseTensor* out) {
+  Conv2dTransposeKernel<T, Context>(ctx,
+                                    x,
+                                    filter,
+                                    strides,
+                                    paddings,
+                                    output_padding,
+                                    output_size,
+                                    padding_algorithm,
+                                    groups,
+                                    dilations,
+                                    data_format,
+                                    out);
+}
 
 }  // namespace phi
+PD_REGISTER_KERNEL(depthwise_conv2d_transpose,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::DepthwiseConv2dTransposeKernel,
+                   float) {}
 
 PD_REGISTER_KERNEL(conv2d_transpose,
                    XPU,
diff --git a/test/xpu/test_depthwise_conv2d_transpose_op_xpu.py b/test/xpu/test_depthwise_conv2d_transpose_op_xpu.py
new file mode 100644
index 0000000000000..e4b290a2e66e1
--- /dev/null
+++ b/test/xpu/test_depthwise_conv2d_transpose_op_xpu.py
@@ -0,0 +1,298 @@
+#   Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from get_test_cover_info import (
+    XPUOpTestWrapper,
+    create_test_class,
+    get_xpu_op_support_types,
+)
+from op_test_xpu import XPUOpTest
+
+import paddle
+
+paddle.enable_static()
+
+
+def depthwiseconv2dtranspose_forward_naive(input_, filter_, attrs):
+    padding_algorithm = attrs['padding_algorithm']
+    if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]:
+        raise ValueError(
+            "Unknown Attr(padding_algorithm): '%s'. "
+            "It can only be 'SAME' or 'VALID'." % str(padding_algorithm)
+        )
+
+    if attrs['data_format'] == 'NHWC':
+        input_ = np.transpose(input_, [0, 3, 1, 2])
+    in_n, in_c, in_h, in_w = input_.shape
+    f_c, f_out_c, f_h, f_w = filter_.shape
+    groups = attrs['groups']
+    assert in_c == f_c
+    out_c = f_out_c * groups
+    sub_in_c = in_c // groups
+
+    stride, pad, dilations = (
+        attrs['strides'],
+        attrs['paddings'],
+        attrs['dilations'],
+    )
+
+    # update pad and dilation
+    def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride):
+        padding = []
+        for input_size, filter_size, stride_size in zip(
+            input_shape, kernel_size, kernel_stride
+        ):
+            out_size = int((input_size + stride_size - 1) / stride_size)
+            pad_sum = np.max(
+                ((out_size - 1) * stride_size + filter_size - input_size, 0)
+            )
+            pad_0 = int(pad_sum / 2)
+            pad_1 = int(pad_sum - pad_0)
+            padding.append(pad_0)
+            padding.append(pad_1)
+        return padding
+
+    ksize = filter_.shape[2:4]
+    if padding_algorithm == "VALID":
+        pad = [0, 0, 0, 0]
+    elif padding_algorithm == "SAME":
+        dilations = [1, 1]
+        input_data_shape = input_.shape[2:4]
+        pad = _get_padding_with_SAME(input_data_shape, ksize, stride)
+
+    pad_h_0, pad_h_1 = pad[0], pad[0]
+    pad_w_0, pad_w_1 = pad[1], pad[1]
+    if len(pad) == 4:
+        pad_h_0, pad_h_1 = pad[0], pad[1]
+        pad_w_0, pad_w_1 = pad[2], pad[3]
+
+    d_bolck_h = dilations[0] * (f_h - 1) + 1
+    d_bolck_w = dilations[1] * (f_w - 1) + 1
+    out_h = (in_h - 1) * stride[0] + d_bolck_h
+    out_w = (in_w - 1) * stride[1] + d_bolck_w
+    if 'output_size' in attrs:
+        output_size = attrs['output_size']
+        out_h = output_size[0] + pad_h_0 + pad_h_1
+        out_w = output_size[1] + pad_w_0 + pad_w_1
+    out_pad_h = 0
+    out_pad_w = 0
+    if 'output_padding' in attrs:
+        out_pad_h = attrs['output_padding'][0]
+        out_pad_w = attrs['output_padding'][1]
+    out = np.zeros(
+        (in_n, out_c, out_h + out_pad_h, out_w + out_pad_w), dtype=input_.dtype
+    )
+
+    for n in range(in_n):
+        for i in range(in_h):
+            for j in range(in_w):
+                for g in range(groups):
+                    input_masked = input_[
+                        n, g * sub_in_c : (g + 1) * sub_in_c, i, j
+                    ]  # (c)
+                    input_masked = np.reshape(input_masked, (sub_in_c, 1, 1))
+                    input_masked = np.tile(input_masked, (1, f_h, f_w))
+
+                    for k in range(f_out_c):
+                        tmp_out = np.sum(
+                            input_masked
+                            * filter_[
+                                g * sub_in_c : (g + 1) * sub_in_c, k, :, :
+                            ],
+                            axis=0,
+                        )
+                        i1, i2 = i * stride[0], i * stride[0] + d_bolck_h
+                        j1, j2 = j * stride[1], j * stride[1] + d_bolck_w
+                        out[
+                            n,
+                            g * f_out_c + k,
+                            i1 : i2 : dilations[0],
+                            j1 : j2 : dilations[1],
+                        ] += tmp_out
+
+    out = out[
+        :,
+        :,
+        pad_h_0 : out_h - pad_h_1 + out_pad_h,
+        pad_w_0 : out_w - pad_w_1 + out_pad_w,
+    ]
+    if attrs['data_format'] == 'NHWC':
+        out = np.transpose(out, [0, 2, 3, 1])
+    return out
+
+
+class XPUTestDepthwiseConv2DTransposeOp(XPUOpTestWrapper):
+    def __init__(self):
+        self.op_name = 'depthwise_conv2d_transpose'
+        self.use_dynamic_create_class = False
+
+    class TestDepthwiseConv2DTransposeOp(XPUOpTest):
+        def setUp(self):
+            # init as conv transpose
+            self.need_check_grad = True
+            self.is_test = False
+            self.use_cudnn = False
+            self.use_mkldnn = False
+            self.output_size = None
+            self.output_padding = []
+            self.data_format = "NCHW"
+            self.pad = [0, 0]
+            self.padding_algorithm = "EXPLICIT"
+            self.init_op_type()
+            self.init_test_case()
+            self.__class__.op_type = "depthwise_conv2d_transpose"
+
+            input_ = np.random.random(self.input_size).astype(self.dtype)
+            filter_ = np.random.random(self.filter_size).astype(self.dtype)
+
+            self.inputs = {'Input': input_, 'Filter': filter_}
+            self.attrs = {
+                'strides': self.stride,
+                'paddings': self.pad,
+                'padding_algorithm': self.padding_algorithm,
+                'groups': self.groups,
+                'dilations': self.dilations,
+                'use_cudnn': self.use_cudnn,
+                'is_test': self.is_test,
+                'use_mkldnn': self.use_mkldnn,
+                'data_format': self.data_format,
+            }
+            if self.output_size is not None:
+                self.attrs['output_size'] = self.output_size
+
+            if len(self.output_padding) > 0:
+                self.attrs['output_padding'] = self.output_padding
+
+            output = depthwiseconv2dtranspose_forward_naive(
+                input_, filter_, self.attrs
+            ).astype(self.dtype)
+
+            self.outputs = {'Output': output}
+
+        def test_check_output(self):
+            self.check_output_with_place(self.place)
+
+        def test_check_grad_no_input(self):
+            if self.need_check_grad:
+                self.check_grad_with_place(
+                    self.place, ['Filter'], 'Output', no_grad_set={'Input'}
+                )
+
+        def test_check_grad_no_filter(self):
+            if self.need_check_grad:
+                self.check_grad_with_place(
+                    self.place, ['Input'], 'Output', no_grad_set={'Filter'}
+                )
+
+        def test_check_grad(self):
+            if self.need_check_grad:
+                self.check_grad_with_place(
+                    self.place, {'Input', 'Filter'}, 'Output'
+                )
+
+        def init_test_case(self):
+            self.pad = [0, 0]
+            self.stride = [1, 1]
+            self.dilations = [1, 1]
+            self.groups = 1
+            self.input_size = [2, 3, 5, 5]  # NCHW
+            f_c = self.input_size[1]
+            self.filter_size = [f_c, 6, 3, 3]
+
+        def init_op_type(self):
+            self.dtype = self.in_type
+            self.place = paddle.XPUPlace(0)
+            self.op_type = "depthwise_conv2d_transpose"
+
+    class TestWithSymmetricPad(TestDepthwiseConv2DTransposeOp):
+        def init_test_case(self):
+            self.pad = [1, 1]
+            self.stride = [1, 1]
+            self.dilations = [1, 1]
+            self.groups = 1
+            self.input_size = [2, 3, 5, 5]  # NCHW
+            f_c = self.input_size[1]
+            self.filter_size = [f_c, 6, 3, 3]
+
+    class TestWithAsymmetricPad(TestDepthwiseConv2DTransposeOp):
+        def init_test_case(self):
+            self.pad = [1, 0, 1, 2]
+            self.stride = [1, 1]
+            self.dilations = [1, 1]
+            self.groups = 1
+            self.input_size = [2, 3, 5, 5]  # NCHW
+            f_c = self.input_size[1]
+            self.filter_size = [f_c, 6, 3, 3]
+
+    class TestWithSAMEPad(TestDepthwiseConv2DTransposeOp):
+        def init_test_case(self):
+            self.stride = [2, 1]
+            self.dilations = [1, 2]
+            self.groups = 1
+            self.input_size = [2, 3, 6, 5]  # NCHW
+            f_c = self.input_size[1]
+            self.filter_size = [f_c, 6, 4, 3]
+            self.padding_algorithm = 'SAME'
+
+    class TestWithVALIDPad(TestDepthwiseConv2DTransposeOp):
+        def init_test_case(self):
+            self.stride = [1, 1]
+            self.dilations = [1, 1]
+            self.groups = 1
+            self.input_size = [2, 3, 5, 5]  # NCHW
+            f_c = self.input_size[1]
+            self.filter_size = [f_c, 6, 3, 3]
+            self.padding_algorithm = 'VALID'
+
+    class TestWithGroups(TestDepthwiseConv2DTransposeOp):
+        def init_test_case(self):
+            self.pad = [1, 1]
+            self.stride = [1, 1]
+            self.dilations = [1, 1]
+            self.groups = 2
+            self.input_size = [2, 4, 5, 5]  # NCHW
+            f_c = self.input_size[1]
+            self.filter_size = [f_c, 3, 3, 3]
+
+    class TestWithStride(TestDepthwiseConv2DTransposeOp):
+        def init_test_case(self):
+            self.pad = [1, 1]
+            self.stride = [2, 2]
+            self.dilations = [1, 1]
+            self.groups = 1
+            self.input_size = [2, 3, 5, 5]  # NCHW
+            f_c = self.input_size[1]
+            self.filter_size = [f_c, 6, 3, 3]
+
+    class TestWithEvenUpsample(TestDepthwiseConv2DTransposeOp):
+        def init_test_case(self):
+            self.pad = [2, 2]
+            self.stride = [2, 2]
+            self.groups = 1
+            self.dilations = [1, 1]
+            self.output_size = [14, 14]
+            self.input_size = [2, 3, 7, 7]  # NCHW
+            f_c = self.input_size[1]
+            self.filter_size = [f_c, 6, 5, 5]
+
+
+support_types = get_xpu_op_support_types('depthwise_conv2d_transpose')
+for stype in support_types:
+    create_test_class(globals(), XPUTestDepthwiseConv2DTransposeOp, stype)
+
+if __name__ == '__main__':
+    unittest.main()

From aebff6d7999518d50984f9a482dec700cbd15798 Mon Sep 17 00:00:00 2001
From: xiaoxiaohehe001 <49090790+xiaoxiaohehe001@users.noreply.github.com>
Date: Thu, 11 May 2023 14:41:00 +0800
Subject: [PATCH 272/405] [Paddle-Inference] Support trt 0dims of expand_as_v2
 and mish. (#53627)

* support_expand_mish
---
 paddle/fluid/inference/tensorrt/op_teller.cc      | 15 +++++++++++++--
 .../ir/inference/test_trt_convert_expand_as_v2.py | 13 +++++++++++--
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index 84712e9aa6294..fb16ff235467e 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -1890,8 +1890,10 @@ struct SimpleOpTypeSetTeller : public Teller {
       auto x_var_name = desc.Input("X")[0];
       auto* x_var_desc = block->FindVar(x_var_name);
       const auto x_shape = x_var_desc->GetShape();
-      if (x_shape.size() == 1) {
-        VLOG(3) << "mish op does not support input's dim is 1 in tensorrt.";
+      if ((!with_dynamic_shape && x_shape.size() == 1) || x_shape.size() == 0) {
+        VLOG(3) << op_type
+                << "mish op does not support input's dim is 1 in tensorrt "
+                   "static shape mode or 0.";
         return false;
       }
     }
@@ -2644,6 +2646,15 @@ struct SimpleOpTypeSetTeller : public Teller {
                    "the pass.";
         return false;
       }
+
+#if IS_TRT_VERSION_LT(8000)
+      auto x_var_name = desc.Input("X")[0];
+      auto* x_var_desc = block->FindVar(x_var_name);
+      const auto x_shape = x_var_desc->GetShape();
+      if (x_shape.size() == 0) {
+        return false;  // not supported 0 dim.
+      }
+#endif
     }
 
     if (op_type == "grid_sampler") {
diff --git a/test/ir/inference/test_trt_convert_expand_as_v2.py b/test/ir/inference/test_trt_convert_expand_as_v2.py
index 46b3a2232e471..be5458cac07dc 100644
--- a/test/ir/inference/test_trt_convert_expand_as_v2.py
+++ b/test/ir/inference/test_trt_convert_expand_as_v2.py
@@ -49,8 +49,11 @@ def generate_input1(attrs: List[Dict[str, Any]]):
             elif self.dims == 1:
                 self.input_shape = [32]
                 return np.random.random([32]).astype(np.float32)
+            elif self.dims == 0:
+                self.input_shape = []
+                return np.random.random([]).astype(np.float32)
 
-        for dims in [1, 2, 3, 4]:
+        for dims in [0, 1, 2, 3, 4]:
             for shape in [
                 [10, 8, 32, 32],
                 [2, 8, 32, 32],
@@ -125,6 +128,10 @@ def generate_dynamic_shape(attrs):
                 self.dynamic_shape.min_input_shape = {"expand_v2_input": [32]}
                 self.dynamic_shape.max_input_shape = {"expand_v2_input": [64]}
                 self.dynamic_shape.opt_input_shape = {"expand_v2_input": [32]}
+            elif self.dims == 0:
+                self.dynamic_shape.min_input_shape = {"expand_v2_input": []}
+                self.dynamic_shape.max_input_shape = {"expand_v2_input": []}
+                self.dynamic_shape.opt_input_shape = {"expand_v2_input": []}
 
         def clear_dynamic_shape():
             self.dynamic_shape.min_input_shape = {}
@@ -132,7 +139,9 @@ def clear_dynamic_shape():
             self.dynamic_shape.opt_input_shape = {}
 
         def generate_trt_nodes_num(attrs, dynamic_shape):
-            if dynamic_shape:
+            ver = paddle_infer.get_trt_compile_version()
+            ver_num = ver[0] * 1000 + ver[1] * 100 + ver[2] * 10
+            if dynamic_shape and (ver_num > 8000 or self.dims > 0):
                 return 1, 2
             else:
                 return 0, 3

From 8075752751bcdfefd1d0ac4f9c6fc25edad6dce7 Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Thu, 11 May 2023 14:48:30 +0800
Subject: [PATCH 273/405] [test]mv fluid [controlflow,detection,dlnne,tensorrt]
 tests to tests (#53470)

* [test]mv fluid controlflow detection dlnne tensorrt tests to tests

* [test]clean dlnne

* [test] fix test_tensorrt_engine_op

* [test] try fix path error

* [test] RollBACK test_tensorrt_engine_op

* [test] RollBACK test_tensorrt_engine_op

* [test]add todo

* Empty-Commit; test=document_fix
---
 paddle/fluid/operators/controlflow/CMakeLists.txt   |  5 -----
 paddle/fluid/operators/detection/CMakeLists.txt     |  4 ----
 paddle/fluid/operators/dlnne/CMakeLists.txt         |  5 -----
 test/cpp/fluid/CMakeLists.txt                       | 13 +++++++++----
 test/cpp/fluid/controlflow/CMakeLists.txt           |  4 ++++
 .../fluid}/controlflow/conditional_block_op_test.cc |  0
 test/cpp/fluid/detection/CMakeLists.txt             |  4 ++++
 .../cpp/fluid}/detection/mask_util_test.cc          |  0
 test/cpp/fluid/dlnne/CMakeLists.txt                 |  4 ++++
 .../cpp/fluid}/dlnne/dlnne_engine_op_test.cc        |  0
 10 files changed, 21 insertions(+), 18 deletions(-)
 create mode 100644 test/cpp/fluid/controlflow/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/controlflow/conditional_block_op_test.cc (100%)
 create mode 100644 test/cpp/fluid/detection/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/detection/mask_util_test.cc (100%)
 create mode 100644 test/cpp/fluid/dlnne/CMakeLists.txt
 rename {paddle/fluid/operators => test/cpp/fluid}/dlnne/dlnne_engine_op_test.cc (100%)

diff --git a/paddle/fluid/operators/controlflow/CMakeLists.txt b/paddle/fluid/operators/controlflow/CMakeLists.txt
index 7d1ae606d1710..7db85eebb4f84 100644
--- a/paddle/fluid/operators/controlflow/CMakeLists.txt
+++ b/paddle/fluid/operators/controlflow/CMakeLists.txt
@@ -27,11 +27,6 @@ cc_library(
   SRCS while_op_helper.cc
   DEPS operator op_variant)
 
-cc_test(
-  conditional_block_op_test
-  SRCS conditional_block_op_test.cc
-  DEPS conditional_block_op standalone_executor executor)
-
 if(WITH_UNITY_BUILD)
   target_link_libraries(paddle_operators_controlflow_unity conditional_block_op)
 else()
diff --git a/paddle/fluid/operators/detection/CMakeLists.txt b/paddle/fluid/operators/detection/CMakeLists.txt
index 3e07b1f155452..1bca2068f83d8 100644
--- a/paddle/fluid/operators/detection/CMakeLists.txt
+++ b/paddle/fluid/operators/detection/CMakeLists.txt
@@ -91,9 +91,5 @@ cc_library(
   mask_util
   SRCS mask_util.cc
   DEPS memory)
-cc_test(
-  mask_util_test
-  SRCS mask_util_test.cc
-  DEPS memory mask_util)
 detection_library(generate_mask_labels_op SRCS generate_mask_labels_op.cc DEPS
                   mask_util)
diff --git a/paddle/fluid/operators/dlnne/CMakeLists.txt b/paddle/fluid/operators/dlnne/CMakeLists.txt
index 6680e1f23e08d..de017546fb34e 100644
--- a/paddle/fluid/operators/dlnne/CMakeLists.txt
+++ b/paddle/fluid/operators/dlnne/CMakeLists.txt
@@ -45,8 +45,3 @@ op_library(
 #endif()
 
 target_link_libraries(dlnne_engine_op ${DLNNE_LIB} ${CURT_LIB})
-
-cc_test(
-  test_dlnne_engine_op
-  SRCS dlnne_engine_op_test.cc
-  DEPS dlnne_engine_op analysis)
diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt
index 34f45371cca2a..7bc33dfda33e8 100644
--- a/test/cpp/fluid/CMakeLists.txt
+++ b/test/cpp/fluid/CMakeLists.txt
@@ -2,9 +2,11 @@ add_subdirectory(benchmark)
 if(WITH_CINN)
   add_subdirectory(cinn)
 endif()
-# add_subdirectory(controlflow)
-# add_subdirectory(detection)
-# add_subdirectory(dlnne)
+add_subdirectory(controlflow)
+add_subdirectory(detection)
+if(WITH_DLNNE)
+  add_subdirectory(dlnne)
+endif()
 add_subdirectory(elementwise)
 add_subdirectory(fused)
 if(WITH_LITE)
@@ -21,7 +23,10 @@ endif()
 add_subdirectory(prim_ops)
 add_subdirectory(reader)
 add_subdirectory(reduce_ops)
-# add_subdirectory(tensorrt)
+# TODO(gouzil): enable this after the bug is fixed. windows: Exit code 0xc000007b, pr: #53470
+# if(WITH_GPU AND TENSORRT_FOUND)
+#   add_subdirectory(tensorrt)
+# endif()
 
 set(COMMON_OP_DEPS ${COMMON_OP_DEPS} executor)
 
diff --git a/test/cpp/fluid/controlflow/CMakeLists.txt b/test/cpp/fluid/controlflow/CMakeLists.txt
new file mode 100644
index 0000000000000..87950fdcd46f2
--- /dev/null
+++ b/test/cpp/fluid/controlflow/CMakeLists.txt
@@ -0,0 +1,4 @@
+cc_test(
+  conditional_block_op_test
+  SRCS conditional_block_op_test.cc
+  DEPS conditional_block_op standalone_executor executor)
diff --git a/paddle/fluid/operators/controlflow/conditional_block_op_test.cc b/test/cpp/fluid/controlflow/conditional_block_op_test.cc
similarity index 100%
rename from paddle/fluid/operators/controlflow/conditional_block_op_test.cc
rename to test/cpp/fluid/controlflow/conditional_block_op_test.cc
diff --git a/test/cpp/fluid/detection/CMakeLists.txt b/test/cpp/fluid/detection/CMakeLists.txt
new file mode 100644
index 0000000000000..bc9e8e0a53603
--- /dev/null
+++ b/test/cpp/fluid/detection/CMakeLists.txt
@@ -0,0 +1,4 @@
+cc_test(
+  mask_util_test
+  SRCS mask_util_test.cc
+  DEPS memory mask_util)
diff --git a/paddle/fluid/operators/detection/mask_util_test.cc b/test/cpp/fluid/detection/mask_util_test.cc
similarity index 100%
rename from paddle/fluid/operators/detection/mask_util_test.cc
rename to test/cpp/fluid/detection/mask_util_test.cc
diff --git a/test/cpp/fluid/dlnne/CMakeLists.txt b/test/cpp/fluid/dlnne/CMakeLists.txt
new file mode 100644
index 0000000000000..5089a41351ce3
--- /dev/null
+++ b/test/cpp/fluid/dlnne/CMakeLists.txt
@@ -0,0 +1,4 @@
+cc_test(
+  test_dlnne_engine_op
+  SRCS dlnne_engine_op_test.cc
+  DEPS dlnne_engine_op analysis)
diff --git a/paddle/fluid/operators/dlnne/dlnne_engine_op_test.cc b/test/cpp/fluid/dlnne/dlnne_engine_op_test.cc
similarity index 100%
rename from paddle/fluid/operators/dlnne/dlnne_engine_op_test.cc
rename to test/cpp/fluid/dlnne/dlnne_engine_op_test.cc

From 4a97ba5d27d66a8846a3ec835a769d211d9e5974 Mon Sep 17 00:00:00 2001
From: Roc <30228238+sljlp@users.noreply.github.com>
Date: Thu, 11 May 2023 15:23:25 +0800
Subject: [PATCH 274/405] [KUNLUN]Revert "revert p2p communication for xpu
 (#53496)" (#53633)

* Revert "revert p2p communication for xpu (#53496)"

This reverts commit eda0c588f2a17fb2d935ee4ad1ed640e7ccd7f71.

* update
---
 .../pp_utils/p2p_communication.py             | 30 +++++++++++++++++--
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
index da20e312a1dc8..fa4b937ba56b7 100644
--- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
+++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 import os
 
 import numpy as np
@@ -27,10 +26,30 @@
 _use_cache = False
 _enable_partial_send_recv = True
 
+_xpu_comm_group_started = False
+
 _sync_send = os.environ.get("PADDLE_P2P_SYNC_SEND", "0")
 _sync_send = _sync_send.lower() in ['1', 'true']
 
 
+def _xpu_comm_group_start():
+    if not paddle.is_compiled_with_xpu():
+        return
+    global _xpu_comm_group_started
+    assert not _xpu_comm_group_started
+    framework.core.ProcessGroupBKCL.group_start()
+    _xpu_comm_group_started = True
+
+
+def _xpu_comm_group_end():
+    if not paddle.is_compiled_with_xpu():
+        return
+    global _xpu_comm_group_started
+    if _xpu_comm_group_started:
+        framework.core.ProcessGroupBKCL.group_end()
+        _xpu_comm_group_started = False
+
+
 def initialize_p2p_groups(hcg, use_cache=True, enable_partial_send_recv=True):
     global _hcg, _use_cache, _enable_partial_send_recv
     _hcg = hcg
@@ -357,6 +376,7 @@ def _p2p_helper(
     # TODO(Yuang Liu): use batch_isend_irecv replace all these comm ops
     tasks = []
     # start to p2p communicate
+
     if _sync_send:
         # Some devices(NPU for example) do not support asynchronized send op, So the order is
         # recv_prev -> send_next -> recv_next -> send_prev
@@ -492,8 +512,8 @@ def _p2p_helper(
                     group=_hcg.send_prev_group,
                     use_calc_stream=False,
                 )
-
     else:
+        _xpu_comm_group_start()
         if tensor_send_prev is not None:
             if isinstance(tensor_send_prev, tuple):
                 for d in tensor_send_prev:
@@ -529,6 +549,7 @@ def _p2p_helper(
                         use_calc_stream=sync_recv,
                     )
                     if sync_recv:
+                        _xpu_comm_group_end()
                         allgather_partial(
                             d,
                             nranks=mp_degree,
@@ -549,6 +570,7 @@ def _p2p_helper(
                 )
 
                 if sync_recv:
+                    _xpu_comm_group_end()
                     allgather_partial(
                         tensor_recv_prev,
                         nranks=mp_degree,
@@ -595,6 +617,7 @@ def _p2p_helper(
                     )
 
                     if sync_recv:
+                        _xpu_comm_group_end()
                         allgather_partial(
                             d,
                             nranks=mp_degree,
@@ -615,6 +638,7 @@ def _p2p_helper(
                     use_calc_stream=sync_recv,
                 )
                 if sync_recv:
+                    _xpu_comm_group_end()
                     allgather_partial(
                         tensor_recv_next,
                         nranks=mp_degree,
@@ -624,7 +648,7 @@ def _p2p_helper(
                     )
                 else:
                     tasks.append(task)
-
+        _xpu_comm_group_end()
     if not sync_recv:
         if framework.in_dygraph_mode():
             # wait irecv tasks in eager dygraph mode with new comm library

From 32dae48abf14089f90264311a67e43fd639fbf69 Mon Sep 17 00:00:00 2001
From: zhoutianzi666 <39978853+zhoutianzi666@users.noreply.github.com>
Date: Thu, 11 May 2023 15:40:46 +0800
Subject: [PATCH 275/405] add unitest for reshpe 0 dims (#53685)

---
 test/ir/inference/test_trt_convert_reshape.py | 94 +++++++++++++++++++
 1 file changed, 94 insertions(+)

diff --git a/test/ir/inference/test_trt_convert_reshape.py b/test/ir/inference/test_trt_convert_reshape.py
index 3f88b39003bb9..c30d973651bad 100644
--- a/test/ir/inference/test_trt_convert_reshape.py
+++ b/test/ir/inference/test_trt_convert_reshape.py
@@ -431,5 +431,99 @@ def test(self):
         self.run_test()
 
 
+class TrtConvertReshapeZeroDimsTest(TrtLayerAutoScanTest):
+    def is_program_valid(self, program_config: ProgramConfig) -> bool:
+        return True
+
+    def sample_program_configs(self):
+        def generate_input1(attrs: List[Dict[str, Any]]):
+            if self.dims > 0:
+                self.input_shape = [1] * self.dims
+                return np.random.random(self.input_shape).astype(np.float32)
+            elif self.dims == 0:
+                self.input_shape = []
+                return np.random.random([]).astype(np.float32)
+
+        for dims in [0, 1, 2, 3]:
+            for shape in [
+                [],
+                [1, 1],
+            ]:
+                dics = [
+                    {
+                        "shape": shape,
+                    },
+                ]
+                self.dims = dims
+                dics_intput = [{"X": ["reshape_input"]}]
+
+                ops_config = [
+                    {
+                        "op_type": "reshape",
+                        "op_inputs": dics_intput[0],
+                        "op_outputs": {"Out": ["reshape_out"]},
+                        "op_attrs": dics[0],
+                    }
+                ]
+                ops = self.generate_op_config(ops_config)
+                program_config = ProgramConfig(
+                    ops=ops,
+                    weights={},
+                    inputs={
+                        "reshape_input": TensorConfig(
+                            data_gen=partial(generate_input1, dics)
+                        )
+                    },
+                    outputs=["reshape_out"],
+                )
+
+                yield program_config
+
+    def sample_predictor_configs(
+        self, program_config
+    ) -> (paddle_infer.Config, List[int], float):
+        def generate_dynamic_shape(attrs):
+            self.dynamic_shape.min_input_shape = {
+                "reshape_input": self.input_shape
+            }
+            self.dynamic_shape.max_input_shape = {
+                "reshape_input": self.input_shape
+            }
+            self.dynamic_shape.opt_input_shape = {
+                "reshape_input": self.input_shape
+            }
+
+        def clear_dynamic_shape():
+            self.dynamic_shape.min_input_shape = {}
+            self.dynamic_shape.max_input_shape = {}
+            self.dynamic_shape.opt_input_shape = {}
+
+        def generate_trt_nodes_num(attrs, dynamic_shape):
+            # only test dynamic shape mode
+            return 1, 2
+
+        attrs = [
+            program_config.ops[i].attrs for i in range(len(program_config.ops))
+        ]
+
+        # for dynamic_shape
+        generate_dynamic_shape(attrs)
+        self.trt_param.precision = paddle_infer.PrecisionType.Float32
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), 1e-5
+        self.trt_param.precision = paddle_infer.PrecisionType.Half
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), 1e-3
+
+    def add_skip_trt_case(self):
+        pass
+
+    def test(self):
+        self.add_skip_trt_case()
+        self.run_test()
+
+
 if __name__ == "__main__":
     unittest.main()

From 314d0418b6a3bf6f1b28a6612ddf86542984fca3 Mon Sep 17 00:00:00 2001
From: jjyaoao <jjyaoao@126.com>
Date: Thu, 11 May 2023 15:54:16 +0800
Subject: [PATCH 276/405] remove a part of npu (#53677)

---
 .../new_executor/interpreter/stream_analyzer.cc     |  2 +-
 paddle/fluid/memory/allocation/allocator_facade.cc  |  4 ++--
 paddle/fluid/operators/beam_search_decode_op.h      |  2 +-
 paddle/fluid/operators/collective/c_embedding_op.cc |  2 +-
 paddle/fluid/operators/reduce_ops/reduce_op.h       | 13 ++++++-------
 paddle/fluid/operators/softmax_op.cc                |  4 ++--
 paddle/fluid/platform/device_event_base.h           |  2 +-
 paddle/fluid/pybind/tensor_py.h                     |  4 ++--
 paddle/phi/backends/device_memory_aligment.h        |  2 +-
 .../meta_parallel/sharding/group_sharded_utils.py   |  2 +-
 python/paddle/distributed/launch/main.py            |  2 +-
 .../paddle/distributed/passes/auto_parallel_fp16.py |  2 +-
 python/paddle/optimizer/adamw.py                    |  4 +---
 tools/get_pr_ut.py                                  |  6 ++----
 14 files changed, 23 insertions(+), 28 deletions(-)

diff --git a/paddle/fluid/framework/new_executor/interpreter/stream_analyzer.cc b/paddle/fluid/framework/new_executor/interpreter/stream_analyzer.cc
index 38c9ce3d8091e..693365c9f47ca 100644
--- a/paddle/fluid/framework/new_executor/interpreter/stream_analyzer.cc
+++ b/paddle/fluid/framework/new_executor/interpreter/stream_analyzer.cc
@@ -150,7 +150,7 @@ DeviceContext* StreamAnalyzer::ParseDeviceContext(
 
   DeviceContext* dev_ctx = nullptr;
 
-  // only gpu needs update. xpu not need, because xpu memcpy op kernel is
+  // only gpu need update. xpu not need, because xpu memcpy op kernel is
   // synchronous.
   if (platform::is_gpu_place(place_) || platform::is_custom_place(place_)) {
     VLOG(6) << "Parse DeviceContext for " << op_type
diff --git a/paddle/fluid/memory/allocation/allocator_facade.cc b/paddle/fluid/memory/allocation/allocator_facade.cc
index 527e843d05bb8..931372a0d9a43 100644
--- a/paddle/fluid/memory/allocation/allocator_facade.cc
+++ b/paddle/fluid/memory/allocation/allocator_facade.cc
@@ -1039,8 +1039,8 @@ AllocationPtr AllocatorFacade::Alloc(const platform::Place& place,
 #elif defined(PADDLE_WITH_XPU)
   return GetAllocator(place)->Allocate(size);
 #else
-  PADDLE_THROW(platform::errors::PreconditionNotMet(
-      "Not compiled with GPU or XPU or NPU."));
+  PADDLE_THROW(
+      platform::errors::PreconditionNotMet("Not compiled with GPU or XPU."));
 #endif
 }
 
diff --git a/paddle/fluid/operators/beam_search_decode_op.h b/paddle/fluid/operators/beam_search_decode_op.h
index a6d807b028c1b..07a1c46ac0923 100644
--- a/paddle/fluid/operators/beam_search_decode_op.h
+++ b/paddle/fluid/operators/beam_search_decode_op.h
@@ -95,7 +95,7 @@ struct BeamSearchDecodeFunctor {
 
     } else {
       BeamSearchDecoder<T> beam_search_decoder(beam_size_, end_id_);
-      // Check if the tensor is on GPU or NPU. If so, use the CPU copy instead
+      // Check if the tensor is on GPU. If so, use the CPU copy instead
       if (tensor_on_gpu_ || tensor_on_npu_) {
         beam_search_decoder.Backtrace(
             step_ids_, step_scores_, id_tensor_, score_tensor_);
diff --git a/paddle/fluid/operators/collective/c_embedding_op.cc b/paddle/fluid/operators/collective/c_embedding_op.cc
index aee2b0c86b81e..2efd5b46bdc09 100644
--- a/paddle/fluid/operators/collective/c_embedding_op.cc
+++ b/paddle/fluid/operators/collective/c_embedding_op.cc
@@ -79,7 +79,7 @@ class CEmbeddingOpMaker : public framework::OpProtoAndCheckerMaker {
              "(Tensor) The input represents embedding tensors, "
              "which is a learnable parameter.");
     AddInput("Ids",
-             "An input with type int32 or int64 in CPU and GPU, int32 in NPU "
+             "An input with type int32 or int64 in CPU and GPU, "
              "contains the ids to be looked up in W.");
     AddOutput("Out", "The lookup results, which have the same type as W.");
 
diff --git a/paddle/fluid/operators/reduce_ops/reduce_op.h b/paddle/fluid/operators/reduce_ops/reduce_op.h
index 5cea4fa9e0573..40c82619db4a3 100644
--- a/paddle/fluid/operators/reduce_ops/reduce_op.h
+++ b/paddle/fluid/operators/reduce_ops/reduce_op.h
@@ -622,13 +622,12 @@ class ReduceBaseOp : public framework::OperatorWithKernel {
     // NOTE(jiahongyu): Above codes originally enclosed by PADDLE_WITH_MKLDNN
 
     if (input_data_type == framework::proto::VarType::FP16) {
-      PADDLE_ENFORCE_EQ(
-          platform::is_gpu_place(ctx.GetPlace()) ||
-              platform::is_xpu_place(ctx.GetPlace()) ||
-              platform::is_custom_place(ctx.GetPlace()),
-          true,
-          platform::errors::InvalidArgument(
-              "float16 can only be used on GPU or NPU or XPU place"));
+      PADDLE_ENFORCE_EQ(platform::is_gpu_place(ctx.GetPlace()) ||
+                            platform::is_xpu_place(ctx.GetPlace()) ||
+                            platform::is_custom_place(ctx.GetPlace()),
+                        true,
+                        platform::errors::InvalidArgument(
+                            "float16 can only be used on GPU or XPU place"));
     }
     return phi::KernelKey(input_data_type, ctx.GetPlace());
   }
diff --git a/paddle/fluid/operators/softmax_op.cc b/paddle/fluid/operators/softmax_op.cc
index 633ef748be698..2fb7883cb3f71 100644
--- a/paddle/fluid/operators/softmax_op.cc
+++ b/paddle/fluid/operators/softmax_op.cc
@@ -47,7 +47,7 @@ class SoftmaxOp : public framework::OperatorWithKernel {
               platform::is_custom_place(ctx.GetPlace()),
           true,
           platform::errors::InvalidArgument(
-              "float16 can only be used on GPU/NPU/XPU and custom place"));
+              "float16 can only be used on GPU/XPU and custom place"));
     }
     return phi::KernelKey(
         ctx.GetPlace(), layout_, phi::TransToPhiDataType(input_data_type));
@@ -130,7 +130,7 @@ class SoftmaxOpGrad : public framework::OperatorWithKernel {
             platform::is_xpu_place(ctx.GetPlace()) ||
             platform::is_custom_place(ctx.GetPlace())))
         PADDLE_THROW(platform::errors::InvalidArgument(
-            "float16 can only be used on GPU/NPU/XPU and custom place"));
+            "float16 can only be used on GPU/XPU and custom place"));
     }
     return phi::KernelKey(
         ctx.GetPlace(), layout_, phi::TransToPhiDataType(input_data_type));
diff --git a/paddle/fluid/platform/device_event_base.h b/paddle/fluid/platform/device_event_base.h
index f56688de09a32..e2de1e5a9abe3 100644
--- a/paddle/fluid/platform/device_event_base.h
+++ b/paddle/fluid/platform/device_event_base.h
@@ -65,7 +65,7 @@ class DeviceEvent {
                           MaxDeviceTypes,
                           type_id_));
 #ifndef PADDLE_WITH_CUSTOM_DEVICE
-    // TODO(Aurelius84): only support CPU/CUDA/NPU.
+    // TODO(Aurelius84): only support CPU/CUDA.
     PADDLE_ENFORCE_LT(type_id_,
                       3,
                       platform::errors::Unavailable(
diff --git a/paddle/fluid/pybind/tensor_py.h b/paddle/fluid/pybind/tensor_py.h
index 65132bc68fa0d..6f8ae115bd12a 100644
--- a/paddle/fluid/pybind/tensor_py.h
+++ b/paddle/fluid/pybind/tensor_py.h
@@ -434,7 +434,7 @@ void SetTensorFromPyArrayT(
     }
 #else
     PADDLE_THROW(platform::errors::PermissionDenied(
-        "Cannot use IPUPlace in CPU/GPU/XPU/NPU version, "
+        "Cannot use IPUPlace in CPU/GPU/XPU version, "
         "Please recompile or reinstall Paddle with IPU support."));
 #endif
   } else if (paddle::platform::is_custom_place(place)) {
@@ -1106,7 +1106,7 @@ inline py::array TensorToPyArray(const phi::DenseTensor &tensor,
     return py_arr;
 #else
     PADDLE_THROW(platform::errors::PermissionDenied(
-        "Cannot use CustomPlace in CPU/GPU/XPU/NPU version, "
+        "Cannot use CustomPlace in CPU/GPU/XPU version, "
         "Please recompile or reinstall Paddle with CustomPlace "
         "support."));
 #endif
diff --git a/paddle/phi/backends/device_memory_aligment.h b/paddle/phi/backends/device_memory_aligment.h
index 72562f0c001f0..06d9b450a83ab 100644
--- a/paddle/phi/backends/device_memory_aligment.h
+++ b/paddle/phi/backends/device_memory_aligment.h
@@ -41,7 +41,7 @@ inline size_t Alignment(size_t size,
       alignment = alignment;
 #else
       PADDLE_THROW(phi::errors::PreconditionNotMet(
-          "Fluid is not compiled with CUDA/XPU/NPU."));
+          "Fluid is not compiled with CUDA/XPU."));
 #endif
     }
   }
diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py
index 928c93df0b40a..4c47cbfcc1d0b 100644
--- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py
+++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py
@@ -323,6 +323,6 @@ def cvt_to_device(x, dev_id, blocking=True):
         place = paddle.XPUPlace(dev_id)
     else:
         raise OSError(
-            "Only supported compiled paddle with gpu/rocm, npu and xpu , but current verison is compiled with cpu."
+            "Only supported compiled paddle with gpu/rocm and xpu , but current verison is compiled with cpu."
         )
     return x._copy_to(place, blocking)
diff --git a/python/paddle/distributed/launch/main.py b/python/paddle/distributed/launch/main.py
index 02099c743933e..a2c3e34f6dee5 100644
--- a/python/paddle/distributed/launch/main.py
+++ b/python/paddle/distributed/launch/main.py
@@ -52,7 +52,7 @@ def launch():
 
         - ``--job_id``: The job unique id, it affects the log files' name. e.g., ``--job_id=job1``. Default ``--job_id=default``.
 
-        - ``--devices``: The selected accelerate devices on nodes, can be gpu/xpu/npu etc.. e.g., ``--devices=0,1,2,3`` will launch four training processes each bound to one device.
+        - ``--devices``: The selected accelerate devices on nodes, can be gpu/xpu etc.. e.g., ``--devices=0,1,2,3`` will launch four training processes each bound to one device.
 
         - ``training_script``: The full path to the single GPU training program/script to be launched in parallel, followed by all the arguments for the training script. e.g., ``training.py``
 
diff --git a/python/paddle/distributed/passes/auto_parallel_fp16.py b/python/paddle/distributed/passes/auto_parallel_fp16.py
index 3216c1b408276..6a763ce15030f 100644
--- a/python/paddle/distributed/passes/auto_parallel_fp16.py
+++ b/python/paddle/distributed/passes/auto_parallel_fp16.py
@@ -685,7 +685,7 @@ def _insert_memcopy(block, idx, src_var, dist_context, direction="D2H"):
         world_process_group.ranks,
     )
 
-    # TODO to support CUDAPinned/NPU/XPU Places
+    # TODO to support CUDAPinned/XPU Places
     if direction == "D2H":
         dst_place_type = 0
     else:
diff --git a/python/paddle/optimizer/adamw.py b/python/paddle/optimizer/adamw.py
index f8e00eabecf5e..c7e550f7aa117 100644
--- a/python/paddle/optimizer/adamw.py
+++ b/python/paddle/optimizer/adamw.py
@@ -181,9 +181,7 @@ def __init__(
                 not core.is_compiled_with_cuda()
                 and not core.is_compiled_with_xpu()
             ):
-                raise NotImplementedError(
-                    "'lr_ratio' is unimplemented in CPU, and NPU"
-                )
+                raise NotImplementedError("'lr_ratio' is unimplemented in CPU.")
 
         if parameters is not None:
             # paddle.Tensor is also iterable, so here we don't check whether
diff --git a/tools/get_pr_ut.py b/tools/get_pr_ut.py
index 8bbe39b3b7659..4da57036b68c8 100644
--- a/tools/get_pr_ut.py
+++ b/tools/get_pr_ut.py
@@ -340,10 +340,8 @@ def get_pr_ut(self):
                         file_list.append(filename)
                     else:
                         filterFiles.append(filename)
-                elif (
-                    ('/xpu/' in filename.lower())
-                    or ('/npu/' in filename.lower())
-                    or ('/ipu/' in filename.lower())
+                elif ('/xpu/' in filename.lower()) or (
+                    '/ipu/' in filename.lower()
                 ):
                     filterFiles.append(filename)
                 else:

From b4024aaffa79134d826b0cecb58fe83ea3f7b512 Mon Sep 17 00:00:00 2001
From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com>
Date: Thu, 11 May 2023 16:12:48 +0800
Subject: [PATCH 277/405] Revert elementwise (#53663)

* modify concat_grad add sum comp rule

* delete default mul_double_grad

* delete high grad test

* recover yaml

* modify yaml
---
 .../eager/auto_code_generator/generator/eager_gen.py |  1 -
 paddle/phi/api/yaml/legacy_backward.yaml             | 12 ++++++++++++
 test/prim/prim/vjp/test_comp_high_grad.py            |  4 ++--
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
index ed21e1171c17c..08a5f57d293af 100644
--- a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
+++ b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
@@ -68,7 +68,6 @@
     "matmul_double_grad",
     "tanh_double_grad",
     "add_double_grad",
-    "multiply_double_grad",
     "subtract_double_grad",
 ]
 
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index 9d1b2ce5b4933..3eaafe2b407ad 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -617,6 +617,7 @@
     func : multiply_double_grad
   optional : grad_x_grad, grad_y_grad
   inplace : (grad_x_grad -> grad_out_grad)
+  backward : multiply_triple_grad
   composite : multiply_double_grad(x, y, grad_out, grad_x_grad, grad_y_grad, axis, x_grad, y_grad, grad_out_grad)
 
 - backward_op : multiply_grad
@@ -631,6 +632,17 @@
   composite: multiply_grad(x, y, out_grad, axis, x_grad, y_grad)
   backward : multiply_double_grad
 
+- backward_op : multiply_triple_grad
+  forward : multiply_double_grad (Tensor x, Tensor y, Tensor fwd_grad_out, Tensor fwd_grad_grad_x, Tensor fwd_grad_grad_y, int aixs = -1) -> Tensor(grad_x), Tensor(grad_y), Tensor(grad_grad_out)
+  args : (Tensor x, Tensor y, Tensor fwd_grad_out, Tensor fwd_grad_grad_x, Tensor fwd_grad_grad_y, Tensor grad_x_grad, Tensor grad_y_grad, Tensor grad_grad_out_grad, int axis = -1)
+  output : Tensor(x_grad), Tensor(y_grad), Tensor(fwd_grad_out_grad), Tensor(fwd_grad_grad_x_grad), Tensor(fwd_grad_grad_y_grad)
+  infer_meta :
+    func : GeneralQuinaryGradInferMeta
+    param : [x, y, fwd_grad_out, fwd_grad_grad_x, fwd_grad_grad_y]
+  kernel :
+    func : multiply_triple_grad
+  optional : fwd_grad_grad_x, fwd_grad_grad_y, grad_x_grad, grad_y_grad, grad_grad_out_grad
+
 - backward_op : norm_grad
   forward : norm (Tensor x, int axis, float epsilon, bool is_test) -> Tensor(out), Tensor(norm)
   args : (Tensor x, Tensor norm, Tensor out_grad, int axis, float epsilon, bool is_test)
diff --git a/test/prim/prim/vjp/test_comp_high_grad.py b/test/prim/prim/vjp/test_comp_high_grad.py
index 76283528e2404..87b3c8f300ecd 100644
--- a/test/prim/prim/vjp/test_comp_high_grad.py
+++ b/test/prim/prim/vjp/test_comp_high_grad.py
@@ -226,6 +226,7 @@ def test_high_grad(self):
             self.func_triple(p)
 
 
+'''
 @param.parameterized_class(
     ('shape1', 'shape2'),
     [
@@ -328,7 +329,6 @@ def test_high_grad(self):
         for p in places:
             self.func_double(p)
             self.func_triple(p)
-
-
+'''
 if __name__ == '__main__':
     unittest.main()

From 9555ae85cedf3c2562f66fe2c4a10a7b4d21680e Mon Sep 17 00:00:00 2001
From: Wilber <jiweibo@baidu.com>
Date: Thu, 11 May 2023 16:33:32 +0800
Subject: [PATCH 278/405] Update openblas version. (#53708)

---
 cmake/external/openblas.cmake | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake
index 180214970be99..2899119cde9a8 100644
--- a/cmake/external/openblas.cmake
+++ b/cmake/external/openblas.cmake
@@ -23,7 +23,8 @@ set(CBLAS_TAG v0.3.7)
 # https://github.com/PaddlePaddle/Paddle/pull/52983
 if(UNIX
    AND NOT APPLE
-   AND NOT WITH_ROCM)
+   AND NOT WITH_ROCM
+   AND NOT WITH_XPU)
   set(CBLAS_TAG v0.3.18)
 endif()
 

From 6f28eb703303c092a3587ea4744a2d215292ff60 Mon Sep 17 00:00:00 2001
From: lijin23 <41257772+lj970926@users.noreply.github.com>
Date: Thu, 11 May 2023 17:36:10 +0800
Subject: [PATCH 279/405] [XPU][PHI Kernels] add pad op for xpu (#53684)

* add pad op for xpu

* add pad op for xpu

* add pad op for xpu
---
 paddle/phi/backends/xpu/xpu2_op_list.cc   |  10 +
 paddle/phi/kernels/xpu/pad_grad_kernel.cc |  56 ++++++
 paddle/phi/kernels/xpu/pad_kernel.cc      |  56 ++++++
 test/xpu/test_pad_op_xpu.py               | 214 ++++++++++++++++++++++
 4 files changed, 336 insertions(+)
 create mode 100644 paddle/phi/kernels/xpu/pad_grad_kernel.cc
 create mode 100644 paddle/phi/kernels/xpu/pad_kernel.cc
 create mode 100644 test/xpu/test_pad_op_xpu.py

diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc
index ed19835b6469d..6f10baf07bb5a 100644
--- a/paddle/phi/backends/xpu/xpu2_op_list.cc
+++ b/paddle/phi/backends/xpu/xpu2_op_list.cc
@@ -534,6 +534,16 @@ XPUOpMap& get_kl2_ops() {
       {"p_norm_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"pad3d_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"pad3d", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
+      {"pad",
+       XPUKernelSet({phi::DataType::FLOAT32,
+                     phi::DataType::INT32,
+                     phi::DataType::INT16,
+                     phi::DataType::FLOAT16})},
+      {"pad_grad",
+       XPUKernelSet({phi::DataType::FLOAT32,
+                     phi::DataType::INT32,
+                     phi::DataType::INT16,
+                     phi::DataType::FLOAT16})},
       {"pixel_shuffle", XPUKernelSet({phi::DataType::FLOAT32})},
       {"pixel_shuffle_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"pool2d_grad",
diff --git a/paddle/phi/kernels/xpu/pad_grad_kernel.cc b/paddle/phi/kernels/xpu/pad_grad_kernel.cc
new file mode 100644
index 0000000000000..45fc3393412cd
--- /dev/null
+++ b/paddle/phi/kernels/xpu/pad_grad_kernel.cc
@@ -0,0 +1,56 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/pad_grad_kernel.h"
+
+#include "paddle/phi/backends/xpu/enforce_xpu.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+template <typename T, typename Context>
+void PadGradKernel(const Context& dev_ctx,
+                   const DenseTensor& d_out,
+                   const std::vector<int>& paddings,
+                   const Scalar& pad_value,
+                   DenseTensor* d_x) {
+  using XPUType = typename XPUTypeTrait<T>::Type;
+  std::vector<int> pad_left, pad_right;
+  std::vector<int> out_shape = vectorize<int>(d_out.dims());
+  dev_ctx.template Alloc<T>(d_x);
+
+  for (size_t i = 0; i < paddings.size() / 2; ++i) {
+    pad_left.push_back(-paddings[i * 2]);
+    pad_right.push_back(-paddings[i * 2 + 1]);
+  }
+
+  XPUType value = static_cast<XPUType>(pad_value.to<T>());
+  int r = xpu::pad<XPUType>(dev_ctx.x_context(),
+                            reinterpret_cast<const XPUType*>(d_out.data<T>()),
+                            reinterpret_cast<XPUType*>(d_x->data<T>()),
+                            out_shape,
+                            pad_left,
+                            pad_right,
+                            value);
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "pad");
+}
+}  // namespace phi
+
+PD_REGISTER_KERNEL(pad_grad,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::PadGradKernel,
+                   float,
+                   int,
+                   int16_t,
+                   phi::dtype::float16) {}
diff --git a/paddle/phi/kernels/xpu/pad_kernel.cc b/paddle/phi/kernels/xpu/pad_kernel.cc
new file mode 100644
index 0000000000000..899503e328607
--- /dev/null
+++ b/paddle/phi/kernels/xpu/pad_kernel.cc
@@ -0,0 +1,56 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/pad_kernel.h"
+
+#include "paddle/phi/backends/xpu/enforce_xpu.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+template <typename T, typename Context>
+void PadKernel(const Context& dev_ctx,
+               const DenseTensor& x,
+               const std::vector<int>& paddings,
+               const Scalar& pad_value,
+               DenseTensor* out) {
+  using XPUType = typename XPUTypeTrait<T>::Type;
+  dev_ctx.template Alloc<T>(out);
+  std::vector<int> pad_left, pad_right;
+  std::vector<int> xshape = vectorize<int>(x.dims());
+
+  for (size_t i = 0; i < paddings.size() / 2; ++i) {
+    pad_left.push_back(paddings[i * 2]);
+    pad_right.push_back(paddings[i * 2 + 1]);
+  }
+
+  XPUType value = static_cast<XPUType>(pad_value.to<T>());
+  int r = xpu::pad<XPUType>(dev_ctx.x_context(),
+                            reinterpret_cast<const XPUType*>(x.data<T>()),
+                            reinterpret_cast<XPUType*>(out->data<T>()),
+                            xshape,
+                            pad_left,
+                            pad_right,
+                            value);
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "pad");
+}
+}  // namespace phi
+
+PD_REGISTER_KERNEL(pad,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::PadKernel,
+                   float,
+                   int,
+                   int16_t,
+                   phi::dtype::float16) {}
diff --git a/test/xpu/test_pad_op_xpu.py b/test/xpu/test_pad_op_xpu.py
new file mode 100644
index 0000000000000..4f4d68ab73d0e
--- /dev/null
+++ b/test/xpu/test_pad_op_xpu.py
@@ -0,0 +1,214 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+
+import numpy as np
+from get_test_cover_info import (
+    XPUOpTestWrapper,
+    create_test_class,
+    get_xpu_op_support_types,
+)
+from op_test_xpu import XPUOpTest
+from test_attribute_var import UnittestBase
+
+import paddle
+from paddle.fluid import Program, program_guard
+
+
+def pad_wrapper(x, paddings, pad_value):
+    return paddle.nn.functional.pad(
+        x, pad=list(paddings), mode='constant', value=pad_value
+    )
+
+
+paddle.enable_static()
+
+
+class XPUTestPadOp(XPUOpTestWrapper):
+    def __init__(self):
+        self.op_name = "pad"
+        self.use_dynamic_create_class = False
+
+    class TestPadOp(XPUOpTest):
+        def setUp(self):
+            self.op_type = "pad"
+            self.place = paddle.XPUPlace(0)
+            self.python_api = pad_wrapper
+            self.public_python_api = pad_wrapper
+            self.init_dtype()
+            self.init_test_case()
+            self.init_data()
+
+        def init_dtype(self):
+            self.dtype = self.in_type
+
+        def init_test_case(self):
+            self.shape = (16, 16)
+            self.paddings = [(0, 1), (2, 3)]
+            self.pad_value = 0.0
+
+        def init_data(self):
+            self.inputs = {'X': np.random.random(self.shape).astype(self.dtype)}
+            self.outputs = {
+                'Out': np.pad(
+                    self.inputs['X'],
+                    self.paddings,
+                    mode='constant',
+                    constant_values=self.pad_value,
+                )
+            }
+            self.attrs = {
+                'paddings': list(np.array(self.paddings).flatten()),
+                'pad_value': self.pad_value,
+            }
+
+        def test_check_output(self):
+            self.check_output_with_place(self.place)
+
+        def test_check_grad_normal(self):
+            self.check_grad_with_place(self.place, ['X'], 'Out')
+
+    class TestCase1(TestPadOp):
+        def init_test_case(self):
+            self.shape = (2, 3, 4, 5)
+            self.paddings = [(0, 1), (2, 3), (2, 1), (1, 1)]
+            self.pad_value = 0.5
+
+    class TestCase2(TestPadOp):
+        def init_test_case(self):
+            self.shape = (5, 5, 5)
+            self.paddings = [(0, 0), (0, 0), (1, 2)]
+            self.pad_value = 1.0
+
+    class TestCase3(TestPadOp):
+        def init_test_case(self):
+            self.shape = 100
+            self.paddings = [(0, 1)]
+            self.pad_value = 0.9
+
+    class TestPadOpError(unittest.TestCase):
+        def test_errors(self):
+            with paddle.fluid.framework._static_guard():
+                with program_guard(Program(), Program()):
+                    input_data = np.random.random((2, 2)).astype("float32")
+
+                def test_Variable():
+                    paddle.nn.functional.pad(x=input_data, pad=[1, 1, 1, 1])
+
+                self.assertRaises(TypeError, test_Variable)
+
+                data = paddle.static.data(
+                    name='data', shape=[4], dtype='float16'
+                )
+                paddle.nn.functional.pad(x=data, pad=[0, 1])
+
+    class TestPaddingValueTensor(UnittestBase):
+        def init_info(self):
+            self.shapes = [[2, 4]]
+            self.save_path = os.path.join(
+                self.temp_dir.name, self.path_prefix()
+            )
+
+        def test_static(self):
+            with paddle.fluid.framework._static_guard():
+                main_prog = Program()
+                starup_prog = Program()
+                with program_guard(main_prog, starup_prog):
+                    fc = paddle.nn.Linear(4, 10)
+                    x = paddle.randn([2, 4])
+                    x.stop_gradient = False
+                    feat = fc(x)  # [2,3,10]
+
+                    out = self.call_func(feat)
+
+                    sgd = paddle.optimizer.SGD()
+                    sgd.minimize(paddle.mean(out))
+                    self.assertTrue(self.var_prefix() in str(main_prog))
+                    exe = paddle.static.Executor(paddle.XPUPlace(0))
+                    exe.run(starup_prog)
+                    res = exe.run(fetch_list=[feat, out])
+                    gt = np.pad(
+                        res[0], [1, 1], 'constant', constant_values=[1.0, 1.0]
+                    )
+                    np.testing.assert_allclose(res[1], gt)
+                    paddle.static.save_inference_model(
+                        self.save_path, [x], [feat, out], exe
+                    )
+                    # Test for Inference Predictor
+                    infer_outs = self.infer_prog()
+                    gt = np.pad(
+                        infer_outs[0],
+                        [1, 1],
+                        'constant',
+                        constant_values=[1.0, 1.0],
+                    )
+                    np.testing.assert_allclose(infer_outs[1], gt)
+
+        def path_prefix(self):
+            return 'padding_value'
+
+        def var_prefix(self):
+            return "Var["
+
+        def call_func(self, x):
+            padding_value = paddle.assign([1.0])
+            out = paddle.nn.functional.pad(
+                x, pad=[1, 1, 1, 1], value=padding_value, mode='constant'
+            )
+            return out
+
+    class TestPaddingValueTensor2(TestPaddingValueTensor):
+        def call_func(self, x):
+            padding_value = paddle.assign([1.0])
+            # test for int value
+            tmp = paddle.nn.functional.pad(x, pad=[1, 1, 1, 1], value=1)
+            out = paddle.nn.functional.pad(
+                x, pad=[1, 1, 1, 1], value=padding_value
+            )
+            return out
+
+    class TestPaddingValueTensor3(unittest.TestCase):
+        def test_static(self):
+            with paddle.fluid.framework._static_guard():
+                np_x = np.random.random((16, 16)).astype('float32')
+                main_prog = Program()
+                starup_prog = Program()
+                with program_guard(main_prog, starup_prog):
+                    x = paddle.assign(np_x).astype('float32')
+                    pad_value = paddle.assign([0.0]).astype('float64')
+                    y = paddle.nn.functional.pad(
+                        x, [0, 1, 2, 3], value=pad_value
+                    )
+                    loss = y.sum()
+                    optimize_ops, params_grads = paddle.optimizer.SGD(
+                        0.01
+                    ).minimize(loss)
+
+                exe = paddle.static.Executor(paddle.XPUPlace(0))
+                res = exe.run(
+                    main_prog, fetch_list=[y] + [g for p, g in params_grads]
+                )
+                pd_out = res[0]
+                np_out = np.pad(np_x, [(0, 1), (2, 3)], constant_values=0.0)
+                np.testing.assert_allclose(pd_out, np_out)
+
+
+support_types = get_xpu_op_support_types("pad")
+for stype in support_types:
+    create_test_class(globals(), XPUTestPadOp, stype)
+
+if __name__ == "__main__":
+    unittest.main()

From 793f3b93a192eadd7f3a429307aa9f01ef68dfaf Mon Sep 17 00:00:00 2001
From: Kaipeng Deng <dengkaipeng@baidu.com>
Date: Thu, 11 May 2023 17:38:02 +0800
Subject: [PATCH 280/405] move DataLoader code to paddle.io (#48699)

* move DataLoader to paddle.io. test=develop
---
 .../distributed/auto_parallel/dist_loader.py  |   8 +-
 python/paddle/fluid/io.py                     |   2 -
 python/paddle/fluid/reader.py                 | 473 +---------------
 .../fleet/dygraph_save_for_auto_infer.py      |   2 +-
 .../test_multiprocess_dataloader_dataset.py   |   2 +-
 .../test_multiprocess_dataloader_exception.py |   2 +-
 python/paddle/incubate/autotune.py            |   6 +-
 python/paddle/io/__init__.py                  |  30 +-
 .../{fluid => io}/dataloader/__init__.py      |  29 +-
 .../{fluid => io}/dataloader/batch_sampler.py |   7 +-
 .../{fluid => io}/dataloader/collate.py       |   9 +-
 .../dataloader/dataloader_iter.py             |  44 +-
 .../{fluid => io}/dataloader/dataset.py       |  13 +-
 .../{fluid => io}/dataloader/fetcher.py       |  55 +-
 .../paddle/{fluid => io}/dataloader/flat.py   |  16 +-
 .../{fluid => io}/dataloader/sampler.py       |  10 +-
 .../paddle/{fluid => io}/dataloader/worker.py |  30 +-
 python/paddle/io/multiprocess_utils.py        | 140 +++++
 python/paddle/io/reader.py                    | 528 ++++++++++++++++++
 .../post_training_quantization.py             |   2 +-
 python/setup.py.in                            |   2 +-
 setup.py                                      |   2 +-
 .../auto_parallel_relaunch_model.py           |   2 +-
 test/auto_parallel/engine_api.py              |   8 +-
 test/auto_parallel/test_dist_attr_v2.py       |   2 +-
 test/auto_parallel/test_dist_context.py       |   2 +-
 test/auto_parallel/test_serialization.py      |   2 +-
 .../auto_parallel/test_while_op_completion.py |   2 +-
 test/auto_parallel/test_while_op_partition.py |   2 +-
 test/dygraph_to_static/test_resnet_v2.py      |   2 +-
 test/dygraph_to_static/test_simnet_v2.py      |   2 +-
 test/legacy_test/test_model.py                |  18 +-
 32 files changed, 787 insertions(+), 667 deletions(-)
 rename python/paddle/{fluid => io}/dataloader/__init__.py (55%)
 rename python/paddle/{fluid => io}/dataloader/batch_sampler.py (98%)
 rename python/paddle/{fluid => io}/dataloader/collate.py (97%)
 rename python/paddle/{fluid => io}/dataloader/dataloader_iter.py (98%)
 rename python/paddle/{fluid => io}/dataloader/dataset.py (98%)
 rename python/paddle/{fluid => io}/dataloader/fetcher.py (60%)
 rename python/paddle/{fluid => io}/dataloader/flat.py (93%)
 rename python/paddle/{fluid => io}/dataloader/sampler.py (98%)
 rename python/paddle/{fluid => io}/dataloader/worker.py (98%)
 create mode 100644 python/paddle/io/multiprocess_utils.py
 create mode 100644 python/paddle/io/reader.py

diff --git a/python/paddle/distributed/auto_parallel/dist_loader.py b/python/paddle/distributed/auto_parallel/dist_loader.py
index 5af0dd12f3ff9..5c0cd89306c90 100644
--- a/python/paddle/distributed/auto_parallel/dist_loader.py
+++ b/python/paddle/distributed/auto_parallel/dist_loader.py
@@ -17,16 +17,16 @@
 import numpy as np
 
 import paddle
-from paddle.fluid.dataloader.batch_sampler import (
+from paddle.io import BatchSampler, IterableDataset
+from paddle.io.dataloader.batch_sampler import (
     DistributedBatchSampler,
     _InfiniteIterableSampler,
 )
-from paddle.fluid.dataloader.dataloader_iter import (
+from paddle.io.dataloader.dataloader_iter import (
     _DatasetKind,
     default_collate_fn,
     default_convert_fn,
 )
-from paddle.io import BatchSampler, IterableDataset
 
 
 class DistributedDataLoaderBase(metaclass=abc.ABCMeta):
@@ -272,7 +272,7 @@ def __next__(self):
         return next(self.data)
 
     def _create_inner_dataloader(self):
-        dataloader = paddle.fluid.io.DataLoader(
+        dataloader = paddle.io.DataLoader(
             self.dataset,
             feed_list=self.feed_list,
             places=self.places,
diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py
index 605db8e932bf8..e0e102e2393f9 100644
--- a/python/paddle/fluid/io.py
+++ b/python/paddle/fluid/io.py
@@ -55,8 +55,6 @@
 from . import reader
 from . import unique_name
 from .reader import *
-from . import dataloader
-from .dataloader import *
 from . import core
 from paddle.utils import deprecated
 from paddle.fluid.framework import static_only
diff --git a/python/paddle/fluid/reader.py b/python/paddle/fluid/reader.py
index 4724077f1c9ec..ed294700b1621 100644
--- a/python/paddle/fluid/reader.py
+++ b/python/paddle/fluid/reader.py
@@ -40,14 +40,6 @@
     _cleanup,
     _set_SIGCHLD_handler,
 )
-from .dataloader import BatchSampler, Dataset, IterableDataset, Subset
-from .dataloader.dataloader_iter import (
-    _DataLoaderIterSingleProcess,
-    _DataLoaderIterMultiProcess,
-    _DatasetKind,
-    default_collate_fn,
-)
-from .dataloader.batch_sampler import _InfiniteIterableSampler
 from .layers.io import (
     monkey_patch_reader_methods,
     _copy_reader_var_,
@@ -69,22 +61,12 @@
 # NOTE: [ avoid hanging & failed quickly ] These value is used in getting data from another process
 QUEUE_GET_TIMEOUT = 60
 
-__all__ = ['PyReader', 'DataLoader', 'default_collate_fn']
+__all__ = ['PyReader', 'DataLoader']
 
 data_loader_unique_name_generator = UniqueNameGenerator()
 
 KEEP_DATA_LOADER_ORDER = True
 USE_PINNED_MEMORY = None
-# AutoTune Flags
-USE_AUTOTUNE = False
-TUNING_STEPS = 500
-
-
-def set_autotune_config(use_autotune, tuning_steps=500):
-    global USE_AUTOTUNE
-    USE_AUTOTUNE = use_autotune
-    global TUNING_STEPS
-    TUNING_STEPS = tuning_steps
 
 
 def keep_data_loader_order(*args):
@@ -171,454 +153,7 @@ def _check_input_array(cls, item):
         return arr
 
 
-class AuToTune:
-    def __init__(self, loader):
-        self.loader = loader
-        self.max_num_worker = multiprocessing.cpu_count() / 2
-
-    def __call__(self):
-        # use default loader
-        if (not USE_AUTOTUNE) or (not self.need_autotune()):
-            return self.loader.num_workers
-
-        # get autotune loader
-        auto_tune_loader = self.get_autotune_loader()
-        if auto_tune_loader is None:
-            return self.loader.num_workers
-
-        # pick the best num_workers
-        auto_tune_start = time.time()
-        logging.debug("========= DataLoader Auto Tune =========")
-        logging.debug(
-            "User config for DataLoader: " + str(self.loader.num_workers)
-        )
-        best_num_workers = 0
-        min_cost = float("inf")
-        logging.debug(
-            "Tuning Range for num_workers: 0 ~ " + str(self.max_num_worker)
-        )
-        num_workers = 0
-        while num_workers < self.max_num_worker:
-            auto_tune_loader.num_workers = num_workers
-            avg_cost = self.evaluate_reader_cost(auto_tune_loader)
-            if min_cost * 0.75 > avg_cost:
-                min_cost = avg_cost
-                best_num_workers = num_workers
-            else:
-                update_num = self.is_best(
-                    auto_tune_loader,
-                    best_num_workers,
-                    min_cost,
-                    self.max_num_worker,
-                )
-                if update_num == best_num_workers:
-                    break
-                else:
-                    best_num_workers = update_num
-            logging.debug(
-                "num_workers: "
-                + str(num_workers)
-                + " avg_cost: "
-                + str(avg_cost)
-            )
-            num_workers += 2
-        logging.info(
-            "auto_tune dataLoader best_num_workers: " + str(best_num_workers)
-        )
-        logging.debug(
-            "AutoTuning Cost for DataLoader: "
-            + str(time.time() - auto_tune_start)
-            + ' seconds'
-        )
-
-        # tune the default loader's num_workers
-        return best_num_workers
-
-    def need_autotune(self):
-        if sys.platform == 'darwin' or sys.platform == 'win32':
-            return False
-        else:
-            return True
-
-    def get_sub_dataset(self, dataset, batch_size):
-        num_samples = min(batch_size * TUNING_STEPS, len(dataset))
-        sub_dataset = Subset(dataset, indices=list(range(num_samples)))
-        return sub_dataset
-
-    def get_autotune_loader(self):
-        loader = copy.copy(self.loader)
-        batch_size = self.loader.batch_sampler.batch_size
-        if isinstance(
-            self.loader.batch_sampler, paddle.io.DistributedBatchSampler
-        ):
-            dataset = self.loader.batch_sampler.dataset
-            sub_dataset = self.get_sub_dataset(dataset, batch_size)
-            loader.batch_sampler = paddle.io.DistributedBatchSampler(
-                dataset=sub_dataset,
-                batch_size=batch_size,
-                num_replicas=self.loader.batch_sampler.nranks,
-                rank=self.loader.batch_sampler.local_rank,
-                shuffle=self.loader.batch_sampler.shuffle,
-                drop_last=self.loader.batch_sampler.drop_last,
-            )
-        elif isinstance(self.loader.batch_sampler, paddle.io.BatchSampler):
-            dataset = self.loader.batch_sampler.sampler.data_source
-            sub_dataset = self.get_sub_dataset(dataset, batch_size)
-            loader.batch_sampler = paddle.io.BatchSampler(
-                dataset=sub_dataset,
-                batch_size=batch_size,
-                drop_last=self.loader.batch_sampler.drop_last,
-            )
-        else:
-            loader = None
-        return loader
-
-    def evaluate_reader_cost(self, reader):
-        costs = []
-        avg_cost = 0
-        start = time.time()
-        for i, data in enumerate(reader):
-            costs.append(time.time() - start)
-            start = time.time()
-        if len(costs) > 2:
-            avg_cost = sum(costs[2:]) / len(costs[2:])
-        else:
-            avg_cost = sum(costs[0:]) / len(costs[0:])
-        return avg_cost
-
-    def is_best(self, reader, best_workers, best_time, num_work_boundary):
-        step = 0
-        num_workers = best_workers + 1
-        boundary = 1
-        while num_workers < num_work_boundary and step < 5:
-            self.loader.num_workers = num_workers
-            time = self.evaluate_reader_cost(reader)
-            logging.debug(
-                "for back num_workers: "
-                + str(num_workers)
-                + " avg_cost: "
-                + str(time)
-            )
-            step += 1
-            if time < best_time * 0.70 * boundary:
-                return num_workers
-            else:
-                num_workers += 1
-            boundary *= 0.80
-        return best_workers
-
-
 class DataLoader:
-    """
-    DataLoader prodives an iterator which iterates given dataset
-    once by the batch_sampler.
-
-    DataLoader supports single-process and multi-prcess data loading,
-    multi-process workers will be used to load data asynchronously if
-    :attr:`num_workers` is set as a positive number.
-
-    DataLoader supports map-style dataset and iterable-style dataset.
-
-    For map-style datast(can get a sample from dataset with a given
-    index), please see :code:`paddle.io.Dataset`.
-
-    For iterable-style datast(get samples from dataset iteratively,
-    like a Python iterator), please see :code:`paddle.io.IterableDataset`.
-
-    For :code:`batch_sampler` please see :code:`paddle.io.BatchSampler`
-
-    .. note::
-        GPU tensor operation is not supported in subprocess currently,
-        please don't use GPU tensor operations in pipeline which will
-        be performed in subprocess, such as dataset transforms, collte_fn,
-        etc. Numpy array and CPU tensor operation is supported.
-
-    **Disable automatic batching**
-
-    In certain cases such as some NLP tasks, instead of automatic batching,
-    handling batching manually in dataset is needed by users. For these
-    cases, automatic batching is disabled if both :attr:`batch_size` and
-    :attr:`batch_sampler` is set as None, each data got from :attr:`dataset`
-    should be batched data and will be processed with function define by
-    :attr:`collate_fn` or :attr:`default_collate_fn`.
-
-
-    .. note::
-        When automatic batching is disabled, :attr:`default_collate_fn` will
-        do nothing to data from dataset.
-
-
-    Args:
-        dataset(Dataset): the dataset to load data from, should be an
-            instance of subclass of :code:`paddle.io.Dataset` or
-            :code:`paddle.io.IterableDataset`.
-        feed_list (list(Tensor)|tuple(Tensor), optional): feed Tensor list.
-            The Tensors should be created by :code:`paddle.static.data()`.
-            :attr:`feed_list` must be set if :attr:`return_list` is
-            False. Default None.
-        places(list(Place)|tuple(Place)|list(str), optional): a list of Place,
-            to put data onto, :attr:`places` can be None, if
-            :attr:`places` is None, default place(CPUPlace or CUDAPlace(0))
-            will be used. Default None. If ``places`` is list of string,
-            the string in the list can be ``cpu``, ``gpu:x`` and ``gpu_pinned``,
-            where ``x`` is the index of the GPUs.
-        return_list (bool, optional): whether the return value on each device is
-            presented as a list. If :attr:`return_list=False`, the return
-            value on each device would be a dict of str -> Tensor, where
-            the key of the dict is the name of each fed Tensors. If
-            :attr:`return_list=True`, the return value on each device would
-            be a list(Tensor). :attr:`return_list` can only be True
-            in dynamic graph mode. Default True.
-        batch_sampler(BatchSampler, optional): an instance of `paddle.io.BatchSampler`
-            to generate batch indices to draw samples from :attr:`dataset`
-            and combine a batch. Default None.
-        batch_size(int|None, optional): sample number in a mini-batch, a substitution
-            parameter for :attr:`batch_sampler`, if :attr:`batch_sampler`
-            is not set, a default `paddle.io.BatchSampler` will be used
-            and initialize by :attr:`batch_size`, :attr:`shuffle` and
-            :attr:`drop_last`. Default 1.
-        shuffle(bool, optional): whther to shuffle indices order before genrate
-            batch indices, a substitution parameter for :attr:`batch_sampler`
-            see :attr:`batch_size`. Default False.
-        drop_last(bool, optional): whether drop the last incomplete batch dataset size
-            is not divisible by the batch size, a substitution parameter
-            for :attr:`batch_sampler`, see :attr:`batch_size`. Default False
-        collate_fn(callable, optional): function to generate mini-batch data by merging
-            the sample list, None for only stack each fields of sample in axis
-            0(same as :attr::`np.stack(..., axis=0)`). Default None
-        num_workers(int, optional): the number of subprocess to load data, 0 for no
-            subprocess used and loading data in main process. Default 0
-        use_buffer_reader (bool, optional): whether to use bufferred reader.
-            If use_buffer_reader=True, the DataLoader would prefetch
-            batch data asynchronously, so it would speed up data feeding
-            and occupies a little more CPU or GPU memory, i.e., the memory
-            of one batch input data. Default True.
-        prefetch_factor (int, optional): Number of batch data the DataLoader would prefetch
-            if use_buffer_reader=True. Default 2.
-        use_shared_memory (bool, optional): whether to use shared memory to speed up
-            putting data into inter-process queue, set :attr:`use_shared_memory`
-            as True only when the shared memory space on your machine(e.g.
-            space of '/dev/shm' on Linux operating sysytem) is large enough.
-            Shared memory will only be enabled in multi-process mode(num_workers
-            > 0). Default True.
-        timeout(int, optional): the timeout value for getting data form output queue
-            of subprocesses. Default 0.
-        worker_init_fn(callable, optional): init function which will be called with
-            worker id on each subproces starting if not set as None. Default
-            None.
-
-    Returns:
-        DataLoader: an iterable object for data iterating, each elemnet of the generated data is a Tensor.
-
-    Examples:
-
-        .. code-block:: python
-
-            import numpy as np
-
-            import paddle
-            import paddle.nn as nn
-            import paddle.nn.functional as F
-            from paddle.io import Dataset, BatchSampler, DataLoader
-
-            BATCH_NUM = 20
-            BATCH_SIZE = 16
-            EPOCH_NUM = 4
-
-            IMAGE_SIZE = 784
-            CLASS_NUM = 10
-
-            # define a random dataset
-            class RandomDataset(Dataset):
-                def __init__(self, num_samples):
-                    self.num_samples = num_samples
-
-                def __getitem__(self, idx):
-                    image = np.random.random([IMAGE_SIZE]).astype('float32')
-                    label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64')
-                    return image, label
-
-                def __len__(self):
-                    return self.num_samples
-
-            dataset = RandomDataset(BATCH_NUM * BATCH_SIZE)
-
-            class SimpleNet(nn.Layer):
-                def __init__(self):
-                    super().__init__()
-                    self.fc = nn.Linear(IMAGE_SIZE, CLASS_NUM)
-
-                def forward(self, image, label=None):
-                    return self.fc(image)
-
-            simple_net = SimpleNet()
-            opt = paddle.optimizer.SGD(learning_rate=1e-3,
-                                      parameters=simple_net.parameters())
-
-            loader = DataLoader(dataset,
-                                batch_size=BATCH_SIZE,
-                                shuffle=True,
-                                drop_last=True,
-                                num_workers=2)
-
-            for e in range(EPOCH_NUM):
-                for i, (image, label) in enumerate(loader()):
-                    out = simple_net(image)
-                    loss = F.cross_entropy(out, label)
-                    avg_loss = paddle.mean(loss)
-                    avg_loss.backward()
-                    opt.minimize(avg_loss)
-                    simple_net.clear_gradients()
-                    print("Epoch {} batch {}: loss = {}".format(e, i, np.mean(loss.numpy())))
-
-
-    .. note::
-        For reading iterable dataset with multiprocess Dataloader,
-        please see :code:`paddle.io.IterableDataset`
-
-    """
-
-    def __init__(
-        self,
-        dataset,
-        feed_list=None,
-        places=None,
-        return_list=True,
-        batch_sampler=None,
-        batch_size=1,
-        shuffle=False,
-        drop_last=False,
-        collate_fn=None,
-        num_workers=0,
-        use_buffer_reader=True,
-        prefetch_factor=2,
-        use_shared_memory=True,
-        timeout=0,
-        worker_init_fn=None,
-        persistent_workers=False,
-    ):
-        self.return_list = return_list
-        self.collate_fn = collate_fn
-        self.use_buffer_reader = use_buffer_reader
-        self.prefetch_factor = prefetch_factor
-        self.worker_init_fn = worker_init_fn
-
-        self.dataset = dataset
-
-        if not return_list and not _non_static_mode():
-            assert (
-                feed_list is not None
-            ), "feed_list should be set when return_list=False"
-        self.feed_list = feed_list
-
-        if places is None:
-            places = _current_expected_place()
-        if isinstance(places, (list, tuple)):
-            places = _get_paddle_place_list(places)
-        else:
-            places = _get_paddle_place(places)
-        self.places = _convert_places(places)
-
-        assert num_workers >= 0, "num_workers should be a non-negative value"
-        if num_workers > 0 and (
-            sys.platform == 'darwin' or sys.platform == 'win32'
-        ):
-            warnings.warn(
-                "DataLoader with multi-process mode is not supported on MacOs and Windows currently."
-                " Please use signle-process mode with num_workers = 0 instead"
-            )
-            num_workers = 0
-        self.num_workers = num_workers
-
-        assert prefetch_factor > 0, "prefetch_factor should be a positive value"
-
-        self.use_shared_memory = use_shared_memory
-        if use_shared_memory and num_workers == 0:
-            self.use_shared_memory = False
-
-        assert timeout >= 0, "timeout should be a non-negative value"
-        self.timeout = timeout
-
-        if isinstance(dataset, IterableDataset):
-            self.dataset_kind = _DatasetKind.ITER
-            if shuffle:
-                raise ValueError(
-                    "IterableDataset not support shuffle, but got shuffle={}".format(
-                        shuffle
-                    )
-                )
-            if batch_sampler is not None:
-                raise ValueError(
-                    "IterableDataset expect unspecified batch_sampler"
-                )
-        else:
-            self.dataset_kind = _DatasetKind.MAP
-
-        if batch_sampler is not None:
-            assert batch_size == 1 and not shuffle and not drop_last, (
-                "batch_size/shuffle/drop_last should not be set when "
-                "batch_sampler is given"
-            )
-            self.batch_sampler = batch_sampler
-            self.batch_size = None
-        elif batch_size is None:
-            self.batch_sampler = None
-            self.batch_size = None
-        else:
-            assert batch_size > 0, (
-                "batch_size should be None or a positive value when "
-                "batch_sampler is not given"
-            )
-            self.batch_size = batch_size
-            if isinstance(dataset, IterableDataset):
-                self.batch_sampler = _InfiniteIterableSampler(
-                    dataset, batch_size
-                )
-            else:
-                self.batch_sampler = BatchSampler(
-                    dataset=dataset,
-                    batch_size=batch_size,
-                    shuffle=shuffle,
-                    drop_last=drop_last,
-                )
-
-        self.drop_last = drop_last
-        self.auto_collate_batch = self.batch_sampler is not None
-
-        self.pin_memory = False
-        if _non_static_mode():
-            self.pin_memory = (
-                True if use_pinned_memory() is None else use_pinned_memory()
-            )
-
-        self._persistent_workers = persistent_workers
-        self._iterator = None
-        self.num_workers = AuToTune(self).__call__()
-
-    def __len__(self):
-        if self.dataset_kind == _DatasetKind.ITER:
-            raise ValueError("length of IterableDataset not supported")
-        else:
-            if self.auto_collate_batch:
-                return len(self.batch_sampler)
-            else:
-                return len(self.dataset)
-
-    def __iter__(self):
-        if self.num_workers == 0:
-            return _DataLoaderIterSingleProcess(self)
-        elif self._persistent_workers:
-            if self._iterator is None:
-                self._iterator = _DataLoaderIterMultiProcess(self)
-            else:
-                self._iterator._reset()
-            return self._iterator
-        else:
-            return _DataLoaderIterMultiProcess(self)
-
-    def __call__(self):
-        return self.__iter__()
-
     @staticmethod
     def from_generator(
         feed_list=None,
@@ -793,7 +328,7 @@ def set_data_source(loader, places):
                 label = static.data(name='label', shape=[None, 1], dtype='int64')
 
                 # Define DataLoader
-                loader = paddle.io.DataLoader.from_generator(feed_list=[image, label], capacity=16, iterable=ITERABLE)
+                loader = paddle.fluid.io.DataLoader.from_generator(feed_list=[image, label], capacity=16, iterable=ITERABLE)
 
                 # Define network
                 loss = simple_net(image, label)
@@ -867,7 +402,7 @@ def forward(self, x):
                 adam = opt.Adam(learning_rate=0.001, parameters=dp_layer.parameters())
 
                 # create data loader
-                loader = paddle.io.DataLoader.from_generator(capacity=5)
+                loader = paddle.fluid.io.DataLoader.from_generator(capacity=5)
                 loader.set_batch_generator(random_batch_reader())
 
                 for epoch_id in range(EPOCH_NUM):
@@ -944,7 +479,7 @@ def from_dataset(dataset, places, drop_last=True):
                     use_var=[image, label])
                 dataset.set_filelist(['a.txt', 'b.txt', 'c.txt'])
 
-                loader = paddle.io.DataLoader.from_dataset(dataset, static.cpu_places())
+                loader = paddle.fluid.io.DataLoader.from_dataset(dataset, static.cpu_places())
         """
         return DatasetLoader(dataset, places, drop_last)
 
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_save_for_auto_infer.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_save_for_auto_infer.py
index c74e2b7adaa22..a2a9c9113271b 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_save_for_auto_infer.py
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_save_for_auto_infer.py
@@ -37,8 +37,8 @@
 )
 from paddle.distributed.sharding.group_sharded import group_sharded_parallel
 from paddle.distributed.utils.log_utils import get_logger
-from paddle.fluid.dataloader.dataset import IterableDataset
 from paddle.incubate.distributed.utils.io import save_for_auto_inference
+from paddle.io import IterableDataset
 from paddle.nn import Linear
 
 logger = get_logger("INFO", __file__)
diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py
index d7e09481a1c71..9e2b89b12860c 100755
--- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py
+++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py
@@ -406,7 +406,7 @@ def setUp(self):
         ]
 
     def test_main(self):
-        from paddle.fluid.dataloader.worker import _generate_states
+        from paddle.io.dataloader.worker import _generate_states
 
         for inp, outp in zip(self.inputs, self.outputs):
             out = _generate_states(*inp)
diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py
index 2b4d1a78d1ea3..bfd08f703c4f6 100644
--- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py
+++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py
@@ -19,8 +19,8 @@
 
 from paddle import fluid
 from paddle.fluid import core
-from paddle.fluid.dataloader.dataloader_iter import _worker_loop
 from paddle.io import BatchSampler, DataLoader, Dataset, IterableDataset
+from paddle.io.dataloader.worker import _worker_loop
 
 
 class RandomDataset(Dataset):
diff --git a/python/paddle/incubate/autotune.py b/python/paddle/incubate/autotune.py
index 742c5bded064a..dfad1dc58c928 100644
--- a/python/paddle/incubate/autotune.py
+++ b/python/paddle/incubate/autotune.py
@@ -84,7 +84,7 @@ def set_config(config=None):
     if config is None:
         core.enable_autotune()
         core.enable_layout_autotune()
-        paddle.fluid.reader.set_autotune_config(use_autotune=True)
+        paddle.io.reader.set_autotune_config(use_autotune=True)
         return
 
     config_dict = {}
@@ -147,7 +147,7 @@ def set_config(config=None):
                 )
         if "tuning_steps" in dataloader_config:
             if isinstance(dataloader_config['tuning_steps'], int):
-                paddle.fluid.reader.set_autotune_config(
+                paddle.io.reader.set_autotune_config(
                     use_autoune, dataloader_config['tuning_steps']
                 )
             else:
@@ -155,4 +155,4 @@ def set_config(config=None):
                     "The auto-tuning configuration of the dataloader is incorrect."
                     "The `tuning_steps` should be int. Use default parameter instead."
                 )
-                paddle.fluid.reader.set_autotune_config(use_autoune)
+                paddle.io.reader.set_autotune_config(use_autoune)
diff --git a/python/paddle/io/__init__.py b/python/paddle/io/__init__.py
index a9c0e9a2f2d2f..6c2e0dae67834 100755
--- a/python/paddle/io/__init__.py
+++ b/python/paddle/io/__init__.py
@@ -14,21 +14,21 @@
 
 # TODO: define all functions about input & output in this directory
 
-from ..fluid.io import DataLoader  # noqa: F401
-from ..fluid.dataloader import Dataset  # noqa: F401
-from ..fluid.dataloader import IterableDataset  # noqa: F401
-from ..fluid.dataloader import BatchSampler  # noqa: F401
-from ..fluid.dataloader import get_worker_info  # noqa: F401
-from ..fluid.dataloader import TensorDataset  # noqa: F401
-from ..fluid.dataloader import Sampler  # noqa: F401
-from ..fluid.dataloader import SequenceSampler  # noqa: F401
-from ..fluid.dataloader import RandomSampler  # noqa: F401
-from ..fluid.dataloader import DistributedBatchSampler  # noqa: F401
-from ..fluid.dataloader import ComposeDataset  # noqa: F401
-from ..fluid.dataloader import ChainDataset  # noqa: F401
-from ..fluid.dataloader import WeightedRandomSampler  # noqa: F401
-from ..fluid.dataloader import Subset  # noqa: F401
-from ..fluid.dataloader import random_split  # noqa: F401
+from .reader import DataLoader  # noqa: F401
+from .dataloader import Dataset  # noqa: F401
+from .dataloader import IterableDataset  # noqa: F401
+from .dataloader import BatchSampler  # noqa: F401
+from .dataloader import get_worker_info  # noqa: F401
+from .dataloader import TensorDataset  # noqa: F401
+from .dataloader import Sampler  # noqa: F401
+from .dataloader import SequenceSampler  # noqa: F401
+from .dataloader import RandomSampler  # noqa: F401
+from .dataloader import DistributedBatchSampler  # noqa: F401
+from .dataloader import ComposeDataset  # noqa: F401
+from .dataloader import ChainDataset  # noqa: F401
+from .dataloader import WeightedRandomSampler  # noqa: F401
+from .dataloader import Subset  # noqa: F401
+from .dataloader import random_split  # noqa: F401
 
 __all__ = [  # noqa
     'Dataset',
diff --git a/python/paddle/fluid/dataloader/__init__.py b/python/paddle/io/dataloader/__init__.py
similarity index 55%
rename from python/paddle/fluid/dataloader/__init__.py
rename to python/paddle/io/dataloader/__init__.py
index c0b2052283b1c..bb65463f70afc 100644
--- a/python/paddle/fluid/dataloader/__init__.py
+++ b/python/paddle/io/dataloader/__init__.py
@@ -12,21 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from . import dataset
-from .dataset import *
+from .dataset import Dataset
+from .dataset import IterableDataset
+from .dataset import TensorDataset
+from .dataset import ComposeDataset
+from .dataset import ChainDataset
+from .dataset import random_split
+from .dataset import Subset
 
-from . import batch_sampler
-from .batch_sampler import *
+from .batch_sampler import BatchSampler
+from .batch_sampler import DistributedBatchSampler
 
-from . import dataloader_iter
-from .dataloader_iter import *
+from .worker import get_worker_info
 
-from . import sampler
-from .sampler import *
-
-__all__ = (
-    dataset.__all__
-    + batch_sampler.__all__
-    + dataloader_iter.__all__
-    + sampler.__all__
-)
+from .sampler import Sampler
+from .sampler import SequenceSampler
+from .sampler import RandomSampler
+from .sampler import WeightedRandomSampler
diff --git a/python/paddle/fluid/dataloader/batch_sampler.py b/python/paddle/io/dataloader/batch_sampler.py
similarity index 98%
rename from python/paddle/fluid/dataloader/batch_sampler.py
rename to python/paddle/io/dataloader/batch_sampler.py
index 3e0449719c4cd..190e9240900f8 100644
--- a/python/paddle/fluid/dataloader/batch_sampler.py
+++ b/python/paddle/io/dataloader/batch_sampler.py
@@ -12,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import numpy as np
 import math
 
-from .sampler import Sampler, SequenceSampler, RandomSampler
-from .dataset import Dataset, IterableDataset
+import numpy as np
 
-__all__ = ["BatchSampler", "DistributedBatchSampler"]
+from .dataset import IterableDataset
+from .sampler import RandomSampler, Sampler, SequenceSampler
 
 
 class BatchSampler(Sampler):
diff --git a/python/paddle/fluid/dataloader/collate.py b/python/paddle/io/dataloader/collate.py
similarity index 97%
rename from python/paddle/fluid/dataloader/collate.py
rename to python/paddle/io/dataloader/collate.py
index dd70a3421409d..141624668f09b 100644
--- a/python/paddle/fluid/dataloader/collate.py
+++ b/python/paddle/io/dataloader/collate.py
@@ -12,13 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import paddle
 import numbers
+from collections.abc import Mapping, Sequence
+
 import numpy as np
-from ..framework import _non_static_mode
-from .. import core, layers
 
-from collections.abc import Sequence, Mapping
+import paddle
+
+from ...framework import core
 
 
 def default_collate_fn(batch):
diff --git a/python/paddle/fluid/dataloader/dataloader_iter.py b/python/paddle/io/dataloader/dataloader_iter.py
similarity index 98%
rename from python/paddle/fluid/dataloader/dataloader_iter.py
rename to python/paddle/io/dataloader/dataloader_iter.py
index 2b06c371ef36f..43b749c869dd6 100644
--- a/python/paddle/fluid/dataloader/dataloader_iter.py
+++ b/python/paddle/io/dataloader/dataloader_iter.py
@@ -12,51 +12,39 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import itertools
+import logging
 import os
+import queue
 import sys
-import time
-import signal
-import numbers
-import logging
-import itertools
 import threading
+import time
 import warnings
-import numpy as np
-from collections import namedtuple
-from paddle.fluid.framework import (
-    _set_expected_place,
-    _current_expected_place,
-    set_flags,
-)
 
-import queue
+import numpy as np
 
 import paddle
-import paddle.profiler as profiler
+from paddle import profiler
+from paddle.fluid.framework import _current_expected_place, _set_expected_place
+from paddle.profiler.timer import benchmark
 from paddle.profiler.utils import in_profiler_mode
-from .. import core, layers
-from ..framework import in_dygraph_mode
+
+from ...framework import core, in_dygraph_mode
 from ..multiprocess_utils import (
-    _set_SIGCHLD_handler,
     MP_STATUS_CHECK_INTERVAL,
     CleanupFuncRegistrar,
+    _set_SIGCHLD_handler,
 )
-from .fetcher import _IterableDatasetFetcher, _MapDatasetFetcher
 from .batch_sampler import _InfiniteIterableSampler
 from .collate import default_collate_fn, default_convert_fn
+from .flat import _flatten_batch, _restore_batch
 from .worker import (
-    ParentWatchDog,
-    get_worker_info,
-    _worker_loop,
     _DatasetKind,
     _IterableDatasetStopIteration,
-    _WorkerException,
     _ResumeIteration,
+    _worker_loop,
+    _WorkerException,
 )
-from .flat import _flatten_batch, _restore_batch
-from paddle.profiler.timer import benchmark
-
-__all__ = ['get_worker_info']
 
 # NOTE: fix `terminate called without an active exception`
 # if for loop break and program exit immediately(with no model
@@ -95,7 +83,7 @@ class _DataLoaderIterBase:
     data by setting in given dataloader.
 
     Args:
-        loader(instance of DataLoader): instance of `fluid.io.DataLoader`
+        loader(instance of DataLoader): instance of `paddle.io.DataLoader`
     """
 
     def __init__(self, loader):
@@ -439,7 +427,7 @@ def __init__(self, loader):
         self._shutdown = False
 
     def _init_workers(self):
-        import paddle.incubate.multiprocessing as multiprocessing
+        from paddle.incubate import multiprocessing
 
         # multiprocess worker and indice queue list initial as empty
         self._workers = []
diff --git a/python/paddle/fluid/dataloader/dataset.py b/python/paddle/io/dataloader/dataset.py
similarity index 98%
rename from python/paddle/fluid/dataloader/dataset.py
rename to python/paddle/io/dataloader/dataset.py
index 3701da0b33ec7..e8bb6bbd364c8 100755
--- a/python/paddle/fluid/dataloader/dataset.py
+++ b/python/paddle/io/dataloader/dataset.py
@@ -13,17 +13,8 @@
 # limitations under the License.
 
 import paddle
-from .. import framework
-
-__all__ = [
-    "Dataset",
-    "IterableDataset",
-    "TensorDataset",
-    "ComposeDataset",
-    "ChainDataset",
-    "random_split",
-    "Subset",
-]
+
+from ... import framework
 
 
 class Dataset:
diff --git a/python/paddle/fluid/dataloader/fetcher.py b/python/paddle/io/dataloader/fetcher.py
similarity index 60%
rename from python/paddle/fluid/dataloader/fetcher.py
rename to python/paddle/io/dataloader/fetcher.py
index b097a315c0c73..309d009cfc106 100644
--- a/python/paddle/fluid/dataloader/fetcher.py
+++ b/python/paddle/io/dataloader/fetcher.py
@@ -12,12 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
-from ..log_helper import get_logger
-from collections.abc import Sequence, Mapping
-
-_WARNING_TO_LOG = True
-
 
 class _DatasetFetcher:
     def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last):
@@ -37,47 +31,8 @@ def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last):
     #       ecah sample processing in the batch
     def fetch(self, batch_indices, done_event=None):
         raise NotImplementedError(
-            "'fetch' not implement for class {}".format(self.__class__.__name__)
-        )
-
-    def _log_warning(self):
-        # only log warning on GPU 0 when distributed launch
-        from ...distributed import get_world_size, get_rank
-
-        if get_world_size() >= 2 and get_rank() != 0:
-            return
-
-        warn_str = (
-            "Detect dataset only contains single fileds, return format "
-            "changed since Paddle 2.1. In Paddle <= 2.0, DataLoader add "
-            "a list surround output data(e.g. return [data]), and in "
-            "Paddle >= 2.1, DataLoader return the single filed directly "
-            "(e.g. return data). For example, in following code: \n\n"
-        )
-        warn_str += (
-            "import numpy as np\n"
-            "from paddle.io import DataLoader, Dataset\n\n"
-            "class RandomDataset(Dataset):\n"
-            "    def __getitem__(self, idx):\n"
-            "        data = np.random.random((2, 3)).astype('float32')\n\n"
-            "        return data\n\n"
-            "    def __len__(self):\n"
-            "        return 10\n\n"
-            "dataset = RandomDataset()\n"
-            "loader = DataLoader(dataset, batch_size=1)\n"
-            "data = next(loader())\n\n"
-        )
-
-        warn_str += (
-            "In Paddle <= 2.0, data is in format '[Tensor(shape=(1, 2, 3), "
-            "dtype=float32)]', and in Paddle >= 2.1, data is in format"
-            " 'Tensor(shape=(1, 2, 3), dtype=float32)'\n"
-        )
-
-        logger = get_logger(
-            "DataLoader", logging.INFO, fmt='%(levelname)s: %(message)s'
+            f"'fetch' not implement for class {self.__class__.__name__}"
         )
-        logger.warning(warn_str)
 
 
 class _IterableDatasetFetcher(_DatasetFetcher):
@@ -103,10 +58,6 @@ def fetch(self, batch_indices, done_event=None):
             ):
                 raise StopIteration
 
-            global _WARNING_TO_LOG
-            if not isinstance(data[0], (Sequence, Mapping)) and _WARNING_TO_LOG:
-                self._log_warning()
-                _WARNING_TO_LOG = False
         else:
             data = next(self.dataset_iter)
 
@@ -128,10 +79,6 @@ def fetch(self, batch_indices, done_event=None):
                 else:
                     return None
 
-            global _WARNING_TO_LOG
-            if not isinstance(data[0], (Sequence, Mapping)) and _WARNING_TO_LOG:
-                self._log_warning()
-                _WARNING_TO_LOG = False
         else:
             data = self.dataset[batch_indices]
 
diff --git a/python/paddle/fluid/dataloader/flat.py b/python/paddle/io/dataloader/flat.py
similarity index 93%
rename from python/paddle/fluid/dataloader/flat.py
rename to python/paddle/io/dataloader/flat.py
index 1e1ed1eebd806..f674d7fb2b4b9 100644
--- a/python/paddle/fluid/dataloader/flat.py
+++ b/python/paddle/io/dataloader/flat.py
@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import paddle
 import numbers
-import numpy as np
+from collections.abc import Mapping, Sequence
 
-from collections.abc import Sequence, Mapping
+import numpy as np
 
+import paddle
 
 FIELD_PREFIX = "_paddle_field_"
 
@@ -38,7 +38,7 @@ def _flatten(batch, flat_batch, structure, field_idx):
                     field,
                     (np.ndarray, paddle.Tensor, paddle.fluid.core.eager.Tensor),
                 ):
-                    structure.append('{}{}'.format(FIELD_PREFIX, field_idx))
+                    structure.append(f'{FIELD_PREFIX}{field_idx}')
                     flat_batch.append(field)
                     field_idx += 1
                 elif isinstance(field, (str, bytes, numbers.Number)):
@@ -61,7 +61,7 @@ def _flatten(batch, flat_batch, structure, field_idx):
                     field,
                     (np.ndarray, paddle.Tensor, paddle.fluid.core.eager.Tensor),
                 ):
-                    structure[k] = '{}{}'.format(FIELD_PREFIX, field_idx)
+                    structure[k] = f'{FIELD_PREFIX}{field_idx}'
                     flat_batch.append(field)
                     field_idx += 1
                 elif isinstance(field, (str, bytes, numbers.Number)):
@@ -79,7 +79,7 @@ def _flatten(batch, flat_batch, structure, field_idx):
                 else:
                     structure[k] = field
         else:
-            raise TypeError("wrong flat data type: {}".format(type(batch)))
+            raise TypeError(f"wrong flat data type: {type(batch)}")
 
         return structure, field_idx
 
@@ -130,7 +130,7 @@ def _restore(structure, field_idx):
                 elif isinstance(field, (Sequence, Mapping)):
                     field_idx = _restore(structure[k], field_idx)
         else:
-            raise TypeError("wrong flat data type: {}".format(type(structure)))
+            raise TypeError(f"wrong flat data type: {type(structure)}")
 
         return field_idx
 
@@ -145,7 +145,7 @@ def _restore(structure, field_idx):
     if isinstance(structure, (str, bytes)):
         assert structure == '{}{}'.format(
             FIELD_PREFIX, 0
-        ), "invalid structure: {}".format(structure)
+        ), f"invalid structure: {structure}"
         return flat_batch[0]
     field_idx = _restore(structure, 0)
     assert field_idx + 1 == len(flat_batch), "Tensor parse incomplete"
diff --git a/python/paddle/fluid/dataloader/sampler.py b/python/paddle/io/dataloader/sampler.py
similarity index 98%
rename from python/paddle/fluid/dataloader/sampler.py
rename to python/paddle/io/dataloader/sampler.py
index a6ec3ffbae9b8..aa8a4e649c76c 100644
--- a/python/paddle/fluid/dataloader/sampler.py
+++ b/python/paddle/io/dataloader/sampler.py
@@ -13,14 +13,8 @@
 # limitations under the License.
 
 import numpy as np
-from .. import core
 
-__all__ = [
-    "Sampler",
-    "SequenceSampler",
-    "RandomSampler",
-    "WeightedRandomSampler",
-]
+from ...framework import core
 
 
 class Sampler:
@@ -317,7 +311,7 @@ def __iter__(self):
         idxs = _weighted_sample(
             self.weights, self.num_samples, self.replacement
         )
-        return iter(idxs.reshape((-1)).tolist())
+        return iter(idxs.reshape(-1).tolist())
 
     def __len__(self):
         mul = np.prod(self.weights.shape) // self.weights.shape[-1]
diff --git a/python/paddle/fluid/dataloader/worker.py b/python/paddle/io/dataloader/worker.py
similarity index 98%
rename from python/paddle/fluid/dataloader/worker.py
rename to python/paddle/io/dataloader/worker.py
index de6c382054e0a..4ca80e09ae65e 100644
--- a/python/paddle/fluid/dataloader/worker.py
+++ b/python/paddle/io/dataloader/worker.py
@@ -13,25 +13,25 @@
 # limitations under the License.
 
 import os
+
+# NOTE: queue has a different name in python2 and python3
+import queue
 import sys
-import paddle
-import numpy as np
 import traceback
-from collections import namedtuple
-from .. import core
-from .fetcher import _IterableDatasetFetcher, _MapDatasetFetcher
+
+import numpy as np
+
+import paddle
+
+from ...framework import core
 from ..multiprocess_utils import (
-    _cleanup_mmap,
-    CleanupFuncRegistrar,
     MP_STATUS_CHECK_INTERVAL,
+    CleanupFuncRegistrar,
+    _cleanup_mmap,
 )
-from ..framework import _non_static_mode, _in_eager_without_dygraph_check
+from .fetcher import _IterableDatasetFetcher, _MapDatasetFetcher
 from .flat import _flatten_batch
 
-import queue
-
-__all__ = ['get_worker_info']
-
 
 class _IterableDatasetStopIteration:
     def __init__(self, worker_id):
@@ -59,7 +59,7 @@ def create_fetcher(
                 dataset, auto_collate_batch, collate_fn, drop_last
             )
         else:
-            raise NotImplementedError("unknown Dataset kind {}".format(kind))
+            raise NotImplementedError(f"unknown Dataset kind {kind}")
 
 
 class ParentWatchDog:
@@ -291,9 +291,9 @@ def _worker_loop(
 
         # set different numpy seed for each worker
         try:
-            import numpy as np
-            import time
             import random
+
+            import numpy as np
         except ImportError:
             pass
         else:
diff --git a/python/paddle/io/multiprocess_utils.py b/python/paddle/io/multiprocess_utils.py
new file mode 100644
index 0000000000000..5792983ceb475
--- /dev/null
+++ b/python/paddle/io/multiprocess_utils.py
@@ -0,0 +1,140 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import atexit
+
+# NOTE: queue has a different name in python2 and python3
+import queue
+import signal
+import sys
+
+from ..framework import core
+
+# multi-process worker check indices queue interval, avoid
+# hanging in subprocess data loading
+MP_STATUS_CHECK_INTERVAL = 5.0
+
+# NOTE: [ mmap files clear ] If there is still data in the multiprocess queue when the main process finishes reading,
+# the data in the queue needs to be popped. Then the LoDTensor read by the main process
+# from the child process will automatically clear the memory-mapped file.
+multiprocess_queue_set = set()
+
+
+def _clear_multiprocess_queue_set():
+    global multiprocess_queue_set
+    for data_queue in multiprocess_queue_set:
+        while True:
+            try:
+                data_queue.get_nowait()
+            except queue.Empty:
+                break
+
+
+# NOTE: main process clear function at exit
+def _cleanup():
+    # NOTE: inter-process Queue shared memory objects clear function
+    _clear_multiprocess_queue_set()
+    # NOTE: main process memory map files clear funciton
+    core._cleanup_mmap_fds()
+
+
+# NOTE: for child process clear function at exit
+def _cleanup_mmap():
+    # clear memory map files in child process
+    core._cleanup_mmap_fds()
+
+
+# NOTE used for register a function to be executed at interpreter exit.
+class CleanupFuncRegistrar:
+    # Record the cleanup functions that have been executed
+    _executed_func_set = set()
+    # Record the cleanup functions that have been registered
+    _registered_func_set = set()
+
+    @classmethod
+    def register(cls, function, signals=[]):
+        def _func_exectuor():
+            if function not in cls._executed_func_set:
+                try:
+                    function()
+                finally:
+                    cls._executed_func_set.add(function)
+
+        def _func_register(function):
+            if not callable(function):
+                raise TypeError("%s is not callable object." % (function))
+            # check function object whether hash-able {function}
+            if function not in cls._registered_func_set:
+                atexit.register(_func_exectuor)
+                cls._registered_func_set.add(function)
+
+        def _signal_handler(signum=None, frame=None):
+            _func_exectuor()
+            if signum is not None:
+                if signum == signal.SIGINT:
+                    raise KeyboardInterrupt
+                sys.exit(signum)
+
+        def _signal_register(signals):
+            signals = set(signals)
+            for sig in signals:
+                orig_handler = signal.signal(sig, _signal_handler)
+                if orig_handler not in (signal.SIG_DFL, signal.SIG_IGN):
+                    if (
+                        sig == signal.SIGINT
+                        and orig_handler is signal.default_int_handler
+                    ):
+                        continue
+                    if orig_handler not in cls._registered_func_set:
+                        atexit.register(orig_handler)
+                        cls._registered_func_set.add(orig_handler)
+
+        # deal with signals
+        _signal_register(signals)
+        # deal with function
+        _func_register(function)
+
+
+# NOTE: [ mmap files clear ] When the main process exits unexpectedly, the remaining
+# shared memory objects in the inter-process Queue and the main process (mostly in the
+# BlockingQueue) may not be completely released, resulting in the corresponding
+# memory-mapped file remaining on the disk (/dev/shm), so register this function
+# to clean up shared memory objects in these two queues before the python interpreter exits.
+# NOTE: Currently multi-process DataLoader only supports Linux platform
+if not (sys.platform == 'darwin' or sys.platform == 'win32'):
+    CleanupFuncRegistrar.register(_cleanup)
+
+# ------------ SIGCHLD handler setting --------------
+_SIGCHLD_handler_set = False
+
+
+def _set_SIGCHLD_handler():
+    global _SIGCHLD_handler_set
+    if _SIGCHLD_handler_set:
+        return
+
+    current_handler = signal.getsignal(signal.SIGCHLD)
+    if not callable(current_handler):
+        current_handler = None
+
+    def __handler__(signum, frame):
+        # NOTE: Here the signum is SIGCHLD, when the child process exits,
+        # this handler will be called whenever the child process exits
+        # normally or abnormally.
+        core._throw_error_if_process_failed()
+        if current_handler is not None:
+            current_handler(signum, frame)
+
+    signal.signal(signal.SIGCHLD, __handler__)
+    _SIGCHLD_handler_set = True
diff --git a/python/paddle/io/reader.py b/python/paddle/io/reader.py
new file mode 100644
index 0000000000000..6698caa435f94
--- /dev/null
+++ b/python/paddle/io/reader.py
@@ -0,0 +1,528 @@
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import multiprocessing
+
+# NOTE: queue has a different name in python2 and python3
+import sys
+import time
+import warnings
+
+import paddle
+from paddle.fluid.framework import logging
+
+from ..fluid.framework import (
+    _current_expected_place,
+    _get_paddle_place,
+    _get_paddle_place_list,
+    _non_static_mode,
+)
+from ..framework import core
+from .dataloader import BatchSampler, IterableDataset, Subset
+from .dataloader.batch_sampler import _InfiniteIterableSampler
+from .dataloader.dataloader_iter import (
+    _DataLoaderIterMultiProcess,
+    _DataLoaderIterSingleProcess,
+    _DatasetKind,
+)
+
+# NOTE: [ avoid hanging & failed quickly ]
+# These value is used in getting data from another process
+QUEUE_GET_TIMEOUT = 60
+
+USE_PINNED_MEMORY = None
+# AutoTune Flags
+USE_AUTOTUNE = False
+TUNING_STEPS = 500
+
+
+def set_autotune_config(use_autotune, tuning_steps=500):
+    global USE_AUTOTUNE
+    USE_AUTOTUNE = use_autotune
+    global TUNING_STEPS
+    TUNING_STEPS = tuning_steps
+
+
+def use_pinned_memory(*args):
+    global USE_PINNED_MEMORY
+    if len(args) == 0:
+        return USE_PINNED_MEMORY
+    else:
+        assert len(args) == 1 and isinstance(args[0], bool)
+        USE_PINNED_MEMORY = args[0]
+
+
+def _convert_places(places):
+    if not isinstance(places, (list, tuple)):
+        places = [places]
+
+    ret = []
+    for p in places:
+        if not isinstance(p, core.Place):
+            tmp = core.Place()
+            tmp.set_place(p)
+            p = tmp
+
+        ret.append(p)
+    return ret
+
+
+class AuToTune:
+    def __init__(self, loader):
+        self.loader = loader
+        self.max_num_worker = multiprocessing.cpu_count() / 2
+
+    def __call__(self):
+        # use default loader
+        if (not USE_AUTOTUNE) or (not self.need_autotune()):
+            return self.loader.num_workers
+
+        # get autotune loader
+        auto_tune_loader = self.get_autotune_loader()
+        if auto_tune_loader is None:
+            return self.loader.num_workers
+
+        # pick the best num_workers
+        auto_tune_start = time.time()
+        logging.debug("========= DataLoader Auto Tune =========")
+        logging.debug(
+            "User config for DataLoader: " + str(self.loader.num_workers)
+        )
+        best_num_workers = 0
+        min_cost = float("inf")
+        logging.debug(
+            "Tuning Range for num_workers: 0 ~ " + str(self.max_num_worker)
+        )
+        num_workers = 0
+        while num_workers < self.max_num_worker:
+            auto_tune_loader.num_workers = num_workers
+            avg_cost = self.evaluate_reader_cost(auto_tune_loader)
+            if min_cost * 0.75 > avg_cost:
+                min_cost = avg_cost
+                best_num_workers = num_workers
+            else:
+                update_num = self.is_best(
+                    auto_tune_loader,
+                    best_num_workers,
+                    min_cost,
+                    self.max_num_worker,
+                )
+                if update_num == best_num_workers:
+                    break
+                else:
+                    best_num_workers = update_num
+            logging.debug(
+                "num_workers: "
+                + str(num_workers)
+                + " avg_cost: "
+                + str(avg_cost)
+            )
+            num_workers += 2
+        logging.info(
+            "auto_tune dataLoader best_num_workers: " + str(best_num_workers)
+        )
+        logging.debug(
+            "AutoTuning Cost for DataLoader: "
+            + str(time.time() - auto_tune_start)
+            + ' seconds'
+        )
+
+        # tune the default loader's num_workers
+        return best_num_workers
+
+    def need_autotune(self):
+        if sys.platform == 'darwin' or sys.platform == 'win32':
+            return False
+        else:
+            return True
+
+    def get_sub_dataset(self, dataset, batch_size):
+        num_samples = min(batch_size * TUNING_STEPS, len(dataset))
+        sub_dataset = Subset(dataset, indices=list(range(num_samples)))
+        return sub_dataset
+
+    def get_autotune_loader(self):
+        loader = copy.copy(self.loader)
+        batch_size = self.loader.batch_sampler.batch_size
+        if isinstance(
+            self.loader.batch_sampler, paddle.io.DistributedBatchSampler
+        ):
+            dataset = self.loader.batch_sampler.dataset
+            sub_dataset = self.get_sub_dataset(dataset, batch_size)
+            loader.batch_sampler = paddle.io.DistributedBatchSampler(
+                dataset=sub_dataset,
+                batch_size=batch_size,
+                num_replicas=self.loader.batch_sampler.nranks,
+                rank=self.loader.batch_sampler.local_rank,
+                shuffle=self.loader.batch_sampler.shuffle,
+                drop_last=self.loader.batch_sampler.drop_last,
+            )
+        elif isinstance(self.loader.batch_sampler, paddle.io.BatchSampler):
+            dataset = self.loader.batch_sampler.sampler.data_source
+            sub_dataset = self.get_sub_dataset(dataset, batch_size)
+            loader.batch_sampler = paddle.io.BatchSampler(
+                dataset=sub_dataset,
+                batch_size=batch_size,
+                drop_last=self.loader.batch_sampler.drop_last,
+            )
+        else:
+            loader = None
+        return loader
+
+    def evaluate_reader_cost(self, reader):
+        costs = []
+        avg_cost = 0
+        start = time.time()
+        for i, data in enumerate(reader):
+            costs.append(time.time() - start)
+            start = time.time()
+        if len(costs) > 2:
+            avg_cost = sum(costs[2:]) / len(costs[2:])
+        else:
+            avg_cost = sum(costs[0:]) / len(costs[0:])
+        return avg_cost
+
+    def is_best(self, reader, best_workers, best_time, num_work_boundary):
+        step = 0
+        num_workers = best_workers + 1
+        boundary = 1
+        while num_workers < num_work_boundary and step < 5:
+            self.loader.num_workers = num_workers
+            time = self.evaluate_reader_cost(reader)
+            logging.debug(
+                "for back num_workers: "
+                + str(num_workers)
+                + " avg_cost: "
+                + str(time)
+            )
+            step += 1
+            if time < best_time * 0.70 * boundary:
+                return num_workers
+            else:
+                num_workers += 1
+            boundary *= 0.80
+        return best_workers
+
+
+class DataLoader:
+    """
+    DataLoader prodives an iterator which iterates given dataset
+    once by the batch_sampler.
+
+    DataLoader supports single-process and multi-prcess data loading,
+    multi-process workers will be used to load data asynchronously if
+    :attr:`num_workers` is set as a positive number.
+
+    DataLoader supports map-style dataset and iterable-style dataset.
+
+    For map-style datast(can get a sample from dataset with a given
+    index), please see :code:`paddle.io.Dataset`.
+
+    For iterable-style datast(get samples from dataset iteratively,
+    like a Python iterator), please see :code:`paddle.io.IterableDataset`.
+
+    For :code:`batch_sampler` please see :code:`paddle.io.BatchSampler`
+
+    .. note::
+        GPU tensor operation is not supported in subprocess currently,
+        please don't use GPU tensor operations in pipeline which will
+        be performed in subprocess, such as dataset transforms, collte_fn,
+        etc. Numpy array and CPU tensor operation is supported.
+
+    **Disable automatic batching**
+
+    In certain cases such as some NLP tasks, instead of automatic batching,
+    handling batching manually in dataset is needed by users. For these
+    cases, automatic batching is disabled if both :attr:`batch_size` and
+    :attr:`batch_sampler` is set as None, each data got from :attr:`dataset`
+    should be batched data and will be processed with function define by
+    :attr:`collate_fn` or :attr:`default_collate_fn`.
+
+
+    .. note::
+        When automatic batching is disabled, :attr:`default_collate_fn` will
+        do nothing to data from dataset.
+
+
+    Args:
+        dataset(Dataset): the dataset to load data from, should be an
+            instance of subclass of :code:`paddle.io.Dataset` or
+            :code:`paddle.io.IterableDataset`.
+        feed_list (list(Tensor)|tuple(Tensor), optional): feed Tensor list.
+            The Tensors should be created by :code:`paddle.static.data()`.
+            :attr:`feed_list` must be set if :attr:`return_list` is
+            False. Default None.
+        places(list(Place)|tuple(Place)|list(str), optional): a list of Place,
+            to put data onto, :attr:`places` can be None, if
+            :attr:`places` is None, default place(CPUPlace or CUDAPlace(0))
+            will be used. Default None. If ``places`` is list of string,
+            the string in the list can be ``cpu``, ``gpu:x`` and ``gpu_pinned``,
+            where ``x`` is the index of the GPUs.
+        return_list (bool, optional): whether the return value on each device is
+            presented as a list. If :attr:`return_list=False`, the return
+            value on each device would be a dict of str -> Tensor, where
+            the key of the dict is the name of each fed Tensors. If
+            :attr:`return_list=True`, the return value on each device would
+            be a list(Tensor). :attr:`return_list` can only be True
+            in dynamic graph mode. Default True.
+        batch_sampler(BatchSampler, optional): an instance of `paddle.io.BatchSampler`
+            to generate batch indices to draw samples from :attr:`dataset`
+            and combine a batch. Default None.
+        batch_size(int|None, optional): sample number in a mini-batch, a substitution
+            parameter for :attr:`batch_sampler`, if :attr:`batch_sampler`
+            is not set, a default `paddle.io.BatchSampler` will be used
+            and initialize by :attr:`batch_size`, :attr:`shuffle` and
+            :attr:`drop_last`. Default 1.
+        shuffle(bool, optional): whther to shuffle indices order before genrate
+            batch indices, a substitution parameter for :attr:`batch_sampler`
+            see :attr:`batch_size`. Default False.
+        drop_last(bool, optional): whether drop the last incomplete batch dataset size
+            is not divisible by the batch size, a substitution parameter
+            for :attr:`batch_sampler`, see :attr:`batch_size`. Default False
+        collate_fn(callable, optional): function to generate mini-batch data by merging
+            the sample list, None for only stack each fields of sample in axis
+            0(same as :attr::`np.stack(..., axis=0)`). Default None
+        num_workers(int, optional): the number of subprocess to load data, 0 for no
+            subprocess used and loading data in main process. Default 0
+        use_buffer_reader (bool, optional): whether to use bufferred reader.
+            If use_buffer_reader=True, the DataLoader would prefetch
+            batch data asynchronously, so it would speed up data feeding
+            and occupies a little more CPU or GPU memory, i.e., the memory
+            of one batch input data. Default True.
+        prefetch_factor (int, optional): Number of batch data the DataLoader would prefetch
+            if use_buffer_reader=True. Default 2.
+        use_shared_memory (bool, optional): whether to use shared memory to speed up
+            putting data into inter-process queue, set :attr:`use_shared_memory`
+            as True only when the shared memory space on your machine(e.g.
+            space of '/dev/shm' on Linux operating sysytem) is large enough.
+            Shared memory will only be enabled in multi-process mode(num_workers
+            > 0). Default True.
+        timeout(int, optional): the timeout value for getting data form output queue
+            of subprocesses. Default 0.
+        worker_init_fn(callable, optional): init function which will be called with
+            worker id on each subproces starting if not set as None. Default
+            None.
+
+    Returns:
+        DataLoader: an iterable object for data iterating, each elemnet of the generated data is a Tensor.
+
+    Examples:
+
+        .. code-block:: python
+
+            import numpy as np
+
+            import paddle
+            import paddle.nn as nn
+            import paddle.nn.functional as F
+            from paddle.io import Dataset, BatchSampler, DataLoader
+
+            BATCH_NUM = 20
+            BATCH_SIZE = 16
+            EPOCH_NUM = 4
+
+            IMAGE_SIZE = 784
+            CLASS_NUM = 10
+
+            # define a random dataset
+            class RandomDataset(Dataset):
+                def __init__(self, num_samples):
+                    self.num_samples = num_samples
+
+                def __getitem__(self, idx):
+                    image = np.random.random([IMAGE_SIZE]).astype('float32')
+                    label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64')
+                    return image, label
+
+                def __len__(self):
+                    return self.num_samples
+
+            dataset = RandomDataset(BATCH_NUM * BATCH_SIZE)
+
+            class SimpleNet(nn.Layer):
+                def __init__(self):
+                    super().__init__()
+                    self.fc = nn.Linear(IMAGE_SIZE, CLASS_NUM)
+
+                def forward(self, image, label=None):
+                    return self.fc(image)
+
+            simple_net = SimpleNet()
+            opt = paddle.optimizer.SGD(learning_rate=1e-3,
+                                      parameters=simple_net.parameters())
+
+            loader = DataLoader(dataset,
+                                batch_size=BATCH_SIZE,
+                                shuffle=True,
+                                drop_last=True,
+                                num_workers=2)
+
+            for e in range(EPOCH_NUM):
+                for i, (image, label) in enumerate(loader()):
+                    out = simple_net(image)
+                    loss = F.cross_entropy(out, label)
+                    avg_loss = paddle.mean(loss)
+                    avg_loss.backward()
+                    opt.minimize(avg_loss)
+                    simple_net.clear_gradients()
+                    print("Epoch {} batch {}: loss = {}".format(e, i, np.mean(loss.numpy())))
+
+
+    .. note::
+        For reading iterable dataset with multiprocess Dataloader,
+        please see :code:`paddle.io.IterableDataset`
+
+    """
+
+    def __init__(
+        self,
+        dataset,
+        feed_list=None,
+        places=None,
+        return_list=True,
+        batch_sampler=None,
+        batch_size=1,
+        shuffle=False,
+        drop_last=False,
+        collate_fn=None,
+        num_workers=0,
+        use_buffer_reader=True,
+        prefetch_factor=2,
+        use_shared_memory=True,
+        timeout=0,
+        worker_init_fn=None,
+        persistent_workers=False,
+    ):
+        self.return_list = return_list
+        self.collate_fn = collate_fn
+        self.use_buffer_reader = use_buffer_reader
+        self.prefetch_factor = prefetch_factor
+        self.worker_init_fn = worker_init_fn
+
+        self.dataset = dataset
+
+        if not return_list and not _non_static_mode():
+            assert (
+                feed_list is not None
+            ), "feed_list should be set when return_list=False"
+        self.feed_list = feed_list
+
+        if places is None:
+            places = _current_expected_place()
+        if isinstance(places, (list, tuple)):
+            places = _get_paddle_place_list(places)
+        else:
+            places = _get_paddle_place(places)
+        self.places = _convert_places(places)
+
+        assert num_workers >= 0, "num_workers should be a non-negative value"
+        if num_workers > 0 and (
+            sys.platform == 'darwin' or sys.platform == 'win32'
+        ):
+            warnings.warn(
+                "DataLoader with multi-process mode is not supported on MacOs and Windows currently."
+                " Please use signle-process mode with num_workers = 0 instead"
+            )
+            num_workers = 0
+        self.num_workers = num_workers
+
+        assert prefetch_factor > 0, "prefetch_factor should be a positive value"
+
+        self.use_shared_memory = use_shared_memory
+        if use_shared_memory and num_workers == 0:
+            self.use_shared_memory = False
+
+        assert timeout >= 0, "timeout should be a non-negative value"
+        self.timeout = timeout
+
+        if isinstance(dataset, IterableDataset):
+            self.dataset_kind = _DatasetKind.ITER
+            if shuffle:
+                raise ValueError(
+                    "IterableDataset not support shuffle, but got shuffle={}".format(
+                        shuffle
+                    )
+                )
+            if batch_sampler is not None:
+                raise ValueError(
+                    "IterableDataset expect unspecified batch_sampler"
+                )
+        else:
+            self.dataset_kind = _DatasetKind.MAP
+
+        if batch_sampler is not None:
+            assert batch_size == 1 and not shuffle and not drop_last, (
+                "batch_size/shuffle/drop_last should not be set when "
+                "batch_sampler is given"
+            )
+            self.batch_sampler = batch_sampler
+            self.batch_size = None
+        elif batch_size is None:
+            self.batch_sampler = None
+            self.batch_size = None
+        else:
+            assert batch_size > 0, (
+                "batch_size should be None or a positive value when "
+                "batch_sampler is not given"
+            )
+            self.batch_size = batch_size
+            if isinstance(dataset, IterableDataset):
+                self.batch_sampler = _InfiniteIterableSampler(
+                    dataset, batch_size
+                )
+            else:
+                self.batch_sampler = BatchSampler(
+                    dataset=dataset,
+                    batch_size=batch_size,
+                    shuffle=shuffle,
+                    drop_last=drop_last,
+                )
+
+        self.drop_last = drop_last
+        self.auto_collate_batch = self.batch_sampler is not None
+
+        self.pin_memory = False
+        if _non_static_mode():
+            self.pin_memory = (
+                True if use_pinned_memory() is None else use_pinned_memory()
+            )
+
+        self._persistent_workers = persistent_workers
+        self._iterator = None
+        self.num_workers = AuToTune(self).__call__()
+
+    def __len__(self):
+        if self.dataset_kind == _DatasetKind.ITER:
+            raise ValueError("length of IterableDataset not supported")
+        else:
+            if self.auto_collate_batch:
+                return len(self.batch_sampler)
+            else:
+                return len(self.dataset)
+
+    def __iter__(self):
+        if self.num_workers == 0:
+            return _DataLoaderIterSingleProcess(self)
+        elif self._persistent_workers:
+            if self._iterator is None:
+                self._iterator = _DataLoaderIterMultiProcess(self)
+            else:
+                self._iterator._reset()
+            return self._iterator
+        else:
+            return _DataLoaderIterMultiProcess(self)
+
+    def __call__(self):
+        return self.__iter__()
diff --git a/python/paddle/static/quantization/post_training_quantization.py b/python/paddle/static/quantization/post_training_quantization.py
index f80b293a12d4b..266c1756a334b 100644
--- a/python/paddle/static/quantization/post_training_quantization.py
+++ b/python/paddle/static/quantization/post_training_quantization.py
@@ -620,7 +620,7 @@ def _load_model_data(self):
                 self._batch_nums if self._batch_nums else len(self._data_loader)
             )
             return
-        self._data_loader = io.DataLoader.from_generator(
+        self._data_loader = reader.DataLoader.from_generator(
             feed_list=feed_vars, capacity=3 * self._batch_size, iterable=True
         )
         if self._sample_generator is not None:
diff --git a/python/setup.py.in b/python/setup.py.in
index 1c59c4aaa4746..ebf949c2b41f3 100644
--- a/python/setup.py.in
+++ b/python/setup.py.in
@@ -445,7 +445,6 @@ packages=['paddle',
           'paddle.fluid.proto',
           'paddle.fluid.proto.profiler',
           'paddle.fluid.layers',
-          'paddle.fluid.dataloader',
           'paddle.fluid.contrib',
           'paddle.fluid.contrib.extend_optimizer',
           'paddle.fluid.incubate',
@@ -492,6 +491,7 @@ packages=['paddle',
           'paddle.sparse.nn.functional',
           'paddle.incubate.xpu',
           'paddle.io',
+          'paddle.io.dataloader',
           'paddle.optimizer',
           'paddle.nn',
           'paddle.nn.functional',
diff --git a/setup.py b/setup.py
index 6a305243bbe3d..297e66eba0499 100644
--- a/setup.py
+++ b/setup.py
@@ -1421,7 +1421,6 @@ def get_setup_parameters():
         'paddle.fluid.proto',
         'paddle.fluid.proto.profiler',
         'paddle.fluid.layers',
-        'paddle.fluid.dataloader',
         'paddle.fluid.contrib',
         'paddle.fluid.contrib.extend_optimizer',
         'paddle.fluid.incubate',
@@ -1468,6 +1467,7 @@ def get_setup_parameters():
         'paddle.sparse.nn.functional',
         'paddle.incubate.xpu',
         'paddle.io',
+        'paddle.io.dataloader',
         'paddle.optimizer',
         'paddle.nn',
         'paddle.nn.functional',
diff --git a/test/auto_parallel/auto_parallel_relaunch_model.py b/test/auto_parallel/auto_parallel_relaunch_model.py
index 290af66485512..6fa3bc9eaa1ff 100644
--- a/test/auto_parallel/auto_parallel_relaunch_model.py
+++ b/test/auto_parallel/auto_parallel_relaunch_model.py
@@ -109,7 +109,7 @@ def mlp_pretrain_forward(train_program, start_program):
         error_cost = paddle.nn.functional.square_error_cost(predict, label)
         loss = paddle.mean(error_cost)
 
-        loader = paddle.io.DataLoader.from_generator(
+        loader = paddle.fluid.io.DataLoader.from_generator(
             feed_list=[input, label], capacity=4 * batch_size, iterable=True
         )
 
diff --git a/test/auto_parallel/engine_api.py b/test/auto_parallel/engine_api.py
index cb0b4f2541e94..a2725a57b8e53 100644
--- a/test/auto_parallel/engine_api.py
+++ b/test/auto_parallel/engine_api.py
@@ -297,7 +297,7 @@ def train_builtin_data_vars():
     with static.program_guard(engine.main_program, engine.startup_program):
         feed_list = engine.inputs + engine.labels
         print(feed_list)
-        loader = paddle.io.DataLoader.from_generator(
+        loader = paddle.fluid.io.DataLoader.from_generator(
             feed_list=feed_list, capacity=4 * batch_size, iterable=False
         )
 
@@ -324,7 +324,7 @@ def train_non_builtin_data_vars():
         )
         label = static.data(name="label", shape=[batch_size, 1], dtype='int64')
 
-        loader = paddle.io.DataLoader.from_generator(
+        loader = paddle.fluid.io.DataLoader.from_generator(
             feed_list=[input, label], capacity=4 * batch_size, iterable=False
         )
         places = static.cuda_places()
@@ -383,7 +383,7 @@ def get_cost():
         )
         label = static.data(name="label", shape=[batch_size, 1], dtype='int64')
 
-        loader = paddle.io.DataLoader.from_generator(
+        loader = paddle.fluid.io.DataLoader.from_generator(
             feed_list=[input, label], capacity=4 * batch_size, iterable=False
         )
         places = static.cuda_places()
@@ -434,7 +434,7 @@ def get_cost_by_default_program():
         )
         label = static.data(name="label", shape=[batch_size, 1], dtype='int64')
 
-        loader = paddle.io.DataLoader.from_generator(
+        loader = paddle.fluid.io.DataLoader.from_generator(
             feed_list=[input, label], capacity=4 * batch_size, iterable=False
         )
         places = static.cuda_places()
diff --git a/test/auto_parallel/test_dist_attr_v2.py b/test/auto_parallel/test_dist_attr_v2.py
index 11c140a812a9f..1d15c34221f90 100644
--- a/test/auto_parallel/test_dist_attr_v2.py
+++ b/test/auto_parallel/test_dist_attr_v2.py
@@ -130,7 +130,7 @@ def get_program():
         )
         data_holder = [input, label]
         # dataloader
-        dataloader = paddle.io.DataLoader.from_generator(
+        dataloader = paddle.fluid.io.DataLoader.from_generator(
             feed_list=data_holder, capacity=4 * batch_size, iterable=False
         )
         dataloader.set_batch_generator(
diff --git a/test/auto_parallel/test_dist_context.py b/test/auto_parallel/test_dist_context.py
index 10f78aedd4fb9..2944b2db2a3fb 100644
--- a/test/auto_parallel/test_dist_context.py
+++ b/test/auto_parallel/test_dist_context.py
@@ -112,7 +112,7 @@ def get_program():
         )
         data_holder = [input, label]
         # dataloader
-        dataloader = paddle.io.DataLoader.from_generator(
+        dataloader = paddle.fluid.io.DataLoader.from_generator(
             feed_list=data_holder, capacity=4 * batch_size, iterable=False
         )
         dataloader.set_batch_generator(
diff --git a/test/auto_parallel/test_serialization.py b/test/auto_parallel/test_serialization.py
index 00a30e8a61d4e..d89c9596f4cdb 100644
--- a/test/auto_parallel/test_serialization.py
+++ b/test/auto_parallel/test_serialization.py
@@ -124,7 +124,7 @@ def get_program():
         )
         data_holder = [input, label]
         # dataloader
-        dataloader = paddle.io.DataLoader.from_generator(
+        dataloader = paddle.fluid.io.DataLoader.from_generator(
             feed_list=data_holder, capacity=4 * batch_size, iterable=False
         )
         dataloader.set_batch_generator(
diff --git a/test/auto_parallel/test_while_op_completion.py b/test/auto_parallel/test_while_op_completion.py
index 6d5264ab971b7..3f9b5b151ab08 100644
--- a/test/auto_parallel/test_while_op_completion.py
+++ b/test/auto_parallel/test_while_op_completion.py
@@ -148,7 +148,7 @@ def get_program():
         )
         data_holder = [input, label]
         # dataloader
-        dataloader = paddle.io.DataLoader.from_generator(
+        dataloader = paddle.fluid.io.DataLoader.from_generator(
             feed_list=data_holder, capacity=4 * batch_size, iterable=False
         )
         dataloader.set_batch_generator(
diff --git a/test/auto_parallel/test_while_op_partition.py b/test/auto_parallel/test_while_op_partition.py
index 6dc02d6834f5b..00f3a70bbcf42 100644
--- a/test/auto_parallel/test_while_op_partition.py
+++ b/test/auto_parallel/test_while_op_partition.py
@@ -136,7 +136,7 @@ def get_program():
 
         data_holder = [input, label]
         # dataloader
-        dataloader = paddle.io.DataLoader.from_generator(
+        dataloader = fluid.io.DataLoader.from_generator(
             feed_list=data_holder, capacity=4 * batch_size, iterable=False
         )
         dataloader.set_batch_generator(
diff --git a/test/dygraph_to_static/test_resnet_v2.py b/test/dygraph_to_static/test_resnet_v2.py
index bf332809ff8f0..2efbe46cedfec 100644
--- a/test/dygraph_to_static/test_resnet_v2.py
+++ b/test/dygraph_to_static/test_resnet_v2.py
@@ -255,7 +255,7 @@ def do_train(self, to_static):
             batch_size=batch_size,
             drop_last=True,
         )
-        data_loader = paddle.io.DataLoader.from_generator(
+        data_loader = paddle.fluid.io.DataLoader.from_generator(
             capacity=5, iterable=True
         )
         data_loader.set_sample_list_generator(train_reader)
diff --git a/test/dygraph_to_static/test_simnet_v2.py b/test/dygraph_to_static/test_simnet_v2.py
index 3e8cb4c10b3d4..a86259cc6d736 100644
--- a/test/dygraph_to_static/test_simnet_v2.py
+++ b/test/dygraph_to_static/test_simnet_v2.py
@@ -132,7 +132,7 @@ def train(conf_dict, to_static):
     global_step = 0
     losses = []
 
-    train_loader = paddle.io.DataLoader.from_generator(
+    train_loader = paddle.fluid.io.DataLoader.from_generator(
         capacity=16, return_list=True, iterable=True, use_double_buffer=True
     )
     get_train_examples = simnet_process.get_reader("train", epoch=args.epoch)
diff --git a/test/legacy_test/test_model.py b/test/legacy_test/test_model.py
index af3cb7bdefb74..c5d62761f7f68 100644
--- a/test/legacy_test/test_model.py
+++ b/test/legacy_test/test_model.py
@@ -199,13 +199,13 @@ def setUpClass(cls):
             mode='test', return_label=False, sample_num=sp_num
         )
 
-        cls.train_loader = fluid.io.DataLoader(
+        cls.train_loader = paddle.io.DataLoader(
             cls.train_dataset, places=cls.device, batch_size=64
         )
-        cls.val_loader = fluid.io.DataLoader(
+        cls.val_loader = paddle.io.DataLoader(
             cls.val_dataset, places=cls.device, batch_size=64
         )
-        cls.test_loader = fluid.io.DataLoader(
+        cls.test_loader = paddle.io.DataLoader(
             cls.test_dataset, places=cls.device, batch_size=64
         )
 
@@ -322,14 +322,14 @@ def fit(self, dynamic, num_replicas=None, rank=None, num_iters=None):
             rank=rank,
         )
 
-        train_loader = fluid.io.DataLoader(
+        train_loader = paddle.io.DataLoader(
             self.train_dataset,
             batch_sampler=train_sampler,
             places=self.device,
             return_list=True,
         )
 
-        val_loader = fluid.io.DataLoader(
+        val_loader = paddle.io.DataLoader(
             self.val_dataset,
             batch_sampler=val_sampler,
             places=self.device,
@@ -375,14 +375,14 @@ def fit_with_tuple_input(self, dynamic, num_replicas=None, rank=None):
             rank=rank,
         )
 
-        train_loader = fluid.io.DataLoader(
+        train_loader = paddle.io.DataLoader(
             self.train_dataset,
             batch_sampler=train_sampler,
             places=self.device,
             return_list=True,
         )
 
-        val_loader = fluid.io.DataLoader(
+        val_loader = paddle.io.DataLoader(
             self.val_dataset,
             batch_sampler=val_sampler,
             places=self.device,
@@ -404,7 +404,7 @@ def evaluate(self, dynamic):
             self.val_dataset, batch_size=64, shuffle=False
         )
 
-        val_loader = fluid.io.DataLoader(
+        val_loader = paddle.io.DataLoader(
             self.val_dataset,
             batch_sampler=sampler,
             places=self.device,
@@ -432,7 +432,7 @@ def predict(self, dynamic):
             self.test_dataset, batch_size=64, shuffle=False
         )
 
-        test_loader = fluid.io.DataLoader(
+        test_loader = paddle.io.DataLoader(
             self.test_dataset,
             batch_sampler=sampler,
             places=self.device,

From 2f56b6dac106bc59c5154c327a43eaa035938c14 Mon Sep 17 00:00:00 2001
From: ronnywang <ronny1996@163.com>
Date: Thu, 11 May 2023 18:45:00 +0800
Subject: [PATCH 281/405] [CustomDevice] fix recompute (#53718)

---
 paddle/fluid/distributed/collective/process_group_custom.cc | 6 ++++--
 .../paddle/distributed/fleet/recompute/recompute_hybrid.py  | 5 ++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/paddle/fluid/distributed/collective/process_group_custom.cc b/paddle/fluid/distributed/collective/process_group_custom.cc
index b6c7063fd6fb7..1e4d1df337bdb 100644
--- a/paddle/fluid/distributed/collective/process_group_custom.cc
+++ b/paddle/fluid/distributed/collective/process_group_custom.cc
@@ -125,12 +125,14 @@ void ProcessGroupCustom::BroadcastUniqueCustomID(
     std::vector<phi::ccl::CCLRootId>& ccl_ids) {  // NOLINT
   if (rank_ == 0) {
     for (size_t i = 0; i < ccl_ids.size(); i++) {
-      auto key = "ProcessGroupCustom/ccl_ids/" + std::to_string(i);
+      auto key = "ProcessGroupCustom/ccl_ids/" + std::to_string(gid_) + "/" +
+                 std::to_string(i);
       store_->set(key, ccl_ids[i]);
     }
   } else {
     for (size_t i = 0; i < ccl_ids.size(); i++) {
-      auto key = "ProcessGroupCustom/ccl_ids/" + std::to_string(i);
+      auto key = "ProcessGroupCustom/ccl_ids/" + std::to_string(gid_) + "/" +
+                 std::to_string(i);
       ccl_ids[i] = store_->get(key);
     }
   }
diff --git a/python/paddle/distributed/fleet/recompute/recompute_hybrid.py b/python/paddle/distributed/fleet/recompute/recompute_hybrid.py
index c6acae878745b..a5689020eb009 100644
--- a/python/paddle/distributed/fleet/recompute/recompute_hybrid.py
+++ b/python/paddle/distributed/fleet/recompute/recompute_hybrid.py
@@ -110,7 +110,10 @@ def forward(
 
         cur_device = paddle.get_device()
         assert (
-            'gpu:' in paddle.get_device() or 'xpu:' in paddle.get_device()
+            'gpu:' in paddle.get_device()
+            or 'xpu:' in paddle.get_device()
+            or cur_device.split(':')[0]
+            in paddle.device.get_all_custom_device_type()
         ), "Recompute with RNG is not support current device: {}.".format(
             cur_device
         )

From dbb6269207c813e7192026a0aabdbada74c2620b Mon Sep 17 00:00:00 2001
From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com>
Date: Thu, 11 May 2023 19:01:46 +0800
Subject: [PATCH 282/405] remove some [-Wunused-parameter] warning (#53683)

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop
---
 paddle/fluid/operators/collective/global_gather_op.h   |  2 +-
 paddle/fluid/operators/collective/global_scatter_op.h  |  2 +-
 .../fluid/operators/collective/partial_allgather_op.h  |  2 +-
 paddle/fluid/operators/collective/partial_recv_op.h    |  2 +-
 paddle/fluid/operators/collective/partial_send_op.h    |  2 +-
 paddle/fluid/operators/collective/recv_v2_op.h         |  2 +-
 paddle/phi/kernels/impl/eigvalsh_grad_kernel_impl.h    |  4 ++--
 paddle/phi/kernels/impl/einsum_impl.h                  |  2 +-
 paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h | 10 ++++++----
 paddle/phi/kernels/impl/lamb_kernel_impl.h             |  2 +-
 paddle/phi/kernels/impl/lu_kernel_impl.h               |  2 +-
 paddle/phi/kernels/impl/unstack_kernel_impl.h          |  2 +-
 paddle/phi/kernels/onednn/conv_handler.h               |  6 +++---
 paddle/phi/kernels/onednn/pool_kernel.cc               |  2 +-
 paddle/phi/kernels/onednn/reduce_kernel_impl.h         |  2 +-
 paddle/phi/kernels/onednn/slice_kernel.cc              |  2 +-
 .../phi/kernels/selected_rows/impl/lamb_kernel_impl.h  |  2 +-
 paddle/phi/kernels/sparse/cpu/conv_grad_kernel.cc      |  8 ++++----
 paddle/phi/kernels/sparse/cpu/conv_kernel.cc           |  2 +-
 19 files changed, 30 insertions(+), 28 deletions(-)

diff --git a/paddle/fluid/operators/collective/global_gather_op.h b/paddle/fluid/operators/collective/global_gather_op.h
index 0d3b4ed92e9b2..723c5e48a5ae4 100644
--- a/paddle/fluid/operators/collective/global_gather_op.h
+++ b/paddle/fluid/operators/collective/global_gather_op.h
@@ -28,7 +28,7 @@ namespace operators {
 template <typename T, typename DeviceContext>
 class GlobalGatherOpCPUKernel : public framework::OpKernel<T> {
  public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
+  void Compute(const framework::ExecutionContext& ctx UNUSED) const override {
     PADDLE_THROW(platform::errors::Unavailable(
         "Do not support global gather op for cpu kernel now."));
   }
diff --git a/paddle/fluid/operators/collective/global_scatter_op.h b/paddle/fluid/operators/collective/global_scatter_op.h
index 3cb2a3c7fc41b..fc4b48500c071 100644
--- a/paddle/fluid/operators/collective/global_scatter_op.h
+++ b/paddle/fluid/operators/collective/global_scatter_op.h
@@ -28,7 +28,7 @@ namespace operators {
 template <typename T, typename DeviceContext>
 class GlobalScatterOpCPUKernel : public framework::OpKernel<T> {
  public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
+  void Compute(const framework::ExecutionContext& ctx UNUSED) const override {
     PADDLE_THROW(platform::errors::Unavailable(
         "Do not support global scatter op for cpu kernel now."));
   }
diff --git a/paddle/fluid/operators/collective/partial_allgather_op.h b/paddle/fluid/operators/collective/partial_allgather_op.h
index 6b827a2656f29..815558d0227eb 100644
--- a/paddle/fluid/operators/collective/partial_allgather_op.h
+++ b/paddle/fluid/operators/collective/partial_allgather_op.h
@@ -29,7 +29,7 @@ namespace operators {
 template <typename T, typename DeviceContext>
 class PartialAllGatherOpCPUKernel : public framework::OpKernel<T> {
  public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
+  void Compute(const framework::ExecutionContext& ctx UNUSED) const override {
     PADDLE_THROW(platform::errors::Unavailable(
         "Do not support partial_allgather for cpu kernel now."));
   }
diff --git a/paddle/fluid/operators/collective/partial_recv_op.h b/paddle/fluid/operators/collective/partial_recv_op.h
index fdf3f02b0d679..baf47ef9dff8d 100644
--- a/paddle/fluid/operators/collective/partial_recv_op.h
+++ b/paddle/fluid/operators/collective/partial_recv_op.h
@@ -27,7 +27,7 @@ namespace operators {
 template <typename T, typename DeviceContext>
 class PartialRecvOpCPUKernel : public framework::OpKernel<T> {
  public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
+  void Compute(const framework::ExecutionContext& ctx UNUSED) const override {
     PADDLE_THROW(platform::errors::Unavailable(
         "Do not support partial_recv for cpu kernel now."));
   }
diff --git a/paddle/fluid/operators/collective/partial_send_op.h b/paddle/fluid/operators/collective/partial_send_op.h
index 773125be7d40f..b7b72789b87ff 100644
--- a/paddle/fluid/operators/collective/partial_send_op.h
+++ b/paddle/fluid/operators/collective/partial_send_op.h
@@ -28,7 +28,7 @@ namespace operators {
 template <typename T, typename DeviceContext>
 class PartialSendOpCPUKernel : public framework::OpKernel<T> {
  public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
+  void Compute(const framework::ExecutionContext& ctx UNUSED) const override {
     PADDLE_THROW(platform::errors::Unavailable(
         "Do not support partial_send for cpu kernel now."));
   }
diff --git a/paddle/fluid/operators/collective/recv_v2_op.h b/paddle/fluid/operators/collective/recv_v2_op.h
index 3430cdb73aa1a..e76e4a7b55197 100644
--- a/paddle/fluid/operators/collective/recv_v2_op.h
+++ b/paddle/fluid/operators/collective/recv_v2_op.h
@@ -27,7 +27,7 @@ namespace operators {
 template <typename T, typename DeviceContext>
 class RecvOpV2CPUKernel : public framework::OpKernel<T> {
  public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
+  void Compute(const framework::ExecutionContext& ctx UNUSED) const override {
     PADDLE_THROW(platform::errors::Unavailable(
         "Do not support recv for cpu kernel now."));
   }
diff --git a/paddle/phi/kernels/impl/eigvalsh_grad_kernel_impl.h b/paddle/phi/kernels/impl/eigvalsh_grad_kernel_impl.h
index 7248985bf294c..18a6a0518a3cc 100644
--- a/paddle/phi/kernels/impl/eigvalsh_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/eigvalsh_grad_kernel_impl.h
@@ -27,8 +27,8 @@ template <typename T, typename Context>
 void EigvalshGradKernel(const Context& dev_ctx,
                         const DenseTensor& out_v,
                         const DenseTensor& out_w_grad,
-                        const std::string& uplo,
-                        bool is_test,
+                        const std::string& uplo UNUSED,
+                        bool is_test UNUSED,
                         DenseTensor* x_grad) {
   auto tV = phi::TransposeLast2Dim<T>(dev_ctx, phi::Conj<T>(dev_ctx, out_v));
 
diff --git a/paddle/phi/kernels/impl/einsum_impl.h b/paddle/phi/kernels/impl/einsum_impl.h
index 400334ad4e04e..92a4f99c6eb77 100644
--- a/paddle/phi/kernels/impl/einsum_impl.h
+++ b/paddle/phi/kernels/impl/einsum_impl.h
@@ -752,7 +752,7 @@ void EinsumKernel(const Context& dev_ctx,
                   const std::string& equation,
                   DenseTensor* out,
                   std::vector<DenseTensor*> cache,
-                  std::vector<DenseTensor*> xshape) {
+                  std::vector<DenseTensor*> xshape UNUSED) {
   std::vector<char> tmp;
   // for the sake of compatibility, we may load and run v2.3 EinsumOp. Output
   // may have nullptr and the cache.size() is not equal to inputs.size(). refer
diff --git a/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h b/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h
index 15f99a58fa5a5..3ce1e721b968e 100644
--- a/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h
@@ -119,7 +119,9 @@ void SubtractDoubleGradImpl(const Context& dev_ctx,
 
 template <typename T>
 struct DivGradDX {
-  HOSTDEVICE T operator()(T x, T y, T out, T dout) const { return dout / y; }
+  HOSTDEVICE T operator()(T x UNUSED, T y, T out UNUSED, T dout) const {
+    return dout / y;
+  }
 };
 
 template <typename T>
@@ -136,7 +138,7 @@ struct DivGradDX<phi::dtype::complex<T>> {
 
 template <typename T>
 struct DivGradDY {
-  HOSTDEVICE T operator()(T x, T y, T out, T dout) const {
+  HOSTDEVICE T operator()(T x UNUSED, T y, T out, T dout) const {
     return -dout * out / y;
   }
 };
@@ -857,14 +859,14 @@ struct MinGradDy {
 
 template <typename T>
 struct HeavisideGradDx {
-  HOSTDEVICE T operator()(T x, T y, T out, T dout) const {
+  HOSTDEVICE T operator()(T x UNUSED, T y UNUSED, T out UNUSED, T dout) const {
     return dout * static_cast<T>(0);
   }
 };
 
 template <typename T>
 struct HeavisideGradDy {
-  HOSTDEVICE T operator()(T x, T y, T out, T dout) const {
+  HOSTDEVICE T operator()(T x, T y UNUSED, T out UNUSED, T dout) const {
     return dout * static_cast<T>(x == static_cast<T>(0));
   }
 };
diff --git a/paddle/phi/kernels/impl/lamb_kernel_impl.h b/paddle/phi/kernels/impl/lamb_kernel_impl.h
index e0850b8aef0d9..b02d2a517a1c5 100644
--- a/paddle/phi/kernels/impl/lamb_kernel_impl.h
+++ b/paddle/phi/kernels/impl/lamb_kernel_impl.h
@@ -128,7 +128,7 @@ void ComputeImpl(const Context& dev_ctx,
                  float beta1_f,
                  float beta2_f,
                  float epsilon_f,
-                 bool multi_precision,
+                 bool multi_precision UNUSED,
                  DenseTensor* param_out,
                  DenseTensor* mom1_out,
                  DenseTensor* mom2_out,
diff --git a/paddle/phi/kernels/impl/lu_kernel_impl.h b/paddle/phi/kernels/impl/lu_kernel_impl.h
index 5663484362a8e..e9ba46d0c162c 100644
--- a/paddle/phi/kernels/impl/lu_kernel_impl.h
+++ b/paddle/phi/kernels/impl/lu_kernel_impl.h
@@ -474,7 +474,7 @@ void Unpack_Pivot(const Context& dev_ctx,
                   const DenseTensor& Pivot,
                   DenseTensor* P,
                   int h,
-                  int w) {
+                  int w UNUSED) {
   auto dims = Pivot.dims();
   auto Pdimvec = vectorize(dims);
   auto prank = Pdimvec.size();
diff --git a/paddle/phi/kernels/impl/unstack_kernel_impl.h b/paddle/phi/kernels/impl/unstack_kernel_impl.h
index 030f4a62c6e00..102126a1e3307 100644
--- a/paddle/phi/kernels/impl/unstack_kernel_impl.h
+++ b/paddle/phi/kernels/impl/unstack_kernel_impl.h
@@ -26,7 +26,7 @@ template <typename T, typename Context>
 void UnStackKernel(const Context &dev_ctx,
                    const DenseTensor &x,
                    int axis,
-                   int num,
+                   int num UNUSED,
                    std::vector<DenseTensor *> outs) {
   auto *dy = &x;
   auto dx = outs;
diff --git a/paddle/phi/kernels/onednn/conv_handler.h b/paddle/phi/kernels/onednn/conv_handler.h
index 2be0ba5649711..fd6cf0c577849 100644
--- a/paddle/phi/kernels/onednn/conv_handler.h
+++ b/paddle/phi/kernels/onednn/conv_handler.h
@@ -240,10 +240,10 @@ class ConvOneDNNHandlerT
                      const std::string& padding_algorithm,
                      const std::vector<int>& dilations_in,
                      int groups,
-                     const std::string& data_format,
+                     const std::string& data_format UNUSED,
                      bool is_test,
-                     phi::DenseTensor* filter_grad,
-                     phi::DenseTensor* in_x_grad,
+                     phi::DenseTensor* filter_grad UNUSED,
+                     phi::DenseTensor* in_x_grad UNUSED,
                      const std::string& unique_name)
       : funcs::OneDNNHandlerT<T,
                               dnnl::convolution_forward,
diff --git a/paddle/phi/kernels/onednn/pool_kernel.cc b/paddle/phi/kernels/onednn/pool_kernel.cc
index 9a4963eefeba7..726014396b124 100644
--- a/paddle/phi/kernels/onednn/pool_kernel.cc
+++ b/paddle/phi/kernels/onednn/pool_kernel.cc
@@ -26,7 +26,7 @@ void Pool2dKernel(const Context& dev_ctx,
                   const std::vector<int>& paddings,
                   bool ceil_mode,
                   bool exclusive,
-                  const std::string& data_format,
+                  const std::string& data_format UNUSED,
                   const std::string& pooling_type,
                   bool global_pooling,
                   bool adaptive,
diff --git a/paddle/phi/kernels/onednn/reduce_kernel_impl.h b/paddle/phi/kernels/onednn/reduce_kernel_impl.h
index 69f667c36624b..7c512c6e3eb4e 100644
--- a/paddle/phi/kernels/onednn/reduce_kernel_impl.h
+++ b/paddle/phi/kernels/onednn/reduce_kernel_impl.h
@@ -118,7 +118,7 @@ void ReduceGradKernel(const Context& dev_ctx,
                       bool reduce_all,
                       DenseTensor* x_grad,
                       dnnl::algorithm binary_type,
-                      dnnl::algorithm reduction_type,
+                      dnnl::algorithm reduction_type UNUSED,
                       float scale_x,
                       float scale_y) {
   reduce_all = recompute_reduce_all(x, dims, reduce_all);
diff --git a/paddle/phi/kernels/onednn/slice_kernel.cc b/paddle/phi/kernels/onednn/slice_kernel.cc
index 6c927018264bf..dfda78de77c06 100644
--- a/paddle/phi/kernels/onednn/slice_kernel.cc
+++ b/paddle/phi/kernels/onednn/slice_kernel.cc
@@ -25,7 +25,7 @@ void SliceKernel(const Context& dev_ctx,
                  const std::vector<int64_t>& axes,
                  const IntArray& starts,
                  const IntArray& ends,
-                 const std::vector<int64_t>& infer_flags,
+                 const std::vector<int64_t>& infer_flags UNUSED,
                  const std::vector<int64_t>& decrease_axis,
                  DenseTensor* out) {
   const auto& onednn_engine = dev_ctx.GetEngine();
diff --git a/paddle/phi/kernels/selected_rows/impl/lamb_kernel_impl.h b/paddle/phi/kernels/selected_rows/impl/lamb_kernel_impl.h
index 4323a23e0e60c..f9c620c925689 100644
--- a/paddle/phi/kernels/selected_rows/impl/lamb_kernel_impl.h
+++ b/paddle/phi/kernels/selected_rows/impl/lamb_kernel_impl.h
@@ -131,7 +131,7 @@ void ComputeRowImpl(const Context& dev_ctx,
                     float beta1_f,
                     float beta2_f,
                     float epsilon_f,
-                    bool multi_precision,
+                    bool multi_precision UNUSED,
                     DenseTensor* param_out,
                     DenseTensor* mom1_out,
                     DenseTensor* mom2_out,
diff --git a/paddle/phi/kernels/sparse/cpu/conv_grad_kernel.cc b/paddle/phi/kernels/sparse/cpu/conv_grad_kernel.cc
index fe8b8ad6f8166..09c307a5c005f 100644
--- a/paddle/phi/kernels/sparse/cpu/conv_grad_kernel.cc
+++ b/paddle/phi/kernels/sparse/cpu/conv_grad_kernel.cc
@@ -38,10 +38,10 @@ void Conv3dCooGradCPUKernel(const CPUContext& dev_ctx,
                             const DenseTensor& rulebook,
                             const DenseTensor& counter,
                             const SparseCooTensor& out_grad,
-                            const std::vector<int>& paddings,
-                            const std::vector<int>& dilations,
-                            const std::vector<int>& strides,
-                            const int groups,
+                            const std::vector<int>& paddings UNUSED,
+                            const std::vector<int>& dilations UNUSED,
+                            const std::vector<int>& strides UNUSED,
+                            const int groups UNUSED,
                             const bool subm,
                             const std::string& key,
                             SparseCooTensor* x_grad,
diff --git a/paddle/phi/kernels/sparse/cpu/conv_kernel.cc b/paddle/phi/kernels/sparse/cpu/conv_kernel.cc
index 48f04ad1ddfa5..7fcbb5cfdd1f2 100644
--- a/paddle/phi/kernels/sparse/cpu/conv_kernel.cc
+++ b/paddle/phi/kernels/sparse/cpu/conv_kernel.cc
@@ -34,7 +34,7 @@ void Conv3dCooCPUKernel(const CPUContext& dev_ctx,
                         const std::vector<int>& paddings,
                         const std::vector<int>& dilations,
                         const std::vector<int>& strides,
-                        const int groups,
+                        const int groups UNUSED,
                         const bool subm,
                         const std::string& key,
                         SparseCooTensor* out,

From 04e5e7b76e95ff1de4955190ec6e75024eef2843 Mon Sep 17 00:00:00 2001
From: Zhang Jun <ewalker@live.cn>
Date: Thu, 11 May 2023 19:27:31 +0800
Subject: [PATCH 283/405] [inference Zero-Dim]add equal, elementwise_op trt 0d
 (#53704)

---
 paddle/fluid/inference/tensorrt/op_teller.cc  |  50 +++---
 .../inference/test_trt_convert_elementwise.py | 156 ++++++++++++++++++
 test/ir/inference/test_trt_convert_equal.py   |  92 ++++++-----
 3 files changed, 230 insertions(+), 68 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index fb16ff235467e..39dfd2cb993f2 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -114,6 +114,25 @@ struct SimpleOpTypeSetTeller : public Teller {
         "sign",       "silu", "logical_not", "reciprocal", "tanh_shrink",
         "logsigmoid", "erf",  "bitwise_not", "equal",      "not_equal",
         "rsqrt"};
+
+    // Static shape does not support 0 or 1 dim's input.
+    if (!with_dynamic_shape) {
+      auto inputs = desc.Inputs();
+      for (auto iter : inputs) {
+        for (auto var_name : iter.second) {
+          auto* block = desc.Block();
+          if (block) {
+            auto* var_desc = block->FindVar(var_name);
+            // Can't get feed op's TensorDesc
+            if (op_type != "feed" && var_desc && !var_desc->Persistable()) {
+              const auto shape = var_desc->GetShape();
+              if (shape.size() == 1 || shape.size() == 0) return false;
+            }
+          }
+        }
+      }
+    }
+
     if (act_op_list.find(op_type) != act_op_list.end()) {
       auto* block = desc.Block();
       if (block == nullptr) {
@@ -122,15 +141,6 @@ struct SimpleOpTypeSetTeller : public Teller {
                    "the pass.";
         return false;
       }
-      auto x_var_name = desc.Input("X")[0];
-      auto* x_var_desc = block->FindVar(x_var_name);
-      const auto x_shape = x_var_desc->GetShape();
-      if (!with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
-        VLOG(3) << op_type
-                << " op does not support input's dim is 1 or 0 in tensorrt "
-                   "static shape mode.";
-        return false;
-      }
 #if !IS_TRT_VERSION_GE(7000)
       if (op_type == "erf") {
         VLOG(3) << op_type << " op does not support tensorrt.";
@@ -138,6 +148,9 @@ struct SimpleOpTypeSetTeller : public Teller {
       }
 #endif
 #if !IS_TRT_VERSION_GE(8600)
+      auto x_var_name = desc.Input("X")[0];
+      auto* x_var_desc = block->FindVar(x_var_name);
+      const auto x_shape = x_var_desc->GetShape();
       if (x_shape.size() == 0 && unary_list.find(op_type) != unary_list.end()) {
         VLOG(3) << op_type
                 << " op does not support 0 dim input when TensorRT < 8.6.";
@@ -145,24 +158,6 @@ struct SimpleOpTypeSetTeller : public Teller {
       }
 #endif
     }
-    // In static shape in Paddle-TRT, we can't allow that one op has a
-    // 1D intermediate tensor as input.
-    if (!with_dynamic_shape) {
-      auto inputs = desc.Inputs();
-      for (auto iter : inputs) {
-        for (auto var_name : iter.second) {
-          auto* block = desc.Block();
-          if (block) {
-            auto* var_desc = block->FindVar(var_name);
-            // Can't get feed op's TensorDesc
-            if (op_type != "feed" && var_desc && !var_desc->Persistable()) {
-              const auto shape = var_desc->GetShape();
-              if (shape.size() == 1) return false;
-            }
-          }
-        }
-      }
-    }
 
     if (op_type == "dropout") {
       /*
@@ -1505,6 +1500,7 @@ struct SimpleOpTypeSetTeller : public Teller {
                    "elementwise op.";
         return false;
       }
+
       if (x_var_desc->Persistable() && !with_dynamic_shape) {
         VLOG(3)
             << "Input X is a parameter which is not supported for "
diff --git a/test/ir/inference/test_trt_convert_elementwise.py b/test/ir/inference/test_trt_convert_elementwise.py
index a6faff0787be5..0ac4a2ba46209 100644
--- a/test/ir/inference/test_trt_convert_elementwise.py
+++ b/test/ir/inference/test_trt_convert_elementwise.py
@@ -1214,5 +1214,161 @@ def test(self):
         self.run_test()
 
 
+class TrtConvertElementwise0D(TrtLayerAutoScanTest):
+    def is_program_valid(self, program_config: ProgramConfig) -> bool:
+        return True
+
+    def sample_program_configs(self):
+        def generate_input(dims, op_type):
+            shape = []
+            if dims == 0:
+                shape = []
+            elif dims == 1:
+                shape = [8]
+            elif dims == 2:
+                shape = [1, 8]
+            elif dims == 3:
+                shape = [1, 8, 8]
+            else:
+                shape = [1, 8, 8, 8]
+
+            # elementwise_floordiv is integer only
+            if op_type == "elementwise_floordiv":
+                return np.random.randint(
+                    low=1, high=10000, size=shape, dtype=np.int32
+                )
+            elif op_type == "elementwise_mod":
+                return np.random.uniform(low=0.1, high=1.0, size=shape).astype(
+                    np.float32
+                )
+            else:
+                return np.random.random(shape).astype(np.float32)
+
+        for dims in [[0, 0], [0, 1], [0, 2], [1, 0], [2, 0]]:
+            for op_type in [
+                "elementwise_add",
+                "elementwise_mul",
+                "elementwise_sub",
+                "elementwise_div",
+                "elementwise_pow",
+                "elementwise_min",
+                "elementwise_max",
+                "elementwise_floordiv",
+                "elementwise_mod",
+            ]:
+                for axis in [-1 if dims[0] == 1 or dims[0] == 0 else 1]:
+                    self.dims = dims[0]
+                    dics = [{"axis": axis}]
+                    ops_config = [
+                        {
+                            "op_type": op_type,
+                            "op_inputs": {
+                                "X": ["input_data"],
+                                "Y": ["weight"],
+                            },
+                            "op_outputs": {"Out": ["output_data"]},
+                            "op_attrs": dics[0],
+                            "outputs_dtype": {
+                                "output_data": np.float32
+                                if op_type != "elementwise_floordiv"
+                                else np.int32
+                            },
+                        }
+                    ]
+                    ops = self.generate_op_config(ops_config)
+
+                    program_config = ProgramConfig(
+                        ops=ops,
+                        weights={
+                            "weight": TensorConfig(
+                                data_gen=partial(
+                                    generate_input, dims[1], op_type
+                                )
+                            )
+                        },
+                        inputs={
+                            "input_data": TensorConfig(
+                                data_gen=partial(
+                                    generate_input, dims[0], op_type
+                                )
+                            ),
+                        },
+                        outputs=["output_data"],
+                    )
+
+                    yield program_config
+
+    def sample_predictor_configs(
+        self, program_config
+    ) -> (paddle_infer.Config, List[int], float):
+        def generate_dynamic_shape(attrs):
+            # The input.dims[1] must be equal to the weight's length.
+            if self.dims == 0:
+                self.dynamic_shape.min_input_shape = {"input_data": []}
+                self.dynamic_shape.max_input_shape = {"input_data": []}
+                self.dynamic_shape.opt_input_shape = {"input_data": []}
+            if self.dims == 1:
+                self.dynamic_shape.min_input_shape = {"input_data": [1]}
+                self.dynamic_shape.max_input_shape = {"input_data": [16]}
+                self.dynamic_shape.opt_input_shape = {"input_data": [8]}
+            elif self.dims == 2:
+                self.dynamic_shape.min_input_shape = {"input_data": [1, 8]}
+                self.dynamic_shape.max_input_shape = {"input_data": [4, 8]}
+                self.dynamic_shape.opt_input_shape = {"input_data": [2, 8]}
+            elif self.dims == 3:
+                self.dynamic_shape.min_input_shape = {"input_data": [1, 1, 4]}
+                self.dynamic_shape.max_input_shape = {"input_data": [4, 16, 16]}
+                self.dynamic_shape.opt_input_shape = {"input_data": [2, 8, 8]}
+            elif self.dims == 4:
+                self.dynamic_shape.min_input_shape = {
+                    "input_data": [1, 8, 8, 8]
+                }
+                self.dynamic_shape.max_input_shape = {
+                    "input_data": [4, 8, 8, 8]
+                }
+                self.dynamic_shape.opt_input_shape = {
+                    "input_data": [4, 8, 8, 8]
+                }
+
+        def clear_dynamic_shape():
+            self.dynamic_shape.max_input_shape = {}
+            self.dynamic_shape.min_input_shape = {}
+            self.dynamic_shape.opt_input_shape = {}
+
+        def generate_trt_nodes_num(attrs, dynamic_shape):
+            if not dynamic_shape and (self.dims == 1 or self.dims == 0):
+                return 0, 3
+            return 1, 2
+
+        attrs = [
+            program_config.ops[i].attrs for i in range(len(program_config.ops))
+        ]
+
+        # for static_shape
+        clear_dynamic_shape()
+        self.trt_param.precision = paddle_infer.PrecisionType.Float32
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, False
+        ), (1e-5, 1e-5)
+        self.trt_param.precision = paddle_infer.PrecisionType.Half
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, False
+        ), (1e-3, 1e-3)
+
+        # # for dynamic_shape
+        generate_dynamic_shape(attrs)
+        self.trt_param.precision = paddle_infer.PrecisionType.Float32
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), (1e-5, 1e-5)
+        self.trt_param.precision = paddle_infer.PrecisionType.Half
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), (1e-3, 1e-3)
+
+    def test(self):
+        self.run_test()
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/test/ir/inference/test_trt_convert_equal.py b/test/ir/inference/test_trt_convert_equal.py
index 4993e830f190b..5879a003d9546 100644
--- a/test/ir/inference/test_trt_convert_equal.py
+++ b/test/ir/inference/test_trt_convert_equal.py
@@ -40,54 +40,64 @@ def generate_input(shape):
             return np.random.random(shape).astype(np.float32)
 
         for op_type in ["equal", "not_equal"]:
-            for batch in [1, 2, 4]:
-                for shape in [[batch, 1], [batch, 1, 32], [batch, 1, 16, 32]]:
-                    for axis in [-1 if len(shape) == 1 else 1]:
-                        self.dims = len(shape)
-                        dics = [{"axis": axis}, {"in_dtype": 0, "out_dtype": 5}]
-                        ops_config = [
-                            {
-                                "op_type": op_type,
-                                "op_inputs": {
-                                    "X": ["input_data1"],
-                                    "Y": ["input_data2"],
-                                },
-                                "op_outputs": {"Out": ["compare_output_data"]},
-                                "op_attrs": dics[0],
-                                "outputs_dtype": {
-                                    "compare_output_data": np.bool_
-                                },
+            for shape in [[], [1, 1], [1, 1, 32], [1, 1, 16, 32]]:
+                for axis in [-1 if len(shape) == 1 or len(shape) == 0 else 1]:
+                    self.dims = len(shape)
+                    dics = [{"axis": axis}, {"in_dtype": 0, "out_dtype": 5}]
+                    ops_config = [
+                        {
+                            "op_type": op_type,
+                            "op_inputs": {
+                                "X": ["input_data1"],
+                                "Y": ["input_data2"],
                             },
-                            {
-                                "op_type": "cast",
-                                "op_inputs": {"X": ["compare_output_data"]},
-                                "op_outputs": {"Out": ["output_data"]},
-                                "op_attrs": dics[1],
-                                "outputs_dtype": {"output_data": np.float32},
-                            },
-                        ]
-                        ops = self.generate_op_config(ops_config)
-
-                        program_config = ProgramConfig(
-                            ops=ops,
-                            weights={},
-                            inputs={
-                                "input_data1": TensorConfig(
-                                    data_gen=partial(generate_input, shape)
-                                ),
-                                "input_data2": TensorConfig(
-                                    data_gen=partial(generate_input, shape)
-                                ),
-                            },
-                            outputs=["output_data"],
-                        )
-                        yield program_config
+                            "op_outputs": {"Out": ["compare_output_data"]},
+                            "op_attrs": dics[0],
+                            "outputs_dtype": {"compare_output_data": np.bool_},
+                        },
+                        {
+                            "op_type": "cast",
+                            "op_inputs": {"X": ["compare_output_data"]},
+                            "op_outputs": {"Out": ["output_data"]},
+                            "op_attrs": dics[1],
+                            "outputs_dtype": {"output_data": np.float32},
+                        },
+                    ]
+                    ops = self.generate_op_config(ops_config)
+
+                    program_config = ProgramConfig(
+                        ops=ops,
+                        weights={},
+                        inputs={
+                            "input_data1": TensorConfig(
+                                data_gen=partial(generate_input, shape)
+                            ),
+                            "input_data2": TensorConfig(
+                                data_gen=partial(generate_input, shape)
+                            ),
+                        },
+                        outputs=["output_data"],
+                    )
+                    yield program_config
 
     def sample_predictor_configs(
         self, program_config
     ) -> (paddle_infer.Config, List[int], float):
         def generate_dynamic_shape(attrs):
             # The input.dims[1] must be equal to the weight's length.
+            if self.dims == 0:
+                self.dynamic_shape.min_input_shape = {
+                    "input_data1": [],
+                    "input_data2": [],
+                }
+                self.dynamic_shape.max_input_shape = {
+                    "input_data1": [],
+                    "input_data2": [],
+                }
+                self.dynamic_shape.opt_input_shape = {
+                    "input_data1": [],
+                    "input_data2": [],
+                }
             if self.dims == 2:
                 self.dynamic_shape.min_input_shape = {
                     "input_data1": [1, 1],

From 4a69a5361922ef6ee966014cdb6733cc3518f586 Mon Sep 17 00:00:00 2001
From: Zhang Jun <ewalker@live.cn>
Date: Thu, 11 May 2023 19:43:35 +0800
Subject: [PATCH 284/405] [inference][trt]add trt sparse weights switch
 (#53562)

---
 paddle/fluid/inference/analysis/argument.h      |  3 +++
 .../fluid/inference/analysis/ir_pass_manager.cc |  2 ++
 .../ir_passes/tensorrt_subgraph_pass.cc         | 17 ++++++++---------
 paddle/fluid/inference/api/analysis_config.cc   |  5 +++++
 .../fluid/inference/api/analysis_predictor.cc   |  1 +
 .../inference/api/paddle_analysis_config.h      |  4 ++++
 paddle/fluid/inference/tensorrt/engine.cc       | 11 ++++-------
 paddle/fluid/inference/tensorrt/engine.h        |  4 ++++
 paddle/fluid/inference/tensorrt/helper.h        |  4 ++++
 paddle/fluid/pybind/inference_api.cc            |  4 ++++
 .../inference/test_trt_inference_predictor.py   | 12 ++++++++++++
 11 files changed, 51 insertions(+), 16 deletions(-)

diff --git a/paddle/fluid/inference/analysis/argument.h b/paddle/fluid/inference/analysis/argument.h
index f44c877b93965..4cac76b47bc0c 100644
--- a/paddle/fluid/inference/analysis/argument.h
+++ b/paddle/fluid/inference/analysis/argument.h
@@ -250,6 +250,9 @@ struct Argument {
                       TensorRtAllowBuildAtRuntime,
                       bool);
   DECL_ARGUMENT_FIELD(tensorrt_use_inspector, TensorRtUseInspector, bool);
+  DECL_ARGUMENT_FIELD(tensorrt_use_sparse_weights,
+                      TensorRtUseSparseWeights,
+                      bool);
 
   DECL_ARGUMENT_FIELD(use_dlnne, UseDlnne, bool);
   DECL_ARGUMENT_FIELD(dlnne_min_subgraph_size, DlnneMinSubgraphSize, int);
diff --git a/paddle/fluid/inference/analysis/ir_pass_manager.cc b/paddle/fluid/inference/analysis/ir_pass_manager.cc
index 4051511906b1b..6408e5ca2d362 100644
--- a/paddle/fluid/inference/analysis/ir_pass_manager.cc
+++ b/paddle/fluid/inference/analysis/ir_pass_manager.cc
@@ -213,6 +213,8 @@ void IRPassManager::CreatePasses(Argument *argument,
       pass->Set("use_static_engine", new bool(use_static_engine));
       pass->Set("model_from_memory", new bool(argument->model_from_memory()));
       pass->Set("use_inspector", new bool(argument->tensorrt_use_inspector()));
+      pass->Set("use_sparse_weights",
+                new bool(argument->tensorrt_use_sparse_weights()));
 
       // tuned trt dynamic_shape
       pass->Set("trt_shape_range_info_path",
diff --git a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc
index 33e5622436a0f..e9733a07ec8b5 100644
--- a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc
+++ b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc
@@ -523,6 +523,7 @@ std::string TensorRtSubgraphPass::CreateTensorRTOp(
   op_desc->SetAttr("allow_build_at_runtime", allow_build_at_runtime);
   op_desc->SetAttr("shape_range_info_path", shape_range_info_path);
   op_desc->SetAttr("use_inspector", Get<bool>("use_inspector"));
+  op_desc->SetAttr("use_sparse_weights", Get<bool>("use_sparse_weights"));
   op_desc->SetAttr("model_precision", Get<int>("model_precision"));
   op_desc->SetAttr("with_dynamic_shape", with_dynamic_shape);
 
@@ -614,17 +615,14 @@ std::string TensorRtSubgraphPass::CreateTensorRTOp(
     opt_input_shape = {};
   }
 
-  auto to_major_version = [&](int full_version) -> float {
-    return (full_version / 100) / 10.0;
-  };
-  const float compile_time_trt_version = to_major_version(TRT_VERSION);
-  const float run_time_trt_version =
-      to_major_version(tensorrt::GetInferLibVersion());
-  if (compile_time_trt_version != run_time_trt_version) {
+  const float trt_compile_version = tensorrt::TrtMajorVersion(TRT_VERSION);
+  const float trt_runtime_version =
+      tensorrt::TrtMajorVersion(tensorrt::GetInferLibVersion());
+  if (trt_compile_version != trt_runtime_version) {
     LOG_FIRST_N(WARNING, 1)
         << "The Paddle Inference library is compiled with "
-        << compile_time_trt_version << " version TensorRT, "
-        << "but the runtime TensorRT you are using is " << run_time_trt_version
+        << trt_compile_version << " version TensorRT, "
+        << "but the runtime TensorRT you are using is " << trt_runtime_version
         << " version. "
            "This might cause serious compatibility issues. We strongly "
            "recommend using the same TRT version at runtime.";
@@ -666,6 +664,7 @@ std::string TensorRtSubgraphPass::CreateTensorRTOp(
   trt_engine->SetUseDLA(Get<bool>("trt_use_dla"));
   trt_engine->SetDLACore(Get<int>("trt_dla_core"));
   trt_engine->SetUseInspector(Get<bool>("use_inspector"));
+  trt_engine->SetUseSparseWeights(Get<bool>("use_sparse_weights"));
   trt_engine->SetWithErnie(
       graph->Has(framework::ir::kEmbEltwiseLayernormPass) &&
       graph->Has(framework::ir::kMultiheadMatmulPass));
diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc
index af7b8574b863d..ae281c5e9de44 100644
--- a/paddle/fluid/inference/api/analysis_config.cc
+++ b/paddle/fluid/inference/api/analysis_config.cc
@@ -451,6 +451,7 @@ AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) {
   CP_MEMBER(collect_shape_range_info_);
   CP_MEMBER(shape_range_info_path_);
   CP_MEMBER(trt_use_inspector_);
+  CP_MEMBER(trt_use_sparse_weights_);
   CP_MEMBER(trt_engine_memory_sharing_);
   CP_MEMBER(trt_engine_memory_sharing_identifier_);
   // Dlnne related
@@ -805,6 +806,10 @@ void AnalysisConfig::EnableTensorRtDLA(int dla_core) {
 
 void AnalysisConfig::EnableTensorRtInspector() { trt_use_inspector_ = true; }
 
+void AnalysisConfig::EnableTensorRtSparseWeights() {
+  trt_use_sparse_weights_ = true;
+}
+
 void AnalysisConfig::Exp_DisableTensorRtOPs(
     const std::vector<std::string> &ops) {
   trt_disabled_ops_.insert(trt_disabled_ops_.end(), ops.begin(), ops.end());
diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc
index a2debb7b43cd3..6732f2dbe366c 100644
--- a/paddle/fluid/inference/api/analysis_predictor.cc
+++ b/paddle/fluid/inference/api/analysis_predictor.cc
@@ -1358,6 +1358,7 @@ void AnalysisPredictor::PrepareArgument() {
     argument_->SetTensorRtAllowBuildAtRuntime(
         config_.trt_allow_build_at_runtime());
     argument_->SetTensorRtUseInspector(config_.trt_use_inspector_);
+    argument_->SetTensorRtUseSparseWeights(config_.trt_use_sparse_weights_);
     argument_->SetTrtEngineMemorySharing(config_.trt_engine_memory_sharing());
   }
 
diff --git a/paddle/fluid/inference/api/paddle_analysis_config.h b/paddle/fluid/inference/api/paddle_analysis_config.h
index b30838c3680e9..28934864c8cfd 100644
--- a/paddle/fluid/inference/api/paddle_analysis_config.h
+++ b/paddle/fluid/inference/api/paddle_analysis_config.h
@@ -742,6 +742,9 @@ struct PD_INFER_DECL AnalysisConfig {
   void EnableTensorRtInspector();
   bool tensorrt_inspector_enabled() { return trt_use_inspector_; }
 
+  void EnableTensorRtSparseWeights();
+  bool tensorrt_sparse_weights_enabled() { return trt_use_sparse_weights_; }
+
   void EnableDlnne(
       int min_subgraph_size = 3,
       int max_batch_size = 1,
@@ -1118,6 +1121,7 @@ struct PD_INFER_DECL AnalysisConfig {
   // tune to get dynamic_shape info.
   bool trt_tuned_dynamic_shape_{false};
   bool trt_use_inspector_{false};
+  bool trt_use_sparse_weights_{false};
 
   // In CollectShapeInfo mode, we will collect the shape information of
   // all intermediate tensors in the compute graph and calculate the
diff --git a/paddle/fluid/inference/tensorrt/engine.cc b/paddle/fluid/inference/tensorrt/engine.cc
index b2ee3f4796595..2864adffdaa52 100644
--- a/paddle/fluid/inference/tensorrt/engine.cc
+++ b/paddle/fluid/inference/tensorrt/engine.cc
@@ -207,12 +207,6 @@ void TensorRTEngine::FreezeNetwork() {
   infer_builder_config_->setMaxWorkspaceSize(max_workspace_);
 #endif
 
-#if IS_TRT_VERSION_GE(8500)
-  infer_builder_config_->setPreviewFeature(
-      nvinfer1::PreviewFeature::kFASTER_DYNAMIC_SHAPES_0805, true);
-#else
-#endif
-
   bool enable_fp16 = (precision_ == AnalysisConfig::Precision::kHalf);
   if (enable_fp16) {
     bool support_fp16 = infer_builder_->platformHasFastFp16();
@@ -363,6 +357,7 @@ void TensorRTEngine::FreezeNetwork() {
                    "opt_shape, false /*disable_trt_plugin_fp16*/)'";
     }
   }
+
 #if IS_TRT_VERSION_GE(8200)
   if (use_inspector_) {
     infer_builder_config_->setProfilingVerbosity(
@@ -374,7 +369,9 @@ void TensorRTEngine::FreezeNetwork() {
   infer_engine_.reset(infer_builder_->buildEngineWithConfig(
       *network(), *infer_builder_config_));
 #else
-  infer_builder_config_->setFlag(nvinfer1::BuilderFlag::kSPARSE_WEIGHTS);
+  if (use_sparse_weights_) {
+    infer_builder_config_->setFlag(nvinfer1::BuilderFlag::kSPARSE_WEIGHTS);
+  }
   ihost_memory_.reset(infer_builder_->buildSerializedNetwork(
       *network(), *infer_builder_config_));
   infer_ptr<nvinfer1::IRuntime> runtime(createInferRuntime(&logger_));
diff --git a/paddle/fluid/inference/tensorrt/engine.h b/paddle/fluid/inference/tensorrt/engine.h
index 1906b8be5171e..3f4df60586573 100644
--- a/paddle/fluid/inference/tensorrt/engine.h
+++ b/paddle/fluid/inference/tensorrt/engine.h
@@ -739,6 +739,9 @@ class TensorRTEngine {
   void GetEngineInfo();
 
   void SetUseInspector(bool use_inspector) { use_inspector_ = use_inspector; }
+  void SetUseSparseWeights(bool use_sparse_weights) {
+    use_sparse_weights_ = use_sparse_weights;
+  }
   void SetScope(const framework::Scope& scope) { scope_ = &scope; }
 
   void SetContextMemorySharing(bool context_memory_sharing) {
@@ -827,6 +830,7 @@ class TensorRTEngine {
 #endif
   std::mutex mutex_;
   bool use_inspector_;
+  bool use_sparse_weights_{false};
 
  public:
   thread_local static int predictor_id_per_thread;
diff --git a/paddle/fluid/inference/tensorrt/helper.h b/paddle/fluid/inference/tensorrt/helper.h
index 71f31313b7cc9..e744b261cbcfa 100644
--- a/paddle/fluid/inference/tensorrt/helper.h
+++ b/paddle/fluid/inference/tensorrt/helper.h
@@ -96,6 +96,10 @@ static std::tuple<int, int, int> GetTrtCompileVersion() {
       NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH};
 }
 
+static float TrtMajorVersion(int full_version) {
+  return (full_version / 100) / 10.0;
+}
+
 template <typename T>
 struct Destroyer {
   void operator()(T* x) {
diff --git a/paddle/fluid/pybind/inference_api.cc b/paddle/fluid/pybind/inference_api.cc
index 09e1e42cfbdd9..0995bc832ac88 100644
--- a/paddle/fluid/pybind/inference_api.cc
+++ b/paddle/fluid/pybind/inference_api.cc
@@ -879,6 +879,10 @@ void BindAnalysisConfig(py::module *m) {
            &AnalysisConfig::EnableTensorRtInspector)
       .def("tensorrt_inspector_enabled",
            &AnalysisConfig::tensorrt_inspector_enabled)
+      .def("enable_tensorrt_sparse_weights",
+           &AnalysisConfig::EnableTensorRtSparseWeights)
+      .def("tensorrt_sparse_weights_enabled",
+           &AnalysisConfig::tensorrt_sparse_weights_enabled)
       .def("tensorrt_engine_enabled", &AnalysisConfig::tensorrt_engine_enabled)
       .def("enable_dlnne",
            &AnalysisConfig::EnableDlnne,
diff --git a/test/ir/inference/test_trt_inference_predictor.py b/test/ir/inference/test_trt_inference_predictor.py
index e334e5eabfd74..ee0b8b48626bb 100644
--- a/test/ir/inference/test_trt_inference_predictor.py
+++ b/test/ir/inference/test_trt_inference_predictor.py
@@ -84,6 +84,8 @@ def load(self, config_arg, inputs=None, outpus=None):
         # enable memory optim
         if not self.args.enable_tune:
             config.enable_memory_optim()
+        if self.args.enable_trt_sparse_weights:
+            config.enable_tensorrt_sparse_weights()
 
         config.set_cpu_math_library_num_threads(self.args.cpu_threads)
         config.switch_ir_optim(True)
@@ -258,6 +260,9 @@ def parse_args():
     parser.add_argument('--enable_dynamic_shape', type=str2bool, default=True)
     parser.add_argument('--enable_tune', type=str2bool, default=False)
     parser.add_argument('--enable_profile', type=str2bool, default=False)
+    parser.add_argument(
+        '--enable_trt_sparse_weights', type=str2bool, default=False
+    )
     parser.add_argument('--enable_benchmark', type=str2bool, default=True)
     parser.add_argument('--save_result', type=str2bool, default=False)
     parser.add_argument('--return_result', type=str2bool, default=False)
@@ -308,6 +313,13 @@ def run_infer(model_path):
     backend.load(conf)
     backend.predict()
 
+    # run inference predictor, enable trt sparse weights
+    conf.enable_tune = False
+    conf.enable_trt_sparse_weights = True
+    backend = BackendPaddle()
+    backend.load(conf)
+    backend.predict()
+
 
 class ConvBNLayer(paddle.nn.Layer):
     def __init__(

From e92a9bb35cfa7368653b644084c6c214d6c0d006 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Thu, 11 May 2023 19:50:47 +0800
Subject: [PATCH 285/405] Correct the condition of whether can use flash-attn.

---
 paddle/fluid/operators/fused/fused_gate_attention.h | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index b8a0ae53cc3a2..19659a46e1cc2 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -200,14 +200,13 @@ struct GateAttentionConfig {
       return false;
     }
 
-    if (merge_qkv && (head_dim % 8 == 0) && (head_dim <= 128)) {
-      return use_flash_attn;
-    } else {
-      return false;
+    if (merge_qkv) {
+      if (head_dim == 32 || head_dim == 64 || head_dim == 128) {
+        return use_flash_attn;
+      }
     }
-#else
-    return false;
 #endif
+    return false;
   }
 
   int64_t GetQuerySize() const {

From 5417382d9ccd8799b5e4de6d8bc19579e17f875a Mon Sep 17 00:00:00 2001
From: zhangkaihuo <zhangkaihuo@baidu.com>
Date: Thu, 11 May 2023 20:00:42 +0800
Subject: [PATCH 286/405] fix doc of compare_accuracy (#53661)

---
 python/paddle/amp/debugging.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/paddle/amp/debugging.py b/python/paddle/amp/debugging.py
index c55d3506be33f..3672ecd6b353d 100644
--- a/python/paddle/amp/debugging.py
+++ b/python/paddle/amp/debugging.py
@@ -436,10 +436,10 @@ def compare_accuracy(
     This is a precision comparison tool that can be used to compare log data of float16 and float32.
 
     Args:
-        dump_path(str): The path of the running log, such as the log for execution using the fp32 type.
-        another_dump_path(str): the path of another running log ,such as the log for execution using the fp16 type.
+        dump_path(str): The path of the running log, such as the log for execution using the float32 data type.
+        another_dump_path(str): the path of another running log ,such as the log for execution using the float16 data type.
         output_filename(str): the excel file nmae of compare output.
-        loss_scale(float): the loss_scale during the training phase.
+        loss_scale(float, optional): the loss_scale during the training phase. Default is 1.
         dump_all_tensors(bool, optional): dump all tensor, It is currently not support. Default is False.
 
     Examples:

From 82c73884ac48032c40d6bd5713705d8797ba3635 Mon Sep 17 00:00:00 2001
From: Yuanle Liu <yuanlehome@163.com>
Date: Thu, 11 May 2023 20:36:07 +0800
Subject: [PATCH 287/405] [inference Zero-Dim]prelu trt converter support zero
 dim tensor (#53634)

* prelu op trt converter support zero dim
---
 .../framework/ir/trt_support_nhwc_pass.cc     |   2 +
 .../inference/tensorrt/convert/prelu_op.cc    |   1 -
 paddle/fluid/inference/tensorrt/op_teller.cc  |  28 +-
 paddle/phi/infermeta/binary.cc                |   2 +-
 test/ir/inference/test_trt_convert_prelu.py   | 288 ++++++++----------
 5 files changed, 148 insertions(+), 173 deletions(-)

diff --git a/paddle/fluid/framework/ir/trt_support_nhwc_pass.cc b/paddle/fluid/framework/ir/trt_support_nhwc_pass.cc
index 86c7b7c9dbbae..0b12559e31deb 100644
--- a/paddle/fluid/framework/ir/trt_support_nhwc_pass.cc
+++ b/paddle/fluid/framework/ir/trt_support_nhwc_pass.cc
@@ -356,6 +356,8 @@ void TrtSupportNHWCPass::ApplyImpl(Graph *graph) const {
     }
   };
   InsertTransposeOp();
+
+  AddStatis(transposed_ops.size());
 }
 
 }  // namespace ir
diff --git a/paddle/fluid/inference/tensorrt/convert/prelu_op.cc b/paddle/fluid/inference/tensorrt/convert/prelu_op.cc
index d655a4ce04fd6..80a2ac46f44dc 100644
--- a/paddle/fluid/inference/tensorrt/convert/prelu_op.cc
+++ b/paddle/fluid/inference/tensorrt/convert/prelu_op.cc
@@ -87,7 +87,6 @@ class PReluOpConverter : public OpConverter {
           if (hw_tensor != nullptr) {
             shape_tensor = Concat(
                 std::vector<nvinfer1::ITensor*>{n_tensor, c_tensor, hw_tensor});
-
           } else {
             shape_tensor =
                 Concat(std::vector<nvinfer1::ITensor*>{n_tensor, c_tensor});
diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index 39dfd2cb993f2..414d4873de3b0 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -1837,28 +1837,28 @@ struct SimpleOpTypeSetTeller : public Teller {
                    "the pass.";
         return false;
       }
-      auto* var_desc = block->FindVar(desc.Input("Alpha")[0]);
-      if (!var_desc) {
+      auto* alpha_var = block->FindVar(desc.Input("Alpha")[0]);
+      if (!alpha_var) {
         VLOG(3) << "Variable Alpha of prelu TRT converter not found.";
         return false;
       }
-
-      auto x_var_name = desc.Input("X")[0];
-      auto* x_var_desc = block->FindVar(x_var_name);
-      const auto x_shape = x_var_desc->GetShape();
-      if (!with_dynamic_shape && x_shape.size() == 1) {
-        VLOG(3) << "prelu op does not support input's dim is 1 in tensorrt "
-                   "with static shape.";
+      auto alpha_shape = alpha_var->GetShape();
+      if (!with_dynamic_shape && alpha_shape.size() == 0) {
+        VLOG(3) << op_type
+                << " op does not support alpha's dim is 0 in tensorrt "
+                   "static shape mode.";
         return false;
       }
 
-#if IS_TRT_VERSION_LT(7000)
-      if (!with_dynamic_shape) {
-        // TODO(inference): fix trt6 static plugin error.
-        VLOG(3) << "prelu static plugin in trt6 has bug.";
+      auto x_var_name = desc.Input("X")[0];
+      auto* x_var = block->FindVar(x_var_name);
+      const auto x_shape = x_var->GetShape();
+      if (!with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
+        VLOG(3) << op_type
+                << " op does not support input's dim is 1 or 0 in tensorrt "
+                   "with static shape.";
         return false;
       }
-#endif
     }
 
     if (op_type == "mish") {
diff --git a/paddle/phi/infermeta/binary.cc b/paddle/phi/infermeta/binary.cc
index efb6458601ba9..e32b1fc241b48 100644
--- a/paddle/phi/infermeta/binary.cc
+++ b/paddle/phi/infermeta/binary.cc
@@ -2340,7 +2340,7 @@ void PReluInferMeta(const MetaTensor& x,
                       1,
                       phi::errors::InvalidArgument(
                           "For mode 'element', rank of input X must be "
-                          "equal or larger than 2. But recevied X's "
+                          "equal or larger than 1. But recevied X's "
                           "rank: %d",
                           x_rank));
     PADDLE_ENFORCE_EQ(
diff --git a/test/ir/inference/test_trt_convert_prelu.py b/test/ir/inference/test_trt_convert_prelu.py
index 14046d45a3b2d..ae53f10ad5761 100644
--- a/test/ir/inference/test_trt_convert_prelu.py
+++ b/test/ir/inference/test_trt_convert_prelu.py
@@ -18,7 +18,7 @@
 
 import numpy as np
 from program_config import ProgramConfig, TensorConfig
-from trt_layer_auto_scan_test import SkipReasons, TrtLayerAutoScanTest
+from trt_layer_auto_scan_test import TrtLayerAutoScanTest
 
 import paddle.inference as paddle_infer
 
@@ -28,170 +28,165 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool:
         return True
 
     def sample_program_configs(self):
-        def generate_input(batch, dim1, dim2, dim3):
-            shape = [batch]
-            if dim1 != 0:
-                shape.append(dim1)
-            if dim2 != 0:
-                shape.append(dim2)
-            if dim3 != 0:
-                shape.append(dim3)
-            return np.random.random(shape).astype(np.float32)
-
-        def generate_alpha(attrs: List[Dict[str, Any]], dim1, dim2, dim3):
+        def generate_input(attrs: List[Dict[str, Any]], batch):
+            if self.dims == 0:
+                return np.random.random([]).astype(np.float32)
+            elif self.dims == 1:
+                return np.random.random([16]).astype(np.float32)
+            elif self.dims == 2:
+                return np.random.random([1, 3]).astype(np.float32)
+            elif self.dims == 3:
+                if attrs[0]["data_format"] == "NCHW":
+                    return np.random.random([batch, 3, 16]).astype(np.float32)
+                elif attrs[0]["data_format"] == "NHWC":
+                    return np.random.random([batch, 16, 3]).astype(np.float32)
+                else:
+                    raise AssertionError()
+            else:
+                if attrs[0]["data_format"] == "NCHW":
+                    return np.random.random([batch, 3, 16, 32]).astype(
+                        np.float32
+                    )
+                else:
+                    return np.random.random([batch, 16, 32, 3]).astype(
+                        np.float32
+                    )
+
+        def generate_alpha(attrs: List[Dict[str, Any]]):
+            if self.dims == 0:
+                return np.random.random([]).astype(np.float32)
             if attrs[0]["mode"] == "all":
-                return np.random.random(size=(1)).astype(np.float32)
-            elif (
-                attrs[0]["mode"] == "channel"
-                and attrs[0]["data_format"] == "NCHW"
-            ):
-                shape = [1]
-                if dim1 != 0:
-                    shape.append(dim1)
-                if dim2 != 0:
-                    shape.append(dim2)
-                if dim3 != 0:
-                    shape.append(dim3)
-                return np.random.random(size=shape[1]).astype(np.float32)
-            elif (
-                attrs[0]["mode"] == "channel"
-                and attrs[0]["data_format"] == "NHWC"
-            ):
-                shape = [1]
-                if dim1 != 0:
-                    shape.append(dim1)
-                if dim2 != 0:
-                    shape.append(dim2)
-                if dim3 != 0:
-                    shape.append(dim3)
-                return np.random.random(size=shape[-1]).astype(np.float32)
+                return np.random.random([1]).astype(np.float32)
+            elif attrs[0]["mode"] == "channel":
+                return np.random.random([3]).astype(np.float32)
             elif attrs[0]["mode"] == "element":
-                shape = [1]
-                if dim1 != 0:
-                    shape.append(dim1)
-                if dim2 != 0:
-                    shape.append(dim2)
-                if dim3 != 0:
-                    shape.append(dim3)
-                return np.random.random(size=shape).astype(np.float32)
+                if self.dims == 1:
+                    return np.random.random([16]).astype(np.float32)
+                elif self.dims == 2:
+                    return np.random.random([1, 3]).astype(np.float32)
+                elif self.dims == 3:
+                    if attrs[0]["data_format"] == "NCHW":
+                        return np.random.random([1, 3, 16]).astype(np.float32)
+                    elif attrs[0]["data_format"] == "NHWC":
+                        return np.random.random([1, 16, 3]).astype(np.float32)
+                    else:
+                        raise AssertionError()
+                else:
+                    if attrs[0]["data_format"] == "NCHW":
+                        return np.random.random([1, 3, 16, 32]).astype(
+                            np.float32
+                        )
+                    elif attrs[0]["data_format"] == "NHWC":
+                        return np.random.random([1, 16, 32, 3]).astype(
+                            np.float32
+                        )
+                    else:
+                        raise AssertionError()
 
         for batch in [1, 4]:
-            for dim1 in [0, 3]:
-                for dim2 in [0, 16]:
-                    for dim3 in [0, 32]:
-                        self.dim1 = dim1
-                        self.dim2 = dim2
-                        self.dim3 = dim3
-
-                        if dim1 == 0 and dim2 != 0:
+            for dims in [0, 1, 2, 3, 4]:
+                for mode in ["all", "element", "channel"]:
+                    for data_format in ["NCHW", "NHWC"]:
+                        if (mode == "element" or mode == "all") and dims == 0:
                             continue
-                        if dim1 == 0 and dim2 == 0 and dim3 != 0:
+                        if mode == "channel" and dims != 4:
                             continue
-
-                        for mode in ["all", "channel", "element"]:
-                            for data_format in ['NCHW', 'NHWC']:
-                                if (
-                                    mode == "channel"
-                                    and dim1 == 0
-                                    and data_format == "NCHW"
-                                ):
-                                    continue
-                                if (
-                                    mode == "channel"
-                                    and dim3 == 0
-                                    and data_format == "NHWC"
-                                ):
-                                    continue
-                                dics = [
-                                    {"mode": mode, "data_format": data_format}
-                                ]
-                                ops_config = [
-                                    {
-                                        "op_type": "prelu",
-                                        "op_inputs": {
-                                            "X": ["input_data"],
-                                            "Alpha": ["alpha_weight"],
-                                        },
-                                        "op_outputs": {"Out": ["output_data"]},
-                                        "op_attrs": dics[0],
-                                    }
-                                ]
-                                ops = self.generate_op_config(ops_config)
-
-                                program_config = ProgramConfig(
-                                    ops=ops,
-                                    weights={
-                                        "alpha_weight": TensorConfig(
-                                            data_gen=partial(
-                                                generate_alpha,
-                                                dics,
-                                                dim1,
-                                                dim2,
-                                                dim3,
-                                            )
-                                        )
-                                    },
-                                    inputs={
-                                        "input_data": TensorConfig(
-                                            data_gen=partial(
-                                                generate_input,
-                                                batch,
-                                                dim1,
-                                                dim2,
-                                                dim3,
-                                            )
-                                        ),
-                                    },
-                                    outputs=["output_data"],
+                        self.dims = dims
+                        dics = [{"mode": mode, "data_format": data_format}]
+                        ops_config = [
+                            {
+                                "op_type": "prelu",
+                                "op_inputs": {
+                                    "X": ["input_data"],
+                                    "Alpha": ["alpha_weight"],
+                                },
+                                "op_outputs": {"Out": ["output_data"]},
+                                "op_attrs": dics[0],
+                            }
+                        ]
+                        ops = self.generate_op_config(ops_config)
+
+                        program_config = ProgramConfig(
+                            ops=ops,
+                            weights={
+                                "alpha_weight": TensorConfig(
+                                    data_gen=partial(generate_alpha, dics)
                                 )
-
-                                yield program_config
+                            },
+                            inputs={
+                                "input_data": TensorConfig(
+                                    data_gen=partial(
+                                        generate_input, dics, batch
+                                    )
+                                ),
+                            },
+                            outputs=["output_data"],
+                        )
+
+                        yield program_config
 
     def sample_predictor_configs(
         self, program_config
     ) -> (paddle_infer.Config, List[int], float):
         def generate_dynamic_shape(attrs):
-            if self.dim1 == 0:
-                self.dynamic_shape.min_input_shape = {
-                    "input_data": [1],
-                }
-                self.dynamic_shape.max_input_shape = {
-                    "input_data": [4],
-                }
-                self.dynamic_shape.opt_input_shape = {
-                    "input_data": [2],
-                }
-            else:
-                if self.dim2 == 0 and self.dim3 == 0:
+            if self.dims == 0:
+                self.dynamic_shape.min_input_shape = {"input_data": []}
+                self.dynamic_shape.max_input_shape = {"input_data": []}
+                self.dynamic_shape.opt_input_shape = {"input_data": []}
+            elif self.dims == 1:
+                self.dynamic_shape.min_input_shape = {"input_data": [16]}
+                self.dynamic_shape.max_input_shape = {"input_data": [16]}
+                self.dynamic_shape.opt_input_shape = {"input_data": [16]}
+            elif self.dims == 2:
+                self.dynamic_shape.min_input_shape = {"input_data": [1, 3]}
+                self.dynamic_shape.max_input_shape = {"input_data": [1, 3]}
+                self.dynamic_shape.opt_input_shape = {"input_data": [1, 3]}
+            elif self.dims == 3:
+                if attrs[0]["data_format"] == "NCHW":
                     self.dynamic_shape.min_input_shape = {
-                        "input_data": [1, 1],
+                        "input_data": [1, 3, 16]
                     }
                     self.dynamic_shape.max_input_shape = {
-                        "input_data": [4, 32],
+                        "input_data": [4, 3, 16]
                     }
                     self.dynamic_shape.opt_input_shape = {
-                        "input_data": [2, 3],
+                        "input_data": [1, 3, 16]
                     }
-                elif self.dim2 != 0 and self.dim3 != 0:
+                elif attrs[0]["data_format"] == "NHWC":
                     self.dynamic_shape.min_input_shape = {
-                        "input_data": [1, 1, 1, 1],
+                        "input_data": [1, 16, 3]
                     }
                     self.dynamic_shape.max_input_shape = {
-                        "input_data": [4, 3, 16, 32],
+                        "input_data": [4, 16, 3]
                     }
                     self.dynamic_shape.opt_input_shape = {
-                        "input_data": [2, 3, 16, 32],
+                        "input_data": [1, 16, 3]
                     }
-                elif self.dim3 == 0:
+                else:
+                    raise AssertionError()
+            else:
+                if attrs[0]["data_format"] == "NCHW":
                     self.dynamic_shape.min_input_shape = {
-                        "input_data": [1, 1, 1],
+                        "input_data": [1, 3, 16, 32]
                     }
                     self.dynamic_shape.max_input_shape = {
-                        "input_data": [4, 3, 32],
+                        "input_data": [4, 3, 16, 32]
                     }
                     self.dynamic_shape.opt_input_shape = {
-                        "input_data": [2, 3, 16],
+                        "input_data": [1, 3, 16, 32]
                     }
+                elif attrs[0]["data_format"] == "NHWC":
+                    self.dynamic_shape.min_input_shape = {
+                        "input_data": [1, 16, 32, 3]
+                    }
+                    self.dynamic_shape.max_input_shape = {
+                        "input_data": [4, 16, 32, 3]
+                    }
+                    self.dynamic_shape.opt_input_shape = {
+                        "input_data": [1, 16, 32, 3]
+                    }
+                else:
+                    raise AssertionError()
 
         def clear_dynamic_shape():
             self.dynamic_shape.max_input_shape = {}
@@ -203,12 +198,7 @@ def clear_dynamic_shape():
         ]
 
         def generate_trt_nodes_num(attrs, dynamic_shape):
-            if (
-                not dynamic_shape
-                and self.dim1 == 0
-                and self.dim2 == 0
-                and self.dim3 == 0
-            ):
+            if not dynamic_shape and (self.dims == 1 or self.dims == 0):
                 return 0, 3
             return 1, 2
 
@@ -234,23 +224,7 @@ def generate_trt_nodes_num(attrs, dynamic_shape):
             attrs, True
         ), (1e-3, 1e-3)
 
-    def add_skip_trt_case(self):
-        ver = paddle_infer.get_trt_compile_version()
-        if ver[0] * 1000 + ver[1] * 100 + ver[0] * 10 < 7000:
-
-            def teller(program_config, predictor_config):
-                if not predictor_config.tensorrt_dynamic_shape_enabled():
-                    return True
-                return False
-
-            self.add_skip_case(
-                teller,
-                SkipReasons.TRT_NOT_IMPLEMENTED,
-                "Need to repair the case: the output of GPU and tensorrt has diff in trt6, the prelu static plugin has bug.",
-            )
-
     def test(self):
-        self.add_skip_trt_case()
         self.run_test()
 
 

From 38886829fa5272fa2e41efe4780c1975d1bf9926 Mon Sep 17 00:00:00 2001
From: lanxianghit <47554610+lanxianghit@users.noreply.github.com>
Date: Thu, 11 May 2023 20:53:17 +0800
Subject: [PATCH 288/405] add cinn bf16 support (#53637)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

添加CINN与Paddle框架的BFloat16类型映射
---
 paddle/fluid/framework/paddle2cinn/transform_desc.cc      | 2 ++
 paddle/fluid/framework/paddle2cinn/transform_type.cc      | 4 ++++
 paddle/fluid/framework/paddle2cinn/transform_type_test.cc | 2 ++
 python/setup.py.in                                        | 5 +++++
 setup.py                                                  | 6 ++++++
 5 files changed, 19 insertions(+)

diff --git a/paddle/fluid/framework/paddle2cinn/transform_desc.cc b/paddle/fluid/framework/paddle2cinn/transform_desc.cc
index 732edff828305..b61a7be544fd2 100644
--- a/paddle/fluid/framework/paddle2cinn/transform_desc.cc
+++ b/paddle/fluid/framework/paddle2cinn/transform_desc.cc
@@ -88,6 +88,7 @@ ::cinn::frontend::paddle::cpp::VarDescAPI::Type TransformVarDataTypeToCinn(
     SET_DATA_TYPE_CASE_ITEM(INT16);
     SET_DATA_TYPE_CASE_ITEM(INT32);
     SET_DATA_TYPE_CASE_ITEM(INT64);
+    SET_DATA_TYPE_CASE_ITEM(BF16);
     SET_DATA_TYPE_CASE_ITEM(FP16);
     SET_DATA_TYPE_CASE_ITEM(FP32);
     SET_DATA_TYPE_CASE_ITEM(FP64);
@@ -139,6 +140,7 @@ ::paddle::framework::proto::VarType::Type TransformVarDataTypeFromCpp(
     SET_DATA_TYPE_CASE_ITEM(INT16);
     SET_DATA_TYPE_CASE_ITEM(INT32);
     SET_DATA_TYPE_CASE_ITEM(INT64);
+    SET_DATA_TYPE_CASE_ITEM(BF16);
     SET_DATA_TYPE_CASE_ITEM(FP16);
     SET_DATA_TYPE_CASE_ITEM(FP32);
     SET_DATA_TYPE_CASE_ITEM(FP64);
diff --git a/paddle/fluid/framework/paddle2cinn/transform_type.cc b/paddle/fluid/framework/paddle2cinn/transform_type.cc
index 0d4d5b57b5788..3858e2145ffa0 100644
--- a/paddle/fluid/framework/paddle2cinn/transform_type.cc
+++ b/paddle/fluid/framework/paddle2cinn/transform_type.cc
@@ -40,6 +40,7 @@ ::phi::DataType TransToPaddleDataType(const ::cinn::common::Type& type) {
   SET_TYPE_CASE_ITEM(UI32, UINT32)
   SET_TYPE_CASE_ITEM(UI64, UINT64)
 
+  SET_TYPE_CASE_ITEM(BF16, BFLOAT16)
   SET_TYPE_CASE_ITEM(F16, FLOAT16)
   SET_TYPE_CASE_ITEM(F32, FLOAT32)
   SET_TYPE_CASE_ITEM(F64, FLOAT64)
@@ -70,6 +71,9 @@ ::phi::DataType TransToPaddleDataType(const cinn_type_t& type) {
 
   SET_TYPE_CASE_ITEM(cinn_float32_t, FLOAT32)
   SET_TYPE_CASE_ITEM(cinn_float64_t, FLOAT64)
+#ifdef CINN_COMMON_BFLOAT16_H
+  SET_TYPE_CASE_ITEM(cinn_bfloat16_t, BFLOAT16)
+#endif  // CINN_COMMON_BFLOAT16_H
 #ifdef CINN_COMMON_FLOAT16_H
   SET_TYPE_CASE_ITEM(cinn_float16_t, FLOAT16)
 #endif  // CINN_COMMON_FLOAT16_H
diff --git a/paddle/fluid/framework/paddle2cinn/transform_type_test.cc b/paddle/fluid/framework/paddle2cinn/transform_type_test.cc
index 365c25b054440..f24ba8f3ac6b1 100644
--- a/paddle/fluid/framework/paddle2cinn/transform_type_test.cc
+++ b/paddle/fluid/framework/paddle2cinn/transform_type_test.cc
@@ -39,6 +39,8 @@ TEST(TransToPaddleDataType, common_type) {
             TransToPaddleDataType(::cinn::common::UI32()));
   ASSERT_EQ(::phi::DataType::UINT64,
             TransToPaddleDataType(::cinn::common::UI64()));
+  ASSERT_EQ(::phi::DataType::BFLOAT16,
+            TransToPaddleDataType(::cinn::common::BF16()));
   ASSERT_EQ(::phi::DataType::FLOAT16,
             TransToPaddleDataType(::cinn::common::F16()));
   ASSERT_EQ(::phi::DataType::FLOAT32,
diff --git a/python/setup.py.in b/python/setup.py.in
index ebf949c2b41f3..89acf5fe09d16 100644
--- a/python/setup.py.in
+++ b/python/setup.py.in
@@ -635,6 +635,11 @@ if '${WITH_CINN}' == 'ON':
         shutil.copy(cinn_fp16_file, libs_path)
         package_data['paddle.libs']+=['float16.h']
 
+    cinn_bf16_file = '${CINN_INCLUDE_DIR}/cinn/runtime/cuda/bfloat16.h'
+    if os.path.exists(cinn_bf16_file):
+        shutil.copy(cinn_bf16_file, libs_path)
+        package_data['paddle.libs']+=['bfloat16.h']
+
     if '${CMAKE_BUILD_TYPE}' == 'Release' and os.name != 'nt':
         command = "patchelf --set-rpath '$ORIGIN/' %s/${CINN_LIB_NAME}" % libs_path
         if os.system(command) != 0:
diff --git a/setup.py b/setup.py
index 297e66eba0499..104120d9e8c2b 100644
--- a/setup.py
+++ b/setup.py
@@ -1070,6 +1070,12 @@ def get_package_data_and_package_dir():
         if os.path.exists(cinn_fp16_file):
             shutil.copy(cinn_fp16_file, libs_path)
             package_data['paddle.libs'] += ['float16.h']
+        cinn_bf16_file = (
+            env_dict.get("CINN_INCLUDE_DIR") + '/cinn/runtime/cuda/bfloat16.h'
+        )
+        if os.path.exists(cinn_bf16_file):
+            shutil.copy(cinn_bf16_file, libs_path)
+            package_data['paddle.libs'] += ['bfloat16.h']
 
         if env_dict.get("CMAKE_BUILD_TYPE") == 'Release' and os.name != 'nt':
             command = (

From dc003fa3f360607ed1d13ebd7f715f7bd5031ac7 Mon Sep 17 00:00:00 2001
From: lijialin03 <124568209+lijialin03@users.noreply.github.com>
Date: Thu, 11 May 2023 22:59:33 +0800
Subject: [PATCH 289/405] revise 'Examples' of LBFGS to create right docs(cn),
 test=docs_preview (#53375)

---
 .../fluid/tests/unittests/test_lbfgs_class.py | 30 +++++++
 python/paddle/optimizer/lbfgs.py              | 87 ++++++++++++++++++-
 2 files changed, 113 insertions(+), 4 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/test_lbfgs_class.py b/python/paddle/fluid/tests/unittests/test_lbfgs_class.py
index 44c854f2119ef..6057e618d1c3a 100644
--- a/python/paddle/fluid/tests/unittests/test_lbfgs_class.py
+++ b/python/paddle/fluid/tests/unittests/test_lbfgs_class.py
@@ -555,6 +555,36 @@ def func(w, x):
 
         self.assertRaises(AssertionError, error_func3)
 
+    def test_error4(self):
+        # test call minimize(loss)
+        paddle.disable_static()
+
+        def error_func4():
+            inputs = np.random.rand(1).astype(np.float32)
+            targets = paddle.to_tensor([inputs * 2])
+            inputs = paddle.to_tensor(inputs)
+
+            extream_point = np.array([-1, 1]).astype('float32')
+
+            def func(extream_point, x):
+                return x * extream_point[0] + 5 * x * extream_point[1]
+
+            net = Net(extream_point, func)
+            opt = lbfgs.LBFGS(
+                learning_rate=1,
+                max_iter=10,
+                max_eval=None,
+                tolerance_grad=1e-07,
+                tolerance_change=1e-09,
+                history_size=5,
+                line_search_fn='strong_wolfe',
+                parameters=net.parameters(),
+            )
+            loss = train_step(inputs, targets, net, opt)
+            opt.minimize(loss)
+
+        self.assertRaises(NotImplementedError, error_func4)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/python/paddle/optimizer/lbfgs.py b/python/paddle/optimizer/lbfgs.py
index aaa933541962f..e0c5e362c4f5c 100644
--- a/python/paddle/optimizer/lbfgs.py
+++ b/python/paddle/optimizer/lbfgs.py
@@ -359,7 +359,6 @@ class LBFGS(Optimizer):
 
             import paddle
             import numpy as np
-            from paddle.incubate.optimizer import LBFGS
 
             paddle.disable_static()
             np.random.seed(0)
@@ -380,7 +379,7 @@ def forward(self, x):
                     return self.w * x
 
             net = Net()
-            opt = LBFGS(learning_rate=1, max_iter=1, max_eval=None, tolerance_grad=1e-07, tolerance_change=1e-09, history_size=100, line_search_fn='strong_wolfe', parameters=net.parameters())
+            opt = paddle.optimizer.LBFGS(learning_rate=1, max_iter=1, max_eval=None, tolerance_grad=1e-07, tolerance_change=1e-09, history_size=100, line_search_fn='strong_wolfe', parameters=net.parameters())
             def train_step(inputs, targets):
                 def closure():
                     outputs = net(inputs)
@@ -453,7 +452,46 @@ def state_dict(self):
 
         Return:
             state, a dict holding current optimization state. Its content
-                differs between optimizer classes.
+            differs between optimizer classes.
+
+        Examples:
+            .. code-block:: python
+
+                import paddle
+
+                paddle.disable_static()
+
+                net = paddle.nn.Linear(10, 10)
+                opt = paddle.optimizer.LBFGS(
+                    learning_rate=1,
+                    max_iter=1,
+                    max_eval=None,
+                    tolerance_grad=1e-07,
+                    tolerance_change=1e-09,
+                    history_size=100,
+                    line_search_fn='strong_wolfe',
+                    parameters=net.parameters(),
+                )
+
+                def train_step(inputs, targets):
+                    def closure():
+                        outputs = net(inputs)
+                        loss = paddle.nn.functional.mse_loss(outputs, targets)
+                        opt.clear_grad()
+                        loss.backward()
+                        return loss
+
+                    opt.step(closure)
+
+                inputs = paddle.rand([10, 10], dtype="float32")
+                targets = paddle.to_tensor([2 * x for x in inputs])
+
+                n_iter = 0
+                while n_iter < 20:
+                    loss = train_step(inputs, targets)
+                    n_iter = opt.state_dict()["state"]["func_evals"]
+                    print("n_iter:", n_iter)
+
         """
 
         packed_state = {}
@@ -512,9 +550,42 @@ def _directional_evaluate(self, closure, x, alpha, d):
     @framework.non_static_only
     def step(self, closure):
         """Performs a single optimization step.
+
         Args:
             closure (callable): A closure that reevaluates the model
-                and returns the loss.
+            and returns the loss.
+
+        Examples:
+            .. code-block:: python
+
+                import paddle
+
+                paddle.disable_static()
+
+                inputs = paddle.rand([10, 10], dtype="float32")
+                targets = paddle.to_tensor([2 * x for x in inputs])
+
+                net = paddle.nn.Linear(10, 10)
+                opt = paddle.optimizer.LBFGS(
+                    learning_rate=1,
+                    max_iter=1,
+                    max_eval=None,
+                    tolerance_grad=1e-07,
+                    tolerance_change=1e-09,
+                    history_size=100,
+                    line_search_fn='strong_wolfe',
+                    parameters=net.parameters(),
+                )
+
+                def closure():
+                    outputs = net(inputs)
+                    loss = paddle.nn.functional.mse_loss(outputs, targets)
+                    print("loss:", loss.item())
+                    opt.clear_grad()
+                    loss.backward()
+                    return loss
+
+                opt.step(closure)
         """
 
         with paddle.no_grad():
@@ -699,3 +770,11 @@ def obj_func(x, alpha, d):
             state['prev_loss'] = prev_loss
 
         return orig_loss
+
+    def minimize(
+        self, loss, startup_program=None, parameters=None, no_grad_set=None
+    ):
+        """Empty method. LBFGS optimizer does not use this way to minimize ``loss``. Please refer 'Examples' of LBFGS() above for usage."""
+        raise NotImplementedError(
+            "LBFGS optimizer does not use this way to minimize loss. Please refer 'Examples' of LBFGS() for usage."
+        )

From b150b1684a4eeb0b6174db4d3d67acf02729e780 Mon Sep 17 00:00:00 2001
From: xiaoxiaohehe001 <49090790+xiaoxiaohehe001@users.noreply.github.com>
Date: Thu, 11 May 2023 23:13:21 +0800
Subject: [PATCH 290/405] [Inference Zero-Dim] Support trt 0dim of gelu,
 hard_swish, hard_sigmoid and leaky_relu (#53714)

* support_act
* delete_silu
---
 paddle/fluid/inference/tensorrt/op_teller.cc  | 70 ++++---------------
 .../inference/test_trt_convert_activation.py  | 15 ++++
 test/ir/inference/test_trt_convert_gelu.py    | 14 ++--
 3 files changed, 37 insertions(+), 62 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index 414d4873de3b0..49e39d7fc1b71 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -105,15 +105,15 @@ struct SimpleOpTypeSetTeller : public Teller {
         "erf",        "floor",       "round",
         "sign",       "silu",        "logical_not",
         "reciprocal", "tanh_shrink", "logsigmoid",
-        "rsqrt",      "swish"};
+        "rsqrt",      "swish",       "hard_sigmoid",
+        "hard_swish", "leaky_relu"};
     std::unordered_set<std::string> unary_list = {
-        "exp",        "log",  "sqrt",        "abs",        "sin",
-        "cos",        "tan",  "tanh",        "sinh",       "cosh",
-        "asin",       "acos", "atan",        "asinh",      "acosh",
-        "atanh",      "ceil", "celu",        "floor",      "round",
-        "sign",       "silu", "logical_not", "reciprocal", "tanh_shrink",
-        "logsigmoid", "erf",  "bitwise_not", "equal",      "not_equal",
-        "rsqrt"};
+        "exp",   "log",         "sqrt",       "abs",         "sin",
+        "cos",   "tan",         "tanh",       "sinh",        "cosh",
+        "asin",  "acos",        "atan",       "asinh",       "acosh",
+        "atanh", "ceil",        "celu",       "floor",       "round",
+        "sign",  "logical_not", "reciprocal", "tanh_shrink", "logsigmoid",
+        "erf",   "bitwise_not", "equal",      "not_equal",   "rsqrt"};
 
     // Static shape does not support 0 or 1 dim's input.
     if (!with_dynamic_shape) {
@@ -962,20 +962,6 @@ struct SimpleOpTypeSetTeller : public Teller {
       }
     }
 
-    if (op_type == "hard_swish") {
-      if (desc.Input("X").size() != 1) {
-        VLOG(3) << "HardSwish op has only 1 input, but got "
-                << desc.Input("X").size();
-        return false;
-      }
-
-      if (desc.Output("Out").size() != 1) {
-        VLOG(3) << "HardSwish op has only 1 output, but got "
-                << desc.Output("Out").size();
-        return false;
-      }
-    }
-
     if (op_type == "squeeze2") {
       // If Attribute is Variable(s), HasAttr() will return False
       if (!desc.HasAttr("axes", /*with_attr_var=*/false)) {
@@ -1642,8 +1628,10 @@ struct SimpleOpTypeSetTeller : public Teller {
       auto x_var_name = desc.Input("X")[0];
       auto* x_var_desc = block->FindVar(x_var_name);
       const auto x_shape = x_var_desc->GetShape();
-      if (x_shape.size() == 1) {
-        VLOG(3) << "gelu op does not support input's dim is 1 in tensorrt.";
+      if (!with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
+        VLOG(3) << op_type
+                << "gelu op does not support input's dim is 1 or 0 in tensorrt "
+                   "static shape mode.";
         return false;
       }
     }
@@ -1733,20 +1721,6 @@ struct SimpleOpTypeSetTeller : public Teller {
       }
     }
 
-    if (op_type == "leaky_relu") {
-      if (desc.Input("X").size() != 1) {
-        VLOG(3) << "Invalid number of TRT leaky_relu op converter "
-                   "inputs. Expected 1, but received "
-                << desc.Input("X").size();
-        return false;
-      }
-      if (desc.Output("Out").size() != 1) {
-        VLOG(3) << "output of leaky_relu op converter should be 1, got "
-                << desc.Output("Out").size();
-        return false;
-      }
-    }
-
     if (op_type == "pad") {
       if (!desc.HasAttr("pad_value") || !desc.HasAttr("paddings")) return false;
       const float pad_value =
@@ -2388,26 +2362,6 @@ struct SimpleOpTypeSetTeller : public Teller {
       }
     }
 
-    if (op_type == "hard_sigmoid") {
-      if (!with_dynamic_shape) {
-        auto* block = desc.Block();
-        if (block == nullptr) {
-          VLOG(3) << "The block desc is nullptr, we can't continue to analyze. "
-                     "Developers need to check whether block_desc is passed in "
-                     "the pass.";
-          return false;
-        }
-        auto x_var_name = desc.Input("X")[0];
-        auto* x_var_desc = block->FindVar(x_var_name);
-        const auto x_shape = x_var_desc->GetShape();
-        if (x_shape.size() == 1) {
-          VLOG(3) << "Hard sigmoid does not support 1-dimensional input in "
-                     "tensorrt";
-          return false;
-        }
-      }
-    }
-
     if (op_type == "cast") {
 // trt 6015 result in Windows ppyolo_mbv3 TRT fp32 diff
 #if !IS_TRT_VERSION_GE(7000)
diff --git a/test/ir/inference/test_trt_convert_activation.py b/test/ir/inference/test_trt_convert_activation.py
index 4b7052e682da7..aac4fc3083bc3 100644
--- a/test/ir/inference/test_trt_convert_activation.py
+++ b/test/ir/inference/test_trt_convert_activation.py
@@ -60,6 +60,9 @@ def generate_input1(dims, batch, attrs: List[Dict[str, Any]]):
                     "logsigmoid",
                     "tanh_shrink",
                     "softplus",
+                    "hard_swish",
+                    "hard_sigmoid",
+                    "leaky_relu",
                 ]:
                     # few samples to reduce time
                     # for beta in [-0.2, 0.5, 0.67, 3]:
@@ -80,6 +83,18 @@ def generate_input1(dims, batch, attrs: List[Dict[str, Any]]):
                                 dics = [{"threshold": alpha}]
                             if op_type == "softplus":
                                 dics = [{"beta": beta}]
+                            if op_type == "hard_swish":
+                                dics = [
+                                    {
+                                        "threshold": 6.0,
+                                        "scale": 6.0,
+                                        "offset": 3.0,
+                                    }
+                                ]
+                            if op_type == "hard_sigmoid":
+                                dics = [{"slope": beta, "offset": alpha}]
+                            if op_type == "leaky_relu":
+                                dics = [{"alpha": alpha}]
 
                             ops_config = [
                                 {
diff --git a/test/ir/inference/test_trt_convert_gelu.py b/test/ir/inference/test_trt_convert_gelu.py
index 6fe2d7819de91..1f3847ff207c0 100644
--- a/test/ir/inference/test_trt_convert_gelu.py
+++ b/test/ir/inference/test_trt_convert_gelu.py
@@ -29,7 +29,9 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool:
 
     def sample_program_configs(self):
         def generate_input1(dims, attrs: List[Dict[str, Any]]):
-            if dims == 1:
+            if dims == 0:
+                return np.ones([]).astype(np.float32)
+            elif dims == 1:
                 return np.ones([32]).astype(np.float32)
             elif dims == 2:
                 return np.ones([3, 32]).astype(np.float32)
@@ -38,7 +40,7 @@ def generate_input1(dims, attrs: List[Dict[str, Any]]):
             else:
                 return np.ones([1, 3, 32, 32]).astype(np.float32)
 
-        for dims in [1, 2, 3, 4]:
+        for dims in [0, 1, 2, 3, 4]:
             for approximate in [True, False]:
                 self.dims = dims
                 dics = [{"approximate": approximate}]
@@ -70,7 +72,11 @@ def sample_predictor_configs(
         self, program_config
     ) -> (paddle_infer.Config, List[int], float):
         def generate_dynamic_shape(attrs):
-            if self.dims == 1:
+            if self.dims == 0:
+                self.dynamic_shape.min_input_shape = {"input_data": []}
+                self.dynamic_shape.max_input_shape = {"input_data": []}
+                self.dynamic_shape.opt_input_shape = {"input_data": []}
+            elif self.dims == 1:
                 self.dynamic_shape.min_input_shape = {"input_data": [1]}
                 self.dynamic_shape.max_input_shape = {"input_data": [64]}
                 self.dynamic_shape.opt_input_shape = {"input_data": [32]}
@@ -104,7 +110,7 @@ def generate_trt_nodes_num(attrs, dynamic_shape):
             runtime_version = paddle_infer.get_trt_runtime_version()
             self.assertTrue(compile_version == runtime_version)
             # Dimension one only runs on Paddle OP
-            if self.dims == 1:
+            if not dynamic_shape and (self.dims == 1 or self.dims == 0):
                 return 0, 3
             if compile_version >= valid_version:
                 return 1, 2

From 13cdaab630bce92a4dd94a9a63aa31467eaa68c3 Mon Sep 17 00:00:00 2001
From: houj04 <35131887+houj04@users.noreply.github.com>
Date: Fri, 12 May 2023 09:00:58 +0800
Subject: [PATCH 291/405] [XPU] bkcl_broadcast support int64_t (#53720)

---
 .../collective/process_group_bkcl.cc          | 34 +++++++++++++------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/paddle/fluid/distributed/collective/process_group_bkcl.cc b/paddle/fluid/distributed/collective/process_group_bkcl.cc
index a3c3e085c6df5..99323d9d45ad5 100644
--- a/paddle/fluid/distributed/collective/process_group_bkcl.cc
+++ b/paddle/fluid/distributed/collective/process_group_bkcl.cc
@@ -334,16 +334,30 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::Broadcast(
                 << ", root: " << root << ", numel: " << input.numel()
                 << ", dtype: " << input.type() << ", sync_op: " << sync_op
                 << ", use_calc_stream: " << use_calc_stream;
-        int r =
-            bkcl_broadcast(comm,
-                           input.data(),
-                           output->data(),
-                           input.numel(),
-                           platform::ToBKCLDataType(
-                               framework::TransToProtoVarType(input.type())),
-                           root,
-                           stream);
-        return r;
+        if (framework::TransToProtoVarType(input.dtype()) ==
+            framework::proto::VarType::INT64) {
+          // special for int64_t, send as int32_t with DOUBLE NUMEL
+          int r = bkcl_broadcast(
+              comm,
+              input.data(),
+              output->data(),
+              input.numel() * 2,
+              platform::ToBKCLDataType(framework::proto::VarType::INT32),
+              root,
+              stream);
+          return r;
+        } else {
+          int r =
+              bkcl_broadcast(comm,
+                             input.data(),
+                             output->data(),
+                             input.numel(),
+                             platform::ToBKCLDataType(
+                                 framework::TransToProtoVarType(input.type())),
+                             root,
+                             stream);
+          return r;
+        }
       },
       CommType::BROADCAST,
       sync_op,

From 6cd7609a735bc3b211dfc1822f48bf70952a103e Mon Sep 17 00:00:00 2001
From: Wangzheee <634486483@qq.com>
Date: Fri, 12 May 2023 10:40:13 +0800
Subject: [PATCH 292/405] fix gpu mem alloc: use phi::memory_utils::Alloc
 (#53721)

---
 .../plugin/fused_token_prune_op_plugin.cu       | 17 +++++++++--------
 .../transformer_input_output_convert_plugin.cu  | 15 +++++++++------
 2 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/plugin/fused_token_prune_op_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/fused_token_prune_op_plugin.cu
index ebc9f2280268c..8e934c1a3326d 100644
--- a/paddle/fluid/inference/tensorrt/plugin/fused_token_prune_op_plugin.cu
+++ b/paddle/fluid/inference/tensorrt/plugin/fused_token_prune_op_plugin.cu
@@ -18,6 +18,9 @@
 
 #include "paddle/phi/kernels/funcs/math_function.h"
 
+#include "paddle/fluid/platform/device/gpu/gpu_info.h"
+#include "paddle/phi/common/memory_utils.h"
+
 #include "paddle/fluid/framework/tensor.h"
 #include "paddle/fluid/framework/tensor_util.h"
 #include "paddle/fluid/platform/device_context.h"
@@ -381,18 +384,16 @@ int FusedTokenPrunePluginDynamic::enqueue(
 
     // 3. compute new pos id
     // Determine temporary device storage requirements
-    void* d_temp_storage = NULL;
     size_t temp_storage_bytes = 0;
-    cub::DeviceScan::ExclusiveSum(d_temp_storage,
-                                  temp_storage_bytes,
-                                  pruned_token_lengths_,
-                                  output3,
-                                  B + 1);
+    cub::DeviceScan::ExclusiveSum(
+        NULL, temp_storage_bytes, pruned_token_lengths_, output3, B + 1);
     // Allocate temporary storage
-    cudaMalloc(&d_temp_storage, temp_storage_bytes);
+
+    platform::CUDAPlace place(platform::GetCurrentDeviceId());
+    auto d_temp_storage = phi::memory_utils::Alloc(place, temp_storage_bytes);
 
     // Run exclusive prefix sum
-    cub::DeviceScan::ExclusiveSum(d_temp_storage,
+    cub::DeviceScan::ExclusiveSum(d_temp_storage->ptr(),
                                   temp_storage_bytes,
                                   pruned_token_lengths_,
                                   output3,
diff --git a/paddle/fluid/inference/tensorrt/plugin/transformer_input_output_convert_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/transformer_input_output_convert_plugin.cu
index 39e2a0b422463..bc197a1423d49 100644
--- a/paddle/fluid/inference/tensorrt/plugin/transformer_input_output_convert_plugin.cu
+++ b/paddle/fluid/inference/tensorrt/plugin/transformer_input_output_convert_plugin.cu
@@ -12,8 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/inference/tensorrt/plugin/transformer_input_output_convert_plugin.h"
+#include "paddle/fluid/platform/device/gpu/gpu_info.h"
+#include "paddle/fluid/platform/device_context.h"
+#include "paddle/phi/common/memory_utils.h"
+
 #include "cub/cub.cuh"
+#include "paddle/fluid/inference/tensorrt/plugin/transformer_input_output_convert_plugin.h"
 
 namespace paddle {
 namespace inference {
@@ -178,16 +182,15 @@ int TransformerInputConvertPlugin::enqueue(
   const int32_t HiddenSize = input0_desc.dims.d[2];  // hidden size
 
   // Determine temporary device storage requirements
-  void* d_temp_storage = NULL;
   size_t temp_storage_bytes = 0;
   cub::DeviceScan::ExclusiveSum(
-      d_temp_storage, temp_storage_bytes, input1, output2, B + 1);
+      NULL, temp_storage_bytes, input1, output2, B + 1);
   // Allocate temporary storage
-  cudaMalloc(&d_temp_storage, temp_storage_bytes);
-
+  platform::CUDAPlace place(platform::GetCurrentDeviceId());
+  auto d_temp_storage = phi::memory_utils::Alloc(place, temp_storage_bytes);
   // Run exclusive prefix sum
   cub::DeviceScan::ExclusiveSum(
-      d_temp_storage, temp_storage_bytes, input1, output2, B + 1);
+      d_temp_storage->ptr(), temp_storage_bytes, input1, output2, B + 1);
   const int32_t vector_length = HiddenSize;
   int32_t num_threads;
   if (vector_length < 1024) {

From 92db839f090933a55076dec1c45cd6acbab2907d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=82=85=E5=89=91=E5=AF=92?= <Xs1580802568@gmail.com>
Date: Fri, 12 May 2023 10:47:46 +0800
Subject: [PATCH 293/405] fix er error msg of index_put Op (#53717)

---
 paddle/phi/kernels/cpu/index_put_grad_kernel.cc | 2 +-
 paddle/phi/kernels/cpu/index_put_kernel.cc      | 2 +-
 paddle/phi/kernels/gpu/index_put_grad_kernel.cu | 2 +-
 paddle/phi/kernels/gpu/index_put_kernel.cu      | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/paddle/phi/kernels/cpu/index_put_grad_kernel.cc b/paddle/phi/kernels/cpu/index_put_grad_kernel.cc
index 7374bcd403d12..7c8ac0624e2d8 100644
--- a/paddle/phi/kernels/cpu/index_put_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/index_put_grad_kernel.cc
@@ -183,7 +183,7 @@ void IndexPutGradKernel(const Context& dev_ctx,
       x.dtype(),
       value.dtype(),
       phi::errors::InvalidArgument(
-          "The data type of tensor in indices must be same to the data type "
+          "The data type of tensor value must be same to the data type "
           "of tensor x."));
   std::vector<DenseTensor> tmp_args;
   std::vector<const phi::DenseTensor*> int_indices_v =
diff --git a/paddle/phi/kernels/cpu/index_put_kernel.cc b/paddle/phi/kernels/cpu/index_put_kernel.cc
index da3e37ac24236..b35f2e1982ec3 100644
--- a/paddle/phi/kernels/cpu/index_put_kernel.cc
+++ b/paddle/phi/kernels/cpu/index_put_kernel.cc
@@ -102,7 +102,7 @@ void IndexPutKernel(const Context& dev_ctx,
       x.dtype(),
       value.dtype(),
       phi::errors::InvalidArgument(
-          "The data type of tensor in indices must be same to the data type "
+          "The data type of tensor value must be same to the data type "
           "of tensor x."));
   PADDLE_ENFORCE_EQ(indices.empty(),
                     false,
diff --git a/paddle/phi/kernels/gpu/index_put_grad_kernel.cu b/paddle/phi/kernels/gpu/index_put_grad_kernel.cu
index 7ae1e42c067cb..9dca49ee7ff0f 100644
--- a/paddle/phi/kernels/gpu/index_put_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/index_put_grad_kernel.cu
@@ -214,7 +214,7 @@ void IndexPutGradKernel(const Context& dev_ctx,
       x.dtype(),
       value.dtype(),
       phi::errors::InvalidArgument(
-          "The data type of tensor in indices must be same to the data type "
+          "The data type of tensor value must be same to the data type "
           "of tensor x."));
   std::vector<DenseTensor> tmp_args;
   std::vector<const phi::DenseTensor*> int_indices_v =
diff --git a/paddle/phi/kernels/gpu/index_put_kernel.cu b/paddle/phi/kernels/gpu/index_put_kernel.cu
index ad27993c35244..fd4476fe11d98 100644
--- a/paddle/phi/kernels/gpu/index_put_kernel.cu
+++ b/paddle/phi/kernels/gpu/index_put_kernel.cu
@@ -110,7 +110,7 @@ void IndexPutKernel(const Context& dev_ctx,
       x.dtype(),
       value.dtype(),
       phi::errors::InvalidArgument(
-          "The data type of tensor in indices must be same to the data type "
+          "The data type of tensor value must be same to the data type "
           "of tensor x."));
   PADDLE_ENFORCE_EQ(indices.empty(),
                     false,

From 3e3297c752eda5d64a4f268725dd8b35cd809467 Mon Sep 17 00:00:00 2001
From: HydrogenSulfate <490868991@qq.com>
Date: Fri, 12 May 2023 10:48:48 +0800
Subject: [PATCH 294/405] fix jacobian and hessian's docstring (#53732)

* fix jacobian and hessian's docstring

* fix hessian's docstring

* fix hessian's docstring
---
 python/paddle/autograd/autograd.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/python/paddle/autograd/autograd.py b/python/paddle/autograd/autograd.py
index cfd5442138c3a..e61a7ff009294 100644
--- a/python/paddle/autograd/autograd.py
+++ b/python/paddle/autograd/autograd.py
@@ -465,12 +465,12 @@ def jacobian(
     The ``xs`` tuples are identical in one-to-one correspondence.
 
     - When ``batch_axis=None``, only 0-dimensional Tensor or 1-dimensional Tensor is
-          supported, assuming the shape of ``xs`` is ``[N, ]``, the shape of ``ys`` is
-          ``[M, ]``, then the output Jacobian matrix shape is ``[M, N]``.
+      supported, assuming the shape of ``xs`` is ``[N, ]``, the shape of ``ys`` is
+      ``[M, ]``, then the output Jacobian matrix shape is ``[M, N]``.
 
     - When ``batch_axis=0``, only 1-dimensional Tensor or 2-dimensional Tensor is
-          supported, assuming the shape of ``xs`` is ``[B, N]``, The shape of ``ys`` is
-          ``[B, M]``, then the output Jacobian matrix shape is ``[B, M, N]``.
+      supported, assuming the shape of ``xs`` is ``[B, N]``, The shape of ``ys`` is
+      ``[B, M]``, then the output Jacobian matrix shape is ``[B, M, N]``.
 
     After the ``Jacobian`` object is created, the actual calculation process does not
     occur, but the lazy evaluation method is used for calculation. It can be
@@ -553,15 +553,14 @@ def hessian(
     ``batch_axis`` means The position of the batch dimension of the parameter data.
 
     When the input ``xs`` is a Tensor tuple, the returned result is a ``Hessian`` tuple,
-    assuming that the internal shape of the ``xs`` tuple is composed of
-    ``([M1, ], [M2, ]) ``, the shape of the returned result consists of
-    ``(([M1, M1], [M1, M2]), ([M2, M1], [M2, M2]))``
+    assuming that the internal shape of the ``xs`` tuple is composed of ``([M1, ], [M2, ])``, the shape of the returned
+    result consists of ``(([M1, M1], [M1, M2]), ([M2, M1], [M2, M2]))``
 
     - When ``batch_axis=None``, only 0-dimensional Tensor or 1-dimensional Tensor is
-        supported, assuming that the shape of ``xs`` is ``[N, ]``, and the shape of ``ys`` is ``[ ]``(0-dimensional Tensor), the final output is a single Hessian matrix whose shape is ``[N, N]``.
+      supported, assuming that the shape of ``xs`` is ``[N, ]``, and the shape of ``ys`` is ``[ ]`` (0-dimensional Tensor), the final output is a single Hessian matrix whose shape is ``[N, N]``.
 
     - When ``batch_axis=0``, only 1-dimensional Tensor or 2-dimensional Tensor is
-        supported, assuming that the shape of ``xs`` is ``[B, N]``, and the shape of ``ys`` is `` [B, ]``, the final output Jacobian matrix shape is ``[B, N, N]``.
+      supported, assuming that the shape of ``xs`` is ``[B, N]``, and the shape of ``ys`` is ``[B, ]``, the final output Jacobian matrix shape is ``[B, N, N]``.
 
     After the ``Hessian`` object is created, the complete calculation process does not
     occur, but a partial lazy evaluation method is used for calculation. It can be

From b73594b4785fdf8a7a54e12b578bd83bad0c8924 Mon Sep 17 00:00:00 2001
From: Xiaoxu Chen <chenxx_id@163.com>
Date: Fri, 12 May 2023 10:51:20 +0800
Subject: [PATCH 295/405] =?UTF-8?q?=E3=80=90Prim=E3=80=91support=20=20high?=
 =?UTF-8?q?er=20order=20autodiff=20for=20dy2static+composite=20(#53171)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* [Dy2St]Fix x grad names when high order gradient

* Polish error msg

* Add inputs var to backward in dy2st

* Fix error

* Get grad names for backward API

* Fix save load

* Polish code

* Add ut

* [prim] fix not support optional grad bugs in higher order autodiff

* [prim] remove duplicate fill_any_like caused by infershape_for_composite

* fix _strip_grad_suffix_ bugs in higher-order autodiff

* [prim] create output for test_static_prim.cc

---------

Co-authored-by: 0x45f <wangzhen45@baidu.com>
---
 .../utils/static/composite_grad_desc_maker.h  |  7 +-
 python/paddle/fluid/backward.py               | 73 ++++++++++++-------
 .../tests/unittests/test_calc_gradient.py     |  2 +
 test/cpp/prim/test_static_prim.cc             | 23 ++++++
 ...test_comp_get_grad_op_desc_prim_enabled.py |  5 ++
 test/prim/test_comp_skip_op_set.py            |  5 ++
 6 files changed, 83 insertions(+), 32 deletions(-)

diff --git a/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h b/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h
index 1ef5161dc0714..a8fe8d3e59616 100644
--- a/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h
+++ b/paddle/fluid/prim/utils/static/composite_grad_desc_maker.h
@@ -350,10 +350,7 @@ class CompositeGradOpMakerBase {
   framework::VarDesc* SingleOutputGrad(const std::string& name) const {
     auto* var = this->SingleForwardOutput(name);
     if (!var) {
-      PADDLE_THROW(platform::errors::InvalidArgument(
-          "GetSingleOutputGrad for %s_grad faild, if it is Optional input,"
-          "please use GetOptionalSingleOutputGrad replaced. ",
-          name));
+      return nullptr;
     }
     auto var_name = var->Name();
     auto grad_var_name = framework::GradVarName(var_name);
@@ -371,7 +368,7 @@ class CompositeGradOpMakerBase {
       return StaticCompositeContext::Instance().GetBlock()->FindVar(
           grad_var_name);
     } else {
-      return StaticCompositeContext::Instance().GetBlock()->Var(grad_var_name);
+      return nullptr;
     }
   }
 
diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py
index e6c01f836851a..ff57966eefbfb 100755
--- a/python/paddle/fluid/backward.py
+++ b/python/paddle/fluid/backward.py
@@ -28,6 +28,8 @@
 
 from collections.abc import Sequence
 
+import re
+
 __all__ = [
     'append_backward',
     'gradients',
@@ -459,10 +461,14 @@ def _strip_grad_suffix_(name):
     """
     Strip the grad suffix from the given variable name
     e.g. x@GRAD ==> x
+         x@GRAD@GRAD ==> x
          y@GRAD@RENAME@1 ==> y
+         z@GRAD_slice_0@GRAD ==> z@GRAD_slice_0
     """
-    pos = name.find(core.grad_var_suffix())
-    new_name = name[:pos] if pos != -1 else name
+    pos = re.search(f'{core.grad_var_suffix()}$', name) or re.search(
+        f'{core.grad_var_suffix()}@', name
+    )
+    new_name = name[: pos.start()] if pos is not None else name
     new_pos = name.rfind('grad/')
     return new_name[new_pos + 5 :] if new_pos != -1 else new_name
 
@@ -1343,15 +1349,17 @@ def update_distop_context(
 
     if core._is_bwd_prim_enabled():
         composite_block = program.clone().current_block()
-        # Infer shape for operators whose output haven't been created.
+        # Create output and infer shape for operators whose output haven't
+        # been created.
         for op in composite_block.ops:
-            if not all(
-                tuple(
-                    composite_block._find_var_recursive(arg)
-                    for arg in op.output_arg_names
-                )
-            ):
-                infershape_for_composite(composite_block, op.desc)
+            for name in op.output_arg_names:
+                if not (
+                    composite_block.desc.has_var_recursive(name.encode())
+                    or name == core.empty_var_name()
+                ):
+                    composite_block.create_var(name=name)
+            op.desc.infer_var_type(composite_block.desc)
+            op.desc.infer_shape(composite_block.desc)
 
     # add grad_op_desc by reversed ops
     for op in reversed(ops):
@@ -1492,6 +1500,15 @@ def find_op_index(block_desc, cur_op_desc):
                 or name in input_grad_names_set
             )
             is_append_grad = False
+
+            # NOTE: In primitive mode, the intermediate variable generated by
+            # decompositing raw grad op are not satisfied the rule of 'XX@GRAD',
+            # which will cause it be pruned according to current pruning logic.
+            # For simplicity, we treate all prmitive operators as one raw
+            # operator, and keep the pruning logic consistent with currently
+            # logic. The drawback of this solution is may lead to some primitive
+            # operators are not pruned, which is needed to fixed.
+            # FIXME: Optimize pruning logic from the perspective of whole graph.
             input_grad_names = []
             for op_desc in grad_op_desc:
                 input_grad_names += [
@@ -1499,20 +1516,20 @@ def find_op_index(block_desc, cur_op_desc):
                     for name in op_desc.input_arg_names()
                     if is_grad_name(name)
                 ]
+
+            # some code of gradient ops, like increment, are not very
+            # standard, there is no @GRAD in these ops' inputs.
             if len(input_grad_names) == 0:
                 is_append_grad = True
-                break
-
-            for op_desc in grad_op_desc:
-
-                # some code of gradient ops, like increment, are not very
-                # standard, there is no @GRAD in these ops' inputs.
+                continue
 
-                if _some_in_set_(input_grad_names, input_grad_names_set):
+            if _some_in_set_(input_grad_names, input_grad_names_set):
+                is_append_grad = True
+                for op_desc in grad_op_desc:
                     grad_op_descs.append(op_desc)
-                    is_append_grad = True
                     for name in op_desc.output_arg_names():
                         input_grad_names_set.add(name)
+
             if is_append_grad:
                 grad_to_var.update(op_grad_to_var)
         else:
@@ -1774,17 +1791,19 @@ def infershape_for_composite(block, grad_op_desc):
         op_desc.check_attrs()
         op_desc.infer_var_type(block.desc)
         op_desc.infer_shape(block.desc)
-        for arg in op_desc.output_arg_names():
-            if arg in new_vars:
-                _infer_var_data_type_shape_(arg, block)
-
         grad_op_desc.copy_from(op_desc)
 
-    # NOTE: Some operator doesn't infer dtype correctly, this patch set the
-    # grad_var dtype same with corresponding forward variable.
-    for arg in grad_op_desc.output_arg_names():
-        if arg in new_vars:
-            _infer_var_data_type_shape_(arg, block)
+    if not framework.OpProtoHolder.instance().has_op_proto(grad_op_desc.type()):
+        # NOTE: Some raw fluid grad operators which hadn't been decomposed may not
+        # implement InferVarType method, such as elementwise_xx_grad, and it will
+        # cause the dtype or shape of corresponding cotangent incorrect. This
+        # patch set the cotangent dtype and shape same with corresponding
+        # forward variable. For primitive operators, we have ensure all
+        # InferVarType method to be executed correctly in PR#52818, we skip
+        # this patch for primitive operators.
+        for arg in grad_op_desc.output_arg_names():
+            if arg in new_vars:
+                _infer_var_data_type_shape_(arg, block)
 
 
 def _rename_grad_(
diff --git a/python/paddle/fluid/tests/unittests/test_calc_gradient.py b/python/paddle/fluid/tests/unittests/test_calc_gradient.py
index 9a4aa568833f2..4cecbd5273650 100644
--- a/python/paddle/fluid/tests/unittests/test_calc_gradient.py
+++ b/python/paddle/fluid/tests/unittests/test_calc_gradient.py
@@ -20,6 +20,8 @@
 from paddle import fluid
 from paddle.fluid.backward import calc_gradient
 
+paddle.enable_static()
+
 
 class TestCalcGradient(unittest.TestCase):
     def test_calc_gradient(self):
diff --git a/test/cpp/prim/test_static_prim.cc b/test/cpp/prim/test_static_prim.cc
index 936ae9babfe36..04305e8ffded1 100644
--- a/test/cpp/prim/test_static_prim.cc
+++ b/test/cpp/prim/test_static_prim.cc
@@ -206,6 +206,11 @@ TEST(StaticPrim, TanhBackwardComposite) {
   auto* forward_opdesc = target_block->AllOps()[0];
   std::unordered_map<std::string, std::string> grad_to_var;
   std::vector<framework::BlockDesc*> grad_sub_block;
+  Tensor out_grad = prim::empty<prim::DescTensor>(
+      shape, phi::DataType::FLOAT32, paddle::Place());
+  framework::VarDesc* out_grad_desc =
+      static_cast<prim::DescTensor*>(out_grad.impl().get())->get_ptr();
+  target_block->RenameVar(out_grad_desc->Name(), "b@GRAD");
   std::vector<std::unique_ptr<framework::OpDesc>> grad_ops =
       std::move(framework::OpInfoMap::Instance()
                     .Get(forward_opdesc->Type())
@@ -288,6 +293,11 @@ TEST(StaticCompositeGradMaker, TestMutiInputMethod) {
   auto* forward_opdesc = target_block->AllOps()[0];
   std::unordered_map<std::string, std::string> grad_to_var;
   std::vector<framework::BlockDesc*> grad_sub_block;
+  Tensor out_grad = prim::empty<prim::DescTensor>(
+      shape, phi::DataType::FLOAT32, paddle::Place());
+  framework::VarDesc* out_grad_desc =
+      static_cast<prim::DescTensor*>(out_grad.impl().get())->get_ptr();
+  target_block->RenameVar(out_grad_desc->Name(), "out@GRAD");
   auto test = TestCompositeGradMaker(*forward_opdesc,
                                      std::unordered_set<std::string>(),
                                      &grad_to_var,
@@ -353,6 +363,19 @@ TEST(StaticCompositeGradMaker, TestMutiOutputMethod) {
   auto* forward_opdesc = target_block->AllOps()[0];
   std::unordered_map<std::string, std::string> grad_to_var;
   std::vector<framework::BlockDesc*> grad_sub_block;
+
+  Tensor out1_grad = prim::empty<prim::DescTensor>(
+      shape, phi::DataType::FLOAT32, paddle::Place());
+  framework::VarDesc* out1_grad_desc =
+      static_cast<prim::DescTensor*>(out1_grad.impl().get())->get_ptr();
+  target_block->RenameVar(out1_grad_desc->Name(), "out1@GRAD");
+
+  Tensor out2_grad = prim::empty<prim::DescTensor>(
+      shape, phi::DataType::FLOAT32, paddle::Place());
+  framework::VarDesc* out2_grad_desc =
+      static_cast<prim::DescTensor*>(out2_grad.impl().get())->get_ptr();
+  target_block->RenameVar(out2_grad_desc->Name(), "out2@GRAD");
+
   auto test = TestCompositeGradMaker(*forward_opdesc,
                                      std::unordered_set<std::string>(),
                                      &grad_to_var,
diff --git a/test/prim/test_comp_get_grad_op_desc_prim_enabled.py b/test/prim/test_comp_get_grad_op_desc_prim_enabled.py
index 0bc173ef4049b..9c89ab72bf18e 100644
--- a/test/prim/test_comp_get_grad_op_desc_prim_enabled.py
+++ b/test/prim/test_comp_get_grad_op_desc_prim_enabled.py
@@ -67,6 +67,11 @@ def setUpClass(cls):
                 for n, vs in cls.outputs.items()
             },
         )
+
+        for _, outs in cls.outputs.items():
+            for out in outs:
+                block.create_var(name=out + core.grad_var_suffix())
+
         cls.fwd = block.ops[0].desc
 
     @classmethod
diff --git a/test/prim/test_comp_skip_op_set.py b/test/prim/test_comp_skip_op_set.py
index bd98f21f235c9..bca3c97ec4d40 100644
--- a/test/prim/test_comp_skip_op_set.py
+++ b/test/prim/test_comp_skip_op_set.py
@@ -46,6 +46,11 @@ def setUp(self):
                 for n, vs in self.outputs.items()
             },
         )
+
+        for _, outs in self.outputs.items():
+            for out in outs:
+                block.create_var(name=out + core.grad_var_suffix())
+
         self.fwd = block.ops[0].desc
 
     def tearDown(self):

From 58916e316602bf11024ce6e1451a5757c402f005 Mon Sep 17 00:00:00 2001
From: Ruibiao Chen <chenruibiao@baidu.com>
Date: Fri, 12 May 2023 11:12:22 +0800
Subject: [PATCH 296/405] Skip fake alloc in static build for some
 communication OPs (#53593)

* Skip fake alloc in static build for depend and nop op

* Skip communication op

* Skip sync op
---
 .../new_executor/interpreter/interpreter_util.cc  |  2 +-
 .../new_executor/interpreter/static_build.cc      | 15 ++++++++++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
index 99c1ba3d98395..2db10435fdae4 100644
--- a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
+++ b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
@@ -445,7 +445,7 @@ void BuildOpFuncList(const platform::Place& place,
   // Step 1: create all ops for current block.
   CreateAllOps(block, &ops_unique);
 
-  VLOG(4) << "Static build: " << static_build;
+  VLOG(1) << "Static build: " << static_build;
 
   if (!execution_config.used_for_jit) {
     // If gc is enabled and block size > 1
diff --git a/paddle/fluid/framework/new_executor/interpreter/static_build.cc b/paddle/fluid/framework/new_executor/interpreter/static_build.cc
index d48f11cf906a7..d91c59f60627b 100644
--- a/paddle/fluid/framework/new_executor/interpreter/static_build.cc
+++ b/paddle/fluid/framework/new_executor/interpreter/static_build.cc
@@ -25,7 +25,20 @@ std::set<std::string> OperatorBasesMustRunInStaticBuild = {
     "create_double_buffer_reader", "create_py_reader"};
 
 std::set<std::string> OpsCanSkipedFakeAllocInStaticBuild = {
-    "create_double_buffer_reader", "create_py_reader", "fetch_v2"};
+    "c_comm_init",
+    "c_comm_init_all",
+    "c_comm_init_multitrainer",
+    "c_gen_bkcl_id",
+    "c_gen_nccl_id",
+    "c_sync_calc_stream",
+    "c_sync_comm_stream",
+    "c_wait_comm",
+    "c_wait_compute",
+    "create_double_buffer_reader",
+    "create_py_reader",
+    "depend",
+    "fetch_v2",
+    "nop"};
 
 // Cannot static analysis these Ops' output dtype or backend because their
 // kernels have not moved to PHI yet.

From 95ae5d5ce77de7508c1dfe9fa258e80c1f213daa Mon Sep 17 00:00:00 2001
From: Aurelius84 <zhangliujie@baidu.com>
Date: Fri, 12 May 2023 11:24:53 +0800
Subject: [PATCH 297/405] Revert "[CINN]Adjust Bert unittest loss ground truth
 (#53628)" (#53731)

This reverts commit 45ce0ad536cd7e189bed2e2a82d3c6077009371d.
---
 test/prim/model/test_bert_prim_cinn.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/test/prim/model/test_bert_prim_cinn.py b/test/prim/model/test_bert_prim_cinn.py
index aed75a9ca5dc3..c6438536673f6 100644
--- a/test/prim/model/test_bert_prim_cinn.py
+++ b/test/prim/model/test_bert_prim_cinn.py
@@ -59,16 +59,16 @@
 ]
 
 DY2ST_PRIM_CINN_GT = [
-    10.861940383911133,
-    10.350666046142578,
-    10.331628799438477,
-    10.267815589904785,
-    10.22376537322998,
-    10.198311805725098,
-    10.16231632232666,
-    10.10612678527832,
-    10.122684478759766,
-    10.0067138671875,
+    10.976988792419434,
+    10.345282554626465,
+    10.33430004119873,
+    10.275235176086426,
+    10.22808837890625,
+    10.200264930725098,
+    10.161019325256348,
+    10.10872745513916,
+    10.121539115905762,
+    9.990568161010742,
 ]
 
 

From 4e416c9930dee59b7d96901fc3bcc6fb3b254bb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=82=85=E5=89=91=E5=AF=92?= <Xs1580802568@gmail.com>
Date: Fri, 12 May 2023 11:34:00 +0800
Subject: [PATCH 298/405] fix doc eror of index_put in develop (#53727)

* fix doc eror of index_put in develop

* fix doc error for index_put; test=document_fix; test=docs_preview
---
 python/paddle/tensor/manipulation.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py
index 7fe717383cd3b..e6bf626cb1a4c 100644
--- a/python/paddle/tensor/manipulation.py
+++ b/python/paddle/tensor/manipulation.py
@@ -4805,7 +4805,7 @@ def index_put_(x, indices, value, accumulate=False, name=None):
     Args:
         x (Tensor) : The Source Tensor. Supported data types are int32, int64, float16, float32, float64, bool.
         indices (Tuple of Tensor): The tuple of Tensor containing the indices to index.
-            The data type of ``tensor in indices`` must be int32, int64 or bool
+            The data type of ``tensor in indices`` must be int32, int64 or bool.
         value (Tensor): The tensor used to be assigned to x.
         accummulate (Bool, optional): Whether the elements in values are added to x. Default: False.
         name(str, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
@@ -4815,6 +4815,7 @@ def index_put_(x, indices, value, accumulate=False, name=None):
 
     Examples:
         .. code-block:: python
+
             import paddle
 
             x = paddle.zeros([3, 3])
@@ -4845,6 +4846,7 @@ def index_put(x, indices, value, accumulate=False, name=None):
 
     Examples:
         .. code-block:: python
+
             import paddle
 
             x = paddle.zeros([3, 3])

From 348565b04bd51a5af8ebebb39b2fbe75135041da Mon Sep 17 00:00:00 2001
From: huangjiyi <43315610+huangjiyi@users.noreply.github.com>
Date: Fri, 12 May 2023 11:40:06 +0800
Subject: [PATCH 299/405] move pow2_decay_with_linear_warmup kernel to phi
 (#53741)

* update

* update
---
 .../pow2_decay_with_linear_warmup_op.cc       |   7 -
 .../pow2_decay_with_linear_warmup_op.h        | 125 ------------------
 .../pow2_decay_with_linear_warmup_op_xpu.cc   |  84 ------------
 .../pow2_decay_with_linear_warmup_kernel.cc}  |  19 +--
 .../pow2_decay_with_linear_warmup_kernel.cu   |  25 ++++
 ...ow2_decay_with_linear_warmup_kernel_impl.h | 110 +++++++++++++++
 .../pow2_decay_with_linear_warmup_kernel.h    |  31 +++++
 .../pow2_decay_with_linear_warmup_kernel.cc   |  71 ++++++++++
 .../pow2_decay_with_linear_warmup_sig.cc      |  30 +++++
 9 files changed, 277 insertions(+), 225 deletions(-)
 delete mode 100644 paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.h
 delete mode 100644 paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op_xpu.cc
 rename paddle/{fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.cu => phi/kernels/cpu/pow2_decay_with_linear_warmup_kernel.cc} (54%)
 create mode 100644 paddle/phi/kernels/gpu/pow2_decay_with_linear_warmup_kernel.cu
 create mode 100644 paddle/phi/kernels/impl/pow2_decay_with_linear_warmup_kernel_impl.h
 create mode 100644 paddle/phi/kernels/pow2_decay_with_linear_warmup_kernel.h
 create mode 100644 paddle/phi/kernels/xpu/pow2_decay_with_linear_warmup_kernel.cc
 create mode 100644 paddle/phi/ops/compat/pow2_decay_with_linear_warmup_sig.cc

diff --git a/paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.cc b/paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.cc
index 8def9c961f757..105c698355e6d 100644
--- a/paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.cc
+++ b/paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.cc
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.h"
-
 #include "paddle/fluid/framework/op_registry.h"
 #include "paddle/fluid/platform/float16.h"
 
@@ -78,12 +76,7 @@ When step_num > total_steps, lr = end_lr
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-namespace plat = paddle::platform;
 
 REGISTER_OP_WITHOUT_GRADIENT(pow2_decay_with_linear_warmup,
                              ops::Pow2DecayWithLinearWarmupOp,
                              ops::Pow2DecayWithLinearWarmupOpMaker);
-REGISTER_OP_CPU_KERNEL(
-    pow2_decay_with_linear_warmup,
-    ops::Pow2DecayWithLinearWarmupOpKernel<phi::CPUContext, double>,
-    ops::Pow2DecayWithLinearWarmupOpKernel<phi::CPUContext, float>);
diff --git a/paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.h b/paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.h
deleted file mode 100644
index 8f3be79cd4c8d..0000000000000
--- a/paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.h
+++ /dev/null
@@ -1,125 +0,0 @@
-// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#pragma once
-
-#include "paddle/fluid/framework/operator.h"
-#include "paddle/fluid/framework/tensor.h"
-#include "paddle/fluid/platform/for_range.h"
-#include "paddle/phi/core/macros.h"
-
-namespace paddle {
-namespace operators {
-
-template <typename T, typename AttrT>
-struct Pow2DecayWithLinearWarmupFunctor {
-  template <typename U>
-  using RestrictPtr = U *PADDLE_RESTRICT;
-
- public:
-  HOSTDEVICE Pow2DecayWithLinearWarmupFunctor(RestrictPtr<T> lr,
-                                              RestrictPtr<int64_t> step,
-                                              size_t warmup_steps,
-                                              size_t total_steps,
-                                              AttrT base_lr,
-                                              AttrT end_lr)
-      : lr_(lr),
-        step_(step),
-        warmup_steps_(warmup_steps),
-        total_steps_(total_steps),
-        base_lr_(base_lr),
-        end_lr_(end_lr) {}
-
-  HOSTDEVICE void operator()(size_t) const {
-    size_t step = static_cast<size_t>(*step_) + 1;
-    *step_ = static_cast<int64_t>(step);
-    if (step <= warmup_steps_) {
-      auto new_lr = static_cast<double>(step) / warmup_steps_ * base_lr_;
-      *lr_ = static_cast<T>(new_lr);
-    } else if (step < total_steps_) {
-      auto factor = 1 - static_cast<double>(step - warmup_steps_) /
-                            (total_steps_ - warmup_steps_);
-      auto new_lr =
-          static_cast<double>(base_lr_ - end_lr_) * (factor * factor) + end_lr_;
-      *lr_ = static_cast<T>(new_lr);
-    } else {
-      *lr_ = static_cast<T>(end_lr_);
-    }
-  }
-
- private:
-  RestrictPtr<T> lr_;
-  RestrictPtr<int64_t> step_;
-  size_t warmup_steps_;
-  size_t total_steps_;
-  AttrT base_lr_;
-  AttrT end_lr_;
-};
-
-template <typename DeviceContext, typename T>
-class Pow2DecayWithLinearWarmupOpKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext &ctx) const {
-    const auto *lr = ctx.Input<phi::DenseTensor>("LearningRate");
-    const auto *step = ctx.Input<phi::DenseTensor>("Step");
-    auto *lr_out = ctx.Output<phi::DenseTensor>("LearningRateOut");
-    auto *step_out = ctx.Output<phi::DenseTensor>("StepOut");
-    PADDLE_ENFORCE_EQ(
-        lr,
-        lr_out,
-        platform::errors::InvalidArgument("Input(LearningRate) and "
-                                          "Output(LearningRateOut) "
-                                          "must be the same."));
-    PADDLE_ENFORCE_NOT_NULL(lr,
-                            platform::errors::InvalidArgument(
-                                "Input(LearingRate) should not be nullptr."));
-    PADDLE_ENFORCE_EQ(step,
-                      step_out,
-                      platform::errors::InvalidArgument(
-                          "Input(Step) and Output(StepOut) must be the same."));
-    PADDLE_ENFORCE_NOT_NULL(step,
-                            platform::errors::InvalidArgument(
-                                "Input(Step) should not be nullptr."));
-    PADDLE_ENFORCE_EQ(
-        step->IsInitialized(),
-        true,
-        platform::errors::InvalidArgument("Input(Step) must be initialized."));
-
-    auto warmup_steps = static_cast<size_t>(ctx.Attr<int64_t>("warmup_steps"));
-    auto total_steps = static_cast<size_t>(ctx.Attr<int64_t>("total_steps"));
-    PADDLE_ENFORCE_LE(warmup_steps,
-                      total_steps,
-                      platform::errors::InvalidArgument(
-                          "warmup_steps must not be larger than total_steps."));
-    auto base_lr = ctx.Attr<float>("base_lr");
-    auto end_lr = ctx.Attr<float>("end_lr");
-
-    auto *lr_data = lr_out->data<T>();
-    auto *step_data = step_out->data<int64_t>();
-    auto &dev_ctx = ctx.template device_context<DeviceContext>();
-    platform::ForRange<DeviceContext> for_range(dev_ctx, 1);
-    using AttrT = double;
-    Pow2DecayWithLinearWarmupFunctor<T, AttrT> functor(
-        lr_data,
-        step_data,
-        warmup_steps,
-        total_steps,
-        static_cast<AttrT>(base_lr),
-        static_cast<AttrT>(end_lr));
-    for_range(functor);
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
diff --git a/paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op_xpu.cc b/paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op_xpu.cc
deleted file mode 100644
index 543a4634c6d71..0000000000000
--- a/paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op_xpu.cc
+++ /dev/null
@@ -1,84 +0,0 @@
-// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef PADDLE_WITH_XPU
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/framework/operator.h"
-#include "paddle/fluid/framework/tensor.h"
-#include "paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.h"
-#include "paddle/fluid/platform/macros.h"
-#include "paddle/phi/backends/xpu/enforce_xpu.h"
-
-namespace paddle {
-namespace operators {
-
-template <typename T>
-class Pow2DecayWithLinearWarmupXPUOpKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext &ctx) const {
-    const auto *lr = ctx.Input<phi::DenseTensor>("LearningRate");
-    const auto *step = ctx.Input<phi::DenseTensor>("Step");
-    auto *lr_out = ctx.Output<phi::DenseTensor>("LearningRateOut");
-    auto *step_out = ctx.Output<phi::DenseTensor>("StepOut");
-    PADDLE_ENFORCE_EQ(
-        lr,
-        lr_out,
-        platform::errors::InvalidArgument("Input(LearningRate) and "
-                                          "Output(LearningRateOut) "
-                                          "must be the same."));
-    PADDLE_ENFORCE_NOT_NULL(lr,
-                            platform::errors::InvalidArgument(
-                                "Input(LearingRate) should not be nullptr."));
-    PADDLE_ENFORCE_EQ(step,
-                      step_out,
-                      platform::errors::InvalidArgument(
-                          "Input(Step) and Output(StepOut) must be the same."));
-    PADDLE_ENFORCE_NOT_NULL(step,
-                            platform::errors::InvalidArgument(
-                                "Input(Step) should not be nullptr."));
-    PADDLE_ENFORCE_EQ(
-        step->IsInitialized(),
-        true,
-        platform::errors::InvalidArgument("Input(Step) must be initialized."));
-
-    auto warmup_steps = static_cast<size_t>(ctx.Attr<int64_t>("warmup_steps"));
-    auto total_steps = static_cast<size_t>(ctx.Attr<int64_t>("total_steps"));
-    PADDLE_ENFORCE_LE(warmup_steps,
-                      total_steps,
-                      platform::errors::InvalidArgument(
-                          "warmup_steps must not be larger than total_steps."));
-    auto base_lr = ctx.Attr<float>("base_lr");
-    auto end_lr = ctx.Attr<float>("end_lr");
-
-    auto *lr_data = lr_out->data<T>();
-    auto *step_data = step_out->data<int64_t>();
-    auto &dev_ctx = ctx.template device_context<platform::XPUDeviceContext>();
-    int r = xpu::pow2_decay_with_linear_warmup(dev_ctx.x_context(),
-                                               lr_data,
-                                               step_data,
-                                               warmup_steps,
-                                               total_steps,
-                                               base_lr,
-                                               end_lr);
-    PADDLE_ENFORCE_XDNN_SUCCESS(r, "pow2_decay_with_linear_warmup");
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-REGISTER_OP_XPU_KERNEL(pow2_decay_with_linear_warmup,
-                       ops::Pow2DecayWithLinearWarmupXPUOpKernel<float>);
-#endif
diff --git a/paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.cu b/paddle/phi/kernels/cpu/pow2_decay_with_linear_warmup_kernel.cc
similarity index 54%
rename from paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.cu
rename to paddle/phi/kernels/cpu/pow2_decay_with_linear_warmup_kernel.cc
index 6419e524f7152..9bdf6bb2c86ab 100644
--- a/paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.cu
+++ b/paddle/phi/kernels/cpu/pow2_decay_with_linear_warmup_kernel.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,13 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/fluid/operators/optimizers/pow2_decay_with_linear_warmup_op.h"
-#include "paddle/fluid/framework/op_registry.h"
+#include "paddle/phi/kernels/pow2_decay_with_linear_warmup_kernel.h"
 
-namespace ops = paddle::operators;
-namespace plat = paddle::platform;
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/impl/pow2_decay_with_linear_warmup_kernel_impl.h"
 
-REGISTER_OP_CUDA_KERNEL(
-    pow2_decay_with_linear_warmup,
-    ops::Pow2DecayWithLinearWarmupOpKernel<phi::GPUContext, double>,
-    ops::Pow2DecayWithLinearWarmupOpKernel<phi::GPUContext, float>);
+PD_REGISTER_KERNEL(pow2_decay_with_linear_warmup,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::Pow2DecayWithLinearWarmupKernel,
+                   float,
+                   double) {}
diff --git a/paddle/phi/kernels/gpu/pow2_decay_with_linear_warmup_kernel.cu b/paddle/phi/kernels/gpu/pow2_decay_with_linear_warmup_kernel.cu
new file mode 100644
index 0000000000000..57162bc7fb2b3
--- /dev/null
+++ b/paddle/phi/kernels/gpu/pow2_decay_with_linear_warmup_kernel.cu
@@ -0,0 +1,25 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/pow2_decay_with_linear_warmup_kernel.h"
+
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/impl/pow2_decay_with_linear_warmup_kernel_impl.h"
+
+PD_REGISTER_KERNEL(pow2_decay_with_linear_warmup,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::Pow2DecayWithLinearWarmupKernel,
+                   float,
+                   double) {}
diff --git a/paddle/phi/kernels/impl/pow2_decay_with_linear_warmup_kernel_impl.h b/paddle/phi/kernels/impl/pow2_decay_with_linear_warmup_kernel_impl.h
new file mode 100644
index 0000000000000..bbca911b404c4
--- /dev/null
+++ b/paddle/phi/kernels/impl/pow2_decay_with_linear_warmup_kernel_impl.h
@@ -0,0 +1,110 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/macros.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
+
+namespace phi {
+
+template <typename T, typename AttrT>
+struct Pow2DecayWithLinearWarmupFunctor {
+  template <typename U>
+  using RestrictPtr = U* PADDLE_RESTRICT;
+
+ public:
+  HOSTDEVICE Pow2DecayWithLinearWarmupFunctor(RestrictPtr<T> lr,
+                                              RestrictPtr<int64_t> step,
+                                              size_t warmup_steps,
+                                              size_t total_steps,
+                                              AttrT base_lr,
+                                              AttrT end_lr)
+      : lr_(lr),
+        step_(step),
+        warmup_steps_(warmup_steps),
+        total_steps_(total_steps),
+        base_lr_(base_lr),
+        end_lr_(end_lr) {}
+
+  HOSTDEVICE void operator()(size_t) const {
+    size_t step = static_cast<size_t>(*step_) + 1;
+    *step_ = static_cast<int64_t>(step);
+    if (step <= warmup_steps_) {
+      auto new_lr = static_cast<double>(step) / warmup_steps_ * base_lr_;
+      *lr_ = static_cast<T>(new_lr);
+    } else if (step < total_steps_) {
+      auto factor = 1 - static_cast<double>(step - warmup_steps_) /
+                            (total_steps_ - warmup_steps_);
+      auto new_lr =
+          static_cast<double>(base_lr_ - end_lr_) * (factor * factor) + end_lr_;
+      *lr_ = static_cast<T>(new_lr);
+    } else {
+      *lr_ = static_cast<T>(end_lr_);
+    }
+  }
+
+ private:
+  RestrictPtr<T> lr_;
+  RestrictPtr<int64_t> step_;
+  size_t warmup_steps_;
+  size_t total_steps_;
+  AttrT base_lr_;
+  AttrT end_lr_;
+};
+
+template <typename T, typename Context>
+void Pow2DecayWithLinearWarmupKernel(const Context& dev_ctx,
+                                     const DenseTensor& lr,
+                                     const DenseTensor& step,
+                                     int64_t warmup_steps,
+                                     int64_t total_steps,
+                                     float base_lr,
+                                     float end_lr,
+                                     DenseTensor* lr_out,
+                                     DenseTensor* step_out) {
+  PADDLE_ENFORCE_EQ(&lr,
+                    lr_out,
+                    phi::errors::InvalidArgument("Input(LearningRate) and "
+                                                 "Output(LearningRateOut) "
+                                                 "must be the same."));
+  PADDLE_ENFORCE_EQ(&step,
+                    step_out,
+                    phi::errors::InvalidArgument(
+                        "Input(Step) and Output(StepOut) must be the same."));
+  PADDLE_ENFORCE_EQ(
+      step.IsInitialized(),
+      true,
+      phi::errors::InvalidArgument("Input(Step) must be initialized."));
+
+  PADDLE_ENFORCE_LE(warmup_steps,
+                    total_steps,
+                    phi::errors::InvalidArgument(
+                        "warmup_steps must not be larger than total_steps."));
+
+  auto* lr_data = lr_out->data<T>();
+  auto* step_data = step_out->data<int64_t>();
+  phi::funcs::ForRange<Context> for_range(dev_ctx, 1);
+  using AttrT = double;
+  Pow2DecayWithLinearWarmupFunctor<T, AttrT> functor(
+      lr_data,
+      step_data,
+      static_cast<size_t>(warmup_steps),
+      static_cast<size_t>(total_steps),
+      static_cast<AttrT>(base_lr),
+      static_cast<AttrT>(end_lr));
+  for_range(functor);
+}
+}  // namespace phi
diff --git a/paddle/phi/kernels/pow2_decay_with_linear_warmup_kernel.h b/paddle/phi/kernels/pow2_decay_with_linear_warmup_kernel.h
new file mode 100644
index 0000000000000..549e8d0add109
--- /dev/null
+++ b/paddle/phi/kernels/pow2_decay_with_linear_warmup_kernel.h
@@ -0,0 +1,31 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void Pow2DecayWithLinearWarmupKernel(const Context& dev_ctx,
+                                     const DenseTensor& lr,
+                                     const DenseTensor& step,
+                                     int64_t warmup_steps,
+                                     int64_t total_steps,
+                                     float base_lr,
+                                     float end_lr,
+                                     DenseTensor* lr_out,
+                                     DenseTensor* step_out);
+}  // namespace phi
diff --git a/paddle/phi/kernels/xpu/pow2_decay_with_linear_warmup_kernel.cc b/paddle/phi/kernels/xpu/pow2_decay_with_linear_warmup_kernel.cc
new file mode 100644
index 0000000000000..8661f2ac2cb22
--- /dev/null
+++ b/paddle/phi/kernels/xpu/pow2_decay_with_linear_warmup_kernel.cc
@@ -0,0 +1,71 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/pow2_decay_with_linear_warmup_kernel.h"
+
+#include "paddle/phi/backends/xpu/enforce_xpu.h"
+#include "paddle/phi/backends/xpu/xpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/macros.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void Pow2DecayWithLinearWarmupKernel(const Context& dev_ctx,
+                                     const DenseTensor& lr,
+                                     const DenseTensor& step,
+                                     int64_t warmup_steps,
+                                     int64_t total_steps,
+                                     float base_lr,
+                                     float end_lr,
+                                     DenseTensor* lr_out,
+                                     DenseTensor* step_out) {
+  PADDLE_ENFORCE_EQ(&lr,
+                    lr_out,
+                    phi::errors::InvalidArgument("Input(LearningRate) and "
+                                                 "Output(LearningRateOut) "
+                                                 "must be the same."));
+  PADDLE_ENFORCE_EQ(&step,
+                    step_out,
+                    phi::errors::InvalidArgument(
+                        "Input(Step) and Output(StepOut) must be the same."));
+  PADDLE_ENFORCE_EQ(
+      step.IsInitialized(),
+      true,
+      phi::errors::InvalidArgument("Input(Step) must be initialized."));
+
+  PADDLE_ENFORCE_LE(warmup_steps,
+                    total_steps,
+                    phi::errors::InvalidArgument(
+                        "warmup_steps must not be larger than total_steps."));
+
+  auto* lr_data = lr_out->data<T>();
+  auto* step_data = step_out->data<int64_t>();
+  int r = xpu::pow2_decay_with_linear_warmup(dev_ctx.x_context(),
+                                             lr_data,
+                                             step_data,
+                                             static_cast<size_t>(warmup_steps),
+                                             static_cast<size_t>(total_steps),
+                                             base_lr,
+                                             end_lr);
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "pow2_decay_with_linear_warmup");
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(pow2_decay_with_linear_warmup,
+                   XPU,
+                   ALL_LAYOUT,
+                   phi::Pow2DecayWithLinearWarmupKernel,
+                   float) {}
diff --git a/paddle/phi/ops/compat/pow2_decay_with_linear_warmup_sig.cc b/paddle/phi/ops/compat/pow2_decay_with_linear_warmup_sig.cc
new file mode 100644
index 0000000000000..9a3323d90bff3
--- /dev/null
+++ b/paddle/phi/ops/compat/pow2_decay_with_linear_warmup_sig.cc
@@ -0,0 +1,30 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/core/compat/op_utils.h"
+
+namespace phi {
+
+KernelSignature Pow2DecayWithLinearWarmupOpArgumentMapping(
+    const ArgumentMappingContext& ctx) {
+  return KernelSignature("pow2_decay_with_linear_warmup",
+                         {"LearningRate", "Step"},
+                         {"warmup_steps", "total_steps", "base_lr", "end_lr"},
+                         {"LearningRateOut", "StepOut"});
+}
+
+}  // namespace phi
+
+PD_REGISTER_ARG_MAPPING_FN(pow2_decay_with_linear_warmup,
+                           phi::Pow2DecayWithLinearWarmupOpArgumentMapping);

From 0603777b435fb33882a73b40b582018df29624db Mon Sep 17 00:00:00 2001
From: RuohengMa <120699764+RuohengMa@users.noreply.github.com>
Date: Fri, 12 May 2023 12:50:58 +0800
Subject: [PATCH 300/405] [PHI] update xpu api version; bind reduce_any_bool
 xpu kernel; remove unnecessary header (#53716)

---
 cmake/external/xpu.cmake                      |  2 +-
 paddle/phi/backends/xpu/xpu2_op_list.cc       |  1 +
 paddle/phi/kernels/reduce_any_kernel.cc       |  4 ++
 paddle/phi/kernels/xpu/reduce_any_kernel.cc   | 51 +++++++++++++++++++
 paddle/phi/kernels/xpu/scatter_grad_kernel.cc |  2 -
 5 files changed, 57 insertions(+), 3 deletions(-)
 create mode 100644 paddle/phi/kernels/xpu/reduce_any_kernel.cc

diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake
index c2be9fab7ee78..caa639280502c 100644
--- a/cmake/external/xpu.cmake
+++ b/cmake/external/xpu.cmake
@@ -8,7 +8,7 @@ set(XPU_API_LIB_NAME "libxpuapi.so")
 set(XPU_RT_LIB_NAME "libxpurt.so")
 set(XPU_XFT_LIB_NAME "libxft.so")
 
-set(XPU_BASE_DATE "20230427")
+set(XPU_BASE_DATE "20230510")
 set(XPU_XCCL_BASE_VERSION "1.0.49.2")
 set(XPU_XFT_BASE_VERSION "latest")
 
diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc
index 6f10baf07bb5a..ba402aed5a6cd 100644
--- a/paddle/phi/backends/xpu/xpu2_op_list.cc
+++ b/paddle/phi/backends/xpu/xpu2_op_list.cc
@@ -574,6 +574,7 @@ XPUOpMap& get_kl2_ops() {
       {"reciprocal", XPUKernelSet({phi::DataType::FLOAT32})},
       {"reciprocal_grad",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})},
+      {"reduce_any", XPUKernelSet({phi::DataType::BOOL})},
       {"reduce_max_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"reduce_max",
        XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::INT32})},
diff --git a/paddle/phi/kernels/reduce_any_kernel.cc b/paddle/phi/kernels/reduce_any_kernel.cc
index 2baa1edb094b9..adb2f8168bc19 100644
--- a/paddle/phi/kernels/reduce_any_kernel.cc
+++ b/paddle/phi/kernels/reduce_any_kernel.cc
@@ -40,3 +40,7 @@ PD_REGISTER_KERNEL(any, GPU, ALL_LAYOUT, phi::AnyKernel, bool) {}
 #if defined(PADDLE_WITH_XPU_KP)
 PD_REGISTER_KERNEL(any, KPS, ALL_LAYOUT, phi::AnyKernel, bool) {}
 #endif
+
+#if defined(PADDLE_WITH_XPU)
+PD_REGISTER_KERNEL(any, XPU, ALL_LAYOUT, phi::AnyKernel, bool) {}
+#endif
diff --git a/paddle/phi/kernels/xpu/reduce_any_kernel.cc b/paddle/phi/kernels/xpu/reduce_any_kernel.cc
new file mode 100644
index 0000000000000..280d3871576e9
--- /dev/null
+++ b/paddle/phi/kernels/xpu/reduce_any_kernel.cc
@@ -0,0 +1,51 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/reduce_any_kernel.h"
+
+#include "paddle/phi/backends/xpu/enforce_xpu.h"
+#include "paddle/phi/backends/xpu/xpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/xpu/reduce.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void AnyRawKernel(const Context& dev_ctx,
+                  const DenseTensor& x,
+                  const std::vector<int64_t>& dims,
+                  bool keep_dim,
+                  bool reduce_all,
+                  DenseTensor* out) {
+  reduce_all = recompute_reduce_all(x, dims, reduce_all);
+  using XPUType = typename XPUTypeTrait<T>::Type;
+  auto f = [](xpu::Context* ctx,
+              const T* x,
+              T* y,
+              const std::vector<int>& xdims,
+              const std::vector<int>& reduce_dims) {
+    return xpu::reduce_any<XPUType>(ctx,
+                                    reinterpret_cast<const XPUType*>(x),
+                                    reinterpret_cast<XPUType*>(y),
+                                    xdims,
+                                    reduce_dims);
+  };
+
+  int r = XPUReduce<Context, T>(dev_ctx, x, dims, keep_dim, reduce_all, out, f);
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "reduce_any");
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(any_raw, XPU, ALL_LAYOUT, phi::AnyRawKernel, bool) {}
diff --git a/paddle/phi/kernels/xpu/scatter_grad_kernel.cc b/paddle/phi/kernels/xpu/scatter_grad_kernel.cc
index 6e8bbd372ba4c..2d4007d6a2a8e 100644
--- a/paddle/phi/kernels/xpu/scatter_grad_kernel.cc
+++ b/paddle/phi/kernels/xpu/scatter_grad_kernel.cc
@@ -17,8 +17,6 @@
 #include "paddle/phi/backends/xpu/enforce_xpu.h"
 #include "paddle/phi/core/kernel_registry.h"
 
-#include "glog/logging.h"
-
 namespace phi {
 
 template <typename T, typename Context>

From d2b1e3c2125b01249000f3142d98830c84331399 Mon Sep 17 00:00:00 2001
From: Wang Xin <xinwang614@gmail.com>
Date: Fri, 12 May 2023 12:59:41 +0800
Subject: [PATCH 301/405] sequence_mask functionalization (#53478)

* sequence_mask functionalization

* fix sequence_mask test
---
 .../sequence_ops/sequence_mask_op.cc          |  14 +-
 .../sequence_ops/sequence_mask_op.cu          |  25 ----
 .../operators/sequence_ops/sequence_mask_op.h | 141 ------------------
 .../sequence_ops/unity_build_rule.cmake       |   1 -
 .../phi/kernels/cpu/sequence_mask_kernel.cc   |  27 ++++
 paddle/phi/kernels/funcs/sequence_mask.h      |  65 ++++++++
 .../phi/kernels/gpu/sequence_mask_kernel.cu   |  27 ++++
 .../kernels/impl/sequence_mask_kernel_impl.h  |  88 +++++++++++
 paddle/phi/kernels/sequence_mask_kernel.h     |  25 ++++
 paddle/phi/ops/compat/sequence_mask_sig.cc    |  24 +++
 test/sequence/test_sequence_mask.py           |  10 ++
 11 files changed, 267 insertions(+), 180 deletions(-)
 delete mode 100644 paddle/fluid/operators/sequence_ops/sequence_mask_op.cu
 delete mode 100644 paddle/fluid/operators/sequence_ops/sequence_mask_op.h
 create mode 100644 paddle/phi/kernels/cpu/sequence_mask_kernel.cc
 create mode 100644 paddle/phi/kernels/funcs/sequence_mask.h
 create mode 100644 paddle/phi/kernels/gpu/sequence_mask_kernel.cu
 create mode 100644 paddle/phi/kernels/impl/sequence_mask_kernel_impl.h
 create mode 100644 paddle/phi/kernels/sequence_mask_kernel.h
 create mode 100644 paddle/phi/ops/compat/sequence_mask_sig.cc

diff --git a/paddle/fluid/operators/sequence_ops/sequence_mask_op.cc b/paddle/fluid/operators/sequence_ops/sequence_mask_op.cc
index 2e58db3d94de8..f50fc2268c4e8 100644
--- a/paddle/fluid/operators/sequence_ops/sequence_mask_op.cc
+++ b/paddle/fluid/operators/sequence_ops/sequence_mask_op.cc
@@ -12,9 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/fluid/operators/sequence_ops/sequence_mask_op.h"
-
-#include <string>
+#include "paddle/fluid/framework/op_registry.h"
 
 namespace paddle {
 namespace operators {
@@ -101,13 +99,3 @@ REGISTER_OPERATOR(
     paddle::operators::SequenceMaskOpMaker,
     paddle::framework::EmptyGradOpMaker<paddle::framework::OpDesc>,
     paddle::framework::EmptyGradOpMaker<paddle::imperative::OpBase>);
-
-namespace ops = paddle::operators;
-PD_REGISTER_STRUCT_KERNEL(sequence_mask,
-                          CPU,
-                          ALL_LAYOUT,
-                          ops::SequenceMaskKernel,
-                          float,
-                          double,
-                          int,
-                          int64_t) {}
diff --git a/paddle/fluid/operators/sequence_ops/sequence_mask_op.cu b/paddle/fluid/operators/sequence_ops/sequence_mask_op.cu
deleted file mode 100644
index b93b250352068..0000000000000
--- a/paddle/fluid/operators/sequence_ops/sequence_mask_op.cu
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/fluid/operators/sequence_ops/sequence_mask_op.h"
-
-namespace ops = paddle::operators;
-PD_REGISTER_STRUCT_KERNEL(sequence_mask,
-                          GPU,
-                          ALL_LAYOUT,
-                          ops::SequenceMaskKernel,
-                          float,
-                          double,
-                          int,
-                          int64_t) {}
diff --git a/paddle/fluid/operators/sequence_ops/sequence_mask_op.h b/paddle/fluid/operators/sequence_ops/sequence_mask_op.h
deleted file mode 100644
index ac69c7d5bf826..0000000000000
--- a/paddle/fluid/operators/sequence_ops/sequence_mask_op.h
+++ /dev/null
@@ -1,141 +0,0 @@
-// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#pragma once
-
-#if defined(__NVCC__) || defined(__HIPCC__)
-#include <thrust/device_ptr.h>
-#include <thrust/functional.h>
-#include <thrust/reduce.h>
-#else
-#include <algorithm>
-#endif
-
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/platform/for_range.h"
-
-namespace paddle {
-namespace operators {
-
-template <typename Tx, typename Ty>
-struct SequenceMaskForRangeFunctor {
-  HOSTDEVICE SequenceMaskForRangeFunctor(const Tx *x, Ty *y, int maxlen)
-      : x_(x), y_(y), maxlen_(maxlen) {}
-
-  HOSTDEVICE void operator()(int y_idx) const {
-    int x_idx = y_idx / maxlen_;
-    int j = y_idx % maxlen_;
-    y_[y_idx] = static_cast<Ty>(j < x_[x_idx] ? 1 : 0);
-  }
-
- private:
-  const Tx *x_;
-  Ty *y_;
-  int maxlen_;
-};
-
-template <typename DeviceContext, typename Tx>
-struct SequenceMaskFunctor {
-  SequenceMaskFunctor(const DeviceContext &ctx,
-                      const Tx *x,
-                      phi::DenseTensor *y,
-                      int limits,
-                      int maxlen)
-      : ctx_(ctx), x_(x), y_(y), limits_(limits), maxlen_(maxlen) {}
-
-  template <typename Ty>
-  void apply() const {
-    auto *y_data = y_->mutable_data<Ty>(ctx_.GetPlace());
-    platform::ForRange<DeviceContext> for_range(ctx_, limits_);
-    for_range(SequenceMaskForRangeFunctor<Tx, Ty>(x_, y_data, maxlen_));
-  }
-
- private:
-  const DeviceContext &ctx_;
-  const Tx *x_;
-  phi::DenseTensor *y_;
-  int limits_;
-  int maxlen_;
-};
-
-template <typename Tx, typename DeviceContext>
-class SequenceMaskKernel : public framework::OpKernel<Tx> {
- public:
-  void Compute(const framework::ExecutionContext &ctx) const override {
-    auto *x = ctx.Input<phi::DenseTensor>("X");
-    auto *y = ctx.Output<phi::DenseTensor>("Y");
-    int maxlen = ctx.Attr<int>("maxlen");
-    if (ctx.HasInput("MaxLenTensor")) {
-      auto max_len_tensor = ctx.Input<phi::DenseTensor>("MaxLenTensor");
-      PADDLE_ENFORCE_NOT_NULL(max_len_tensor,
-                              platform::errors::InvalidArgument(
-                                  "Input(MaxLenTensor) should not be NULL."
-                                  "But received Input(MaxLenTensor) is NULL"));
-      if (platform::is_gpu_place(max_len_tensor->place())) {
-        phi::DenseTensor temp;
-        paddle::framework::TensorCopySync(
-            *max_len_tensor, platform::CPUPlace(), &temp);
-        maxlen = *temp.data<int32_t>();
-      } else {
-        maxlen = *max_len_tensor->data<int32_t>();
-      }
-
-      auto y_dim = phi::vectorize<int>(x->dims());
-      y_dim.push_back(maxlen);
-      y->Resize(phi::make_ddim(y_dim));
-
-      PADDLE_ENFORCE_GT(
-          maxlen,
-          0,
-          platform::errors::InvalidArgument(
-              "Input(MaxLenTensor) value should be greater than 0. But "
-              "received Input(MaxLenTensor) value = %d.",
-              maxlen));
-    }
-
-    auto *x_data = x->data<Tx>();
-    auto x_numel = x->numel();
-
-    if (maxlen < 0) {
-      if (x_numel == 0) {
-        maxlen = 0;
-      } else {
-#if defined(__NVCC__) || defined(__HIPCC__)
-        VLOG(10)
-            << "SequenceMaskOp on GPU may be slow when maxlen is not provided.";
-        maxlen = static_cast<int>(
-            thrust::reduce(thrust::device_pointer_cast(x_data),
-                           thrust::device_pointer_cast(x_data) + x_numel,
-                           static_cast<Tx>(0),
-                           thrust::maximum<Tx>()));
-#else
-        maxlen = static_cast<int>(*std::max_element(x_data, x_data + x_numel));
-#endif
-      }
-      auto y_dim = phi::vectorize<int>(x->dims());
-      y_dim.push_back(maxlen);
-      y->Resize(phi::make_ddim(y_dim));
-    }
-
-    auto out_dtype = static_cast<framework::proto::VarType::Type>(
-        ctx.Attr<int>("out_dtype"));
-    auto &dev_ctx = ctx.template device_context<DeviceContext>();
-    framework::VisitDataType(out_dtype,
-                             SequenceMaskFunctor<DeviceContext, Tx>(
-                                 dev_ctx, x_data, y, x_numel * maxlen, maxlen));
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
diff --git a/paddle/fluid/operators/sequence_ops/unity_build_rule.cmake b/paddle/fluid/operators/sequence_ops/unity_build_rule.cmake
index b256a227a9646..a22e6865cf103 100644
--- a/paddle/fluid/operators/sequence_ops/unity_build_rule.cmake
+++ b/paddle/fluid/operators/sequence_ops/unity_build_rule.cmake
@@ -28,7 +28,6 @@ register_unity_group(
   sequence_enumerate_op.cu
   sequence_erase_op.cu
   sequence_expand_op.cu
-  sequence_mask_op.cu
   sequence_pad_op.cu
   sequence_expand_as_op.cu
   sequence_reshape_op.cu
diff --git a/paddle/phi/kernels/cpu/sequence_mask_kernel.cc b/paddle/phi/kernels/cpu/sequence_mask_kernel.cc
new file mode 100644
index 0000000000000..20f999f4be228
--- /dev/null
+++ b/paddle/phi/kernels/cpu/sequence_mask_kernel.cc
@@ -0,0 +1,27 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/sequence_mask_kernel.h"
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/impl/sequence_mask_kernel_impl.h"
+
+PD_REGISTER_KERNEL(sequence_mask,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::SequenceMaskKernel,
+                   float,
+                   double,
+                   int,
+                   int64_t) {}
diff --git a/paddle/phi/kernels/funcs/sequence_mask.h b/paddle/phi/kernels/funcs/sequence_mask.h
new file mode 100644
index 0000000000000..8e14dc89b86f3
--- /dev/null
+++ b/paddle/phi/kernels/funcs/sequence_mask.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
+
+namespace phi {
+namespace funcs {
+
+template <typename Tx, typename Ty>
+struct SequenceMaskForRangeFunctor {
+  HOSTDEVICE SequenceMaskForRangeFunctor(const Tx *x, Ty *y, int maxlen)
+      : x_(x), y_(y), maxlen_(maxlen) {}
+
+  HOSTDEVICE void operator()(int y_idx) const {
+    int x_idx = y_idx / maxlen_;
+    int j = y_idx % maxlen_;
+    y_[y_idx] = static_cast<Ty>(j < x_[x_idx] ? 1 : 0);
+  }
+
+ private:
+  const Tx *x_;
+  Ty *y_;
+  int maxlen_;
+};
+
+template <typename DeviceContext, typename Tx>
+struct SequenceMaskFunctor {
+  SequenceMaskFunctor(const DeviceContext &ctx,
+                      const Tx *x,
+                      phi::DenseTensor *y,
+                      int limits,
+                      int maxlen)
+      : ctx_(ctx), x_(x), y_(y), limits_(limits), maxlen_(maxlen) {}
+  template <typename Ty>
+  void apply() const {
+    ctx_.template Alloc<Ty>(y_);
+    auto *y_data = y_->data<Ty>();
+    phi::funcs::ForRange<DeviceContext> for_range(ctx_, limits_);
+    for_range(SequenceMaskForRangeFunctor<Tx, Ty>(x_, y_data, maxlen_));
+  }
+
+ private:
+  const DeviceContext &ctx_;
+  const Tx *x_;
+  phi::DenseTensor *y_;
+  int limits_;
+  int maxlen_;
+};
+
+}  // namespace funcs
+}  // namespace phi
diff --git a/paddle/phi/kernels/gpu/sequence_mask_kernel.cu b/paddle/phi/kernels/gpu/sequence_mask_kernel.cu
new file mode 100644
index 0000000000000..3e37e3e2419c2
--- /dev/null
+++ b/paddle/phi/kernels/gpu/sequence_mask_kernel.cu
@@ -0,0 +1,27 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/sequence_mask_kernel.h"
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/impl/sequence_mask_kernel_impl.h"
+
+PD_REGISTER_KERNEL(sequence_mask,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::SequenceMaskKernel,
+                   float,
+                   double,
+                   int,
+                   int64_t) {}
diff --git a/paddle/phi/kernels/impl/sequence_mask_kernel_impl.h b/paddle/phi/kernels/impl/sequence_mask_kernel_impl.h
new file mode 100644
index 0000000000000..80834fae85411
--- /dev/null
+++ b/paddle/phi/kernels/impl/sequence_mask_kernel_impl.h
@@ -0,0 +1,88 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#if defined(__NVCC__) || defined(__HIPCC__)
+#include <thrust/device_ptr.h>
+#include <thrust/functional.h>
+#include <thrust/reduce.h>
+#else
+#include <algorithm>
+#endif
+
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/kernels/funcs/sequence_mask.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void SequenceMaskKernel(const Context& ctx,
+                        const DenseTensor& x,
+                        const paddle::optional<DenseTensor>& max_len_tensor,
+                        int maxlen,
+                        int out_dtype,
+                        DenseTensor* y) {
+  if (max_len_tensor) {
+    bool is_gpu_place = ctx.GetPlace().GetType() == phi::AllocationType::GPU;
+    if (is_gpu_place) {
+      phi::DenseTensor temp;
+      phi::Copy(ctx, *max_len_tensor.get_ptr(), phi::CPUPlace(), false, &temp);
+      maxlen = *temp.data<int32_t>();
+    } else {
+      maxlen = *max_len_tensor.get_ptr()->data<int32_t>();
+    }
+
+    auto y_dim = phi::vectorize<int>(x.dims());
+    y_dim.push_back(maxlen);
+    y->Resize(phi::make_ddim(y_dim));
+
+    PADDLE_ENFORCE_GT(
+        maxlen,
+        0,
+        phi::errors::InvalidArgument(
+            "Input(MaxLenTensor) value should be greater than 0. But "
+            "received Input(MaxLenTensor) value = %d.",
+            maxlen));
+  }
+
+  auto* x_data = x.data<T>();
+  auto x_numel = x.numel();
+
+  if (maxlen < 0) {
+    if (x_numel == 0) {
+      maxlen = 0;
+    } else {
+#if defined(__NVCC__) || defined(__HIPCC__)
+      VLOG(10)
+          << "SequenceMaskOp on GPU may be slow when maxlen is not provided.";
+      maxlen = static_cast<int>(
+          thrust::reduce(thrust::device_pointer_cast(x_data),
+                         thrust::device_pointer_cast(x_data) + x_numel,
+                         static_cast<T>(0),
+                         thrust::maximum<T>()));
+#else
+      maxlen = static_cast<int>(*std::max_element(x_data, x_data + x_numel));
+#endif
+    }
+    auto y_dim = phi::vectorize<int>(x.dims());
+    y_dim.push_back(maxlen);
+    y->Resize(phi::make_ddim(y_dim));
+  }
+
+  phi::VisitDataType(phi::TransToPhiDataType(out_dtype),
+                     phi::funcs::SequenceMaskFunctor<Context, T>(
+                         ctx, x_data, y, x_numel * maxlen, maxlen));
+}
+}  // namespace phi
diff --git a/paddle/phi/kernels/sequence_mask_kernel.h b/paddle/phi/kernels/sequence_mask_kernel.h
new file mode 100644
index 0000000000000..ff547ee2522ec
--- /dev/null
+++ b/paddle/phi/kernels/sequence_mask_kernel.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+template <typename T, typename Context>
+void SequenceMaskKernel(const Context& ctx,
+                        const DenseTensor& x,
+                        const paddle::optional<DenseTensor>& max_len_tensor,
+                        int maxlen,
+                        int out_dtype,
+                        DenseTensor* y);
+
+}  // namespace phi
diff --git a/paddle/phi/ops/compat/sequence_mask_sig.cc b/paddle/phi/ops/compat/sequence_mask_sig.cc
new file mode 100644
index 0000000000000..93895e79f83af
--- /dev/null
+++ b/paddle/phi/ops/compat/sequence_mask_sig.cc
@@ -0,0 +1,24 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/core/compat/op_utils.h"
+
+namespace phi {
+
+KernelSignature SequenceMaskOpArgumentMapping(
+    const ArgumentMappingContext& ctx) {
+  return KernelSignature(
+      "sequence_mask", {"X", "MaxLenTensor"}, {"maxlen", "out_dtype"}, {"Y"});
+}
+
+}  // namespace phi
+
+PD_REGISTER_ARG_MAPPING_FN(sequence_mask, phi::SequenceMaskOpArgumentMapping);
diff --git a/test/sequence/test_sequence_mask.py b/test/sequence/test_sequence_mask.py
index 0d2174d5dbc70..ab464d451b7ee 100644
--- a/test/sequence/test_sequence_mask.py
+++ b/test/sequence/test_sequence_mask.py
@@ -28,9 +28,18 @@
 from eager_op_test import OpTest
 
 
+def sequence_mask_wraper(x, maxlen_tensor=None, maxlen=-1, mask_dtype='int64'):
+    if maxlen_tensor is not None:
+        maxlen = maxlen_tensor
+    return paddle.nn.functional.sequence_mask(
+        x, maxlen=maxlen, dtype=mask_dtype
+    )
+
+
 class SequenceMaskTestBase(OpTest):
     def initDefaultParameters(self):
         self.op_type = 'sequence_mask'
+        self.python_api = sequence_mask_wraper
         self.maxlen = 10
         self.mask_dtype = 'int64'
         self.x = [[0, 3, 4], [5, 7, 9]]
@@ -100,6 +109,7 @@ def initParameters(self):
 class SequenceMaskTestBase_tensor_attr(OpTest):
     def initDefaultParameters(self):
         self.op_type = 'sequence_mask'
+        self.python_api = sequence_mask_wraper
         self.maxlen = 10
         self.maxlen_tensor = np.ones((1), 'int32') * 10
         self.mask_dtype = 'int64'

From 05d3fc81fa9ad0f37ae0e46ca7326fa909b520f8 Mon Sep 17 00:00:00 2001
From: Yuanle Liu <yuanlehome@163.com>
Date: Fri, 12 May 2023 13:27:54 +0800
Subject: [PATCH 302/405] [inference zero dim] softmax, stack op trt converter
 support zero dim (#53729)

* softmax support

* support stack
---
 .../tensorrt/convert/expand_v2_op.cc          |  5 ++-
 .../inference/tensorrt/convert/op_converter.h |  8 ++---
 .../inference/tensorrt/convert/softmax_op.cc  | 34 +++++++++++++++---
 .../inference/tensorrt/convert/stack_op.cc    | 11 +++---
 paddle/fluid/inference/tensorrt/op_teller.cc  | 35 +++++++++++++++++++
 test/ir/inference/test_trt_convert_softmax.py | 16 ++++++++-
 test/ir/inference/test_trt_convert_stack.py   | 28 +++++++++++++--
 7 files changed, 118 insertions(+), 19 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/convert/expand_v2_op.cc b/paddle/fluid/inference/tensorrt/convert/expand_v2_op.cc
index 452f1f8b92057..377df302f4f9f 100644
--- a/paddle/fluid/inference/tensorrt/convert/expand_v2_op.cc
+++ b/paddle/fluid/inference/tensorrt/convert/expand_v2_op.cc
@@ -83,7 +83,10 @@ class ExpandOpConverter : public OpConverter {
       input_shape_tensor = Shape(input);
     }
 
-    auto* newInputTensor = Reshape(input, input_shape_tensor);
+    auto* newInputTensor =
+        Reshape(input,
+                input_shape_tensor,
+                ("expand_v2: reshape: (Output(" + output_name + ")").c_str());
 
     std::vector<int32_t> start_vec(shape_rank, 0);
     nvinfer1::Dims start;
diff --git a/paddle/fluid/inference/tensorrt/convert/op_converter.h b/paddle/fluid/inference/tensorrt/convert/op_converter.h
index fcd6146a5f400..91bc8f7f2d8e0 100644
--- a/paddle/fluid/inference/tensorrt/convert/op_converter.h
+++ b/paddle/fluid/inference/tensorrt/convert/op_converter.h
@@ -407,13 +407,11 @@ class OpConverter {
   }
 
   nvinfer1::ITensor* Reshape(nvinfer1::ITensor* input,
-                             nvinfer1::ITensor* newShape) {
-    nvinfer1::ITensor* oldShape = Shape(input);
-    if (oldShape == newShape) {
-      return input;
-    }
+                             nvinfer1::ITensor* newShape,
+                             const std::string& name = "reshape") {
     auto* shuffle = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input);
     shuffle->setInput(1, *newShape);
+    shuffle->setName(name.c_str());
     return shuffle->getOutput(0);
   }
 
diff --git a/paddle/fluid/inference/tensorrt/convert/softmax_op.cc b/paddle/fluid/inference/tensorrt/convert/softmax_op.cc
index 0cbb5a102347c..8e101075768e0 100644
--- a/paddle/fluid/inference/tensorrt/convert/softmax_op.cc
+++ b/paddle/fluid/inference/tensorrt/convert/softmax_op.cc
@@ -38,8 +38,23 @@ class SoftMaxOpConverter : public OpConverter {
                    ? PADDLE_GET_CONST(int, op_desc.GetAttr("axis"))
                    : -1;
 
-    auto* layer = TRT_ENGINE_ADD_LAYER(
-        engine_, SoftMax, *const_cast<nvinfer1::ITensor*>(input1));
+    // support 0 or 1 dims input
+    bool is_0_dims = input_dims == 0;
+    bool is_1_dims = input_dims == 1;
+    if (is_0_dims || is_1_dims) {
+      auto reshaped_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input1);
+      nvinfer1::Dims reshaped_dims;
+      reshaped_dims.nbDims = 2;
+      reshaped_dims.d[0] = 1;
+      reshaped_dims.d[1] = is_0_dims ? 1 : input_shape.d[0];
+      reshaped_layer->setReshapeDimensions(reshaped_dims);
+      input1 = reshaped_layer->getOutput(0);
+      input_shape = input1->getDimensions();
+      input_dims = input_shape.nbDims;
+      axis = -1;
+    }
+
+    auto* layer = TRT_ENGINE_ADD_LAYER(engine_, SoftMax, *input1);
     uint32_t axes = std::max(0, input_dims - 3);
     // TODO(cryoco): Poor workaround. Fix padded dims problem when TRT layers
     // support Nd.
@@ -68,11 +83,22 @@ class SoftMaxOpConverter : public OpConverter {
       }
     }
     layer->setAxes(1 << axes);
-    auto output_name = op_desc.Output("Out")[0];
-    RreplenishLayerAndOutput(layer, "softmax", {output_name}, test_mode);
 
     // The trt will not run int for softmax.
     engine_->SetTensorDynamicRange(input1, 1.0);
+    auto output_name = op_desc.Output("Out")[0];
+
+    // support 0 or 1 dims input
+    if (is_0_dims || is_1_dims) {
+      auto reshaped_layer =
+          TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *layer->getOutput(0));
+      reshaped_layer->setReshapeDimensions(
+          engine_->GetITensor(op_desc.Input("X")[0])->getDimensions());
+      RreplenishLayerAndOutput(
+          reshaped_layer, "reshape_softmax_reshape", {output_name}, test_mode);
+    } else {
+      RreplenishLayerAndOutput(layer, "softmax", {output_name}, test_mode);
+    }
   }
 };
 
diff --git a/paddle/fluid/inference/tensorrt/convert/stack_op.cc b/paddle/fluid/inference/tensorrt/convert/stack_op.cc
index 2135387b32ab6..1b0722a467a58 100644
--- a/paddle/fluid/inference/tensorrt/convert/stack_op.cc
+++ b/paddle/fluid/inference/tensorrt/convert/stack_op.cc
@@ -65,12 +65,11 @@ class StackOpConverter : public OpConverter {
     auto* after_shape_tensor = Concat(shape_tensor_vec);
 
     for (int i = 0; i < input_num; ++i) {
-      auto* reshape_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *inputs[i]);
-      reshape_layer->setInput(1, *after_shape_tensor);
-      inputs[i] = reshape_layer->getOutput(0);
-      reshape_layer->setName(("stack: reshape: (Output( " + std::to_string(i) +
-                              " )" + output_name + ")")
-                                 .c_str());
+      inputs[i] = Reshape(inputs[i],
+                          after_shape_tensor,
+                          ("stack: reshape: (Output( " + std::to_string(i) +
+                           " )" + output_name + ")")
+                              .c_str());
     }
 
     auto* layer = TRT_ENGINE_ADD_LAYER(
diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index 49e39d7fc1b71..28fb2e985cead 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -404,6 +404,7 @@ struct SimpleOpTypeSetTeller : public Teller {
       return false;
 #endif
     }
+
     if (op_type == "softmax") {
       auto* block = desc.Block();
       if (block == nullptr) {
@@ -415,7 +416,23 @@ struct SimpleOpTypeSetTeller : public Teller {
       auto x_var_name = desc.Input("X")[0];
       auto* x_var_desc = block->FindVar(x_var_name);
       const auto x_shape = x_var_desc->GetShape();
+      if (!with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
+        VLOG(3) << op_type
+                << " op does not support input's dim is 1 or 0 in tensorrt "
+                   "with static shape.";
+        return false;
+      }
+
+      if (with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
+        int axis = desc.HasAttr("axis")
+                       ? PADDLE_GET_CONST(int, desc.GetAttr("axis"))
+                       : -1;
+        if (axis > 0) {
+          return false;
+        }
+      }
     }
+
     if (op_type == "group_norm") {
       if (!desc.HasAttr("epsilon") || !desc.HasAttr("groups") ||
           !desc.HasAttr("data_layout"))
@@ -1529,6 +1546,24 @@ struct SimpleOpTypeSetTeller : public Teller {
                "mode.";
         return false;
       }
+
+      auto* block = desc.Block();
+      if (block == nullptr) {
+        VLOG(3) << "The block desc is nullptr, we can't continue to analyze. "
+                   "Developers need to check whether block_desc is passed in "
+                   "the pass.";
+        return false;
+      }
+      auto x_var_name = desc.Input("X")[0];
+      auto* x_var_desc = block->FindVar(x_var_name);
+      const auto x_shape = x_var_desc->GetShape();
+      int rank = x_shape.size();
+      int axis = desc.HasAttr("axis")
+                     ? PADDLE_GET_CONST(int, desc.GetAttr("axis"))
+                     : -1;
+      if (axis > rank || axis < -(rank + 1)) {
+        return false;
+      }
     }
 
     if (op_type == "sum") {
diff --git a/test/ir/inference/test_trt_convert_softmax.py b/test/ir/inference/test_trt_convert_softmax.py
index 8a546d71b2c36..7fb9a69bb63ee 100644
--- a/test/ir/inference/test_trt_convert_softmax.py
+++ b/test/ir/inference/test_trt_convert_softmax.py
@@ -47,8 +47,12 @@ def generate_input1(attrs: List[Dict[str, Any]], batch):
                 return np.ones([batch, 3, 24]).astype(np.float32)
             elif self.dims == 2:
                 return np.ones([batch, 32]).astype(np.float32)
+            elif self.dims == 1:
+                return np.ones([batch]).astype(np.float32)
+            elif self.dims == 0:
+                return np.ones([]).astype(np.float32)
 
-        for dims in [2, 3, 4]:
+        for dims in [0, 1, 2, 3, 4]:
             for batch in [1, 2, 4]:
                 for axis in [-1, 0, 1, 2, 3]:
                     self.dims = dims
@@ -103,6 +107,14 @@ def generate_dynamic_shape(attrs):
                 self.dynamic_shape.min_input_shape = {"softmax_input": [1, 32]}
                 self.dynamic_shape.max_input_shape = {"softmax_input": [4, 64]}
                 self.dynamic_shape.opt_input_shape = {"softmax_input": [1, 32]}
+            elif self.dims == 1:
+                self.dynamic_shape.min_input_shape = {"softmax_input": [1]}
+                self.dynamic_shape.max_input_shape = {"softmax_input": [4]}
+                self.dynamic_shape.opt_input_shape = {"softmax_input": [1]}
+            elif self.dims == 0:
+                self.dynamic_shape.min_input_shape = {"softmax_input": []}
+                self.dynamic_shape.max_input_shape = {"softmax_input": []}
+                self.dynamic_shape.opt_input_shape = {"softmax_input": []}
 
         def clear_dynamic_shape():
             self.dynamic_shape.min_input_shape = {}
@@ -110,6 +122,8 @@ def clear_dynamic_shape():
             self.dynamic_shape.opt_input_shape = {}
 
         def generate_trt_nodes_num(attrs, dynamic_shape):
+            if not dynamic_shape and (self.dims == 1 or self.dims == 0):
+                return 0, 3
             return 1, 2
 
         attrs = [
diff --git a/test/ir/inference/test_trt_convert_stack.py b/test/ir/inference/test_trt_convert_stack.py
index 087ba953288fa..39f729e2be712 100644
--- a/test/ir/inference/test_trt_convert_stack.py
+++ b/test/ir/inference/test_trt_convert_stack.py
@@ -32,9 +32,11 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool:
         attrs = [
             program_config.ops[i].attrs for i in range(len(program_config.ops))
         ]
-        # The input dimension should be less than the set axis.
+        # axis must be inside [-(rank+1), rank+1)
         if len(inputs['stack_input1'].shape) < attrs[0]['axis']:
             return False
+        if -(len(inputs['stack_input1'].shape) + 1) > attrs[0]['axis']:
+            return False
 
         return True
 
@@ -48,6 +50,8 @@ def generate_input1(attrs: List[Dict[str, Any]], batch):
                 return np.random.random([batch, 24]).astype(np.float32)
             elif self.dims == 1:
                 return np.random.random([24]).astype(np.float32)
+            elif self.dims == 0:
+                return np.random.random([]).astype(np.float32)
 
         def generate_input2(attrs: List[Dict[str, Any]], batch):
             if self.dims == 4:
@@ -58,6 +62,8 @@ def generate_input2(attrs: List[Dict[str, Any]], batch):
                 return np.random.random([batch, 24]).astype(np.float32)
             elif self.dims == 1:
                 return np.random.random([24]).astype(np.float32)
+            elif self.dims == 0:
+                return np.random.random([]).astype(np.float32)
 
         def generate_input3(attrs: List[Dict[str, Any]], batch):
             if self.dims == 4:
@@ -68,8 +74,10 @@ def generate_input3(attrs: List[Dict[str, Any]], batch):
                 return np.random.random([batch, 24]).astype(np.float32)
             elif self.dims == 1:
                 return np.random.random([24]).astype(np.float32)
+            elif self.dims == 0:
+                return np.random.random([]).astype(np.float32)
 
-        for dims in [1, 2, 3, 4]:
+        for dims in [0, 1, 2, 3, 4]:
             for batch in [1, 4]:
                 for axis in [-2, -1, 0, 1, 2, 3]:
                     self.dims = dims
@@ -176,6 +184,22 @@ def generate_dynamic_shape(attrs):
                     "stack_input2": [24],
                     "stack_input3": [24],
                 }
+            elif self.dims == 0:
+                self.dynamic_shape.min_input_shape = {
+                    "stack_input1": [],
+                    "stack_input2": [],
+                    "stack_input3": [],
+                }
+                self.dynamic_shape.max_input_shape = {
+                    "stack_input1": [],
+                    "stack_input2": [],
+                    "stack_input3": [],
+                }
+                self.dynamic_shape.opt_input_shape = {
+                    "stack_input1": [],
+                    "stack_input2": [],
+                    "stack_input3": [],
+                }
 
         def clear_dynamic_shape():
             self.dynamic_shape.min_input_shape = {}

From eb97f4f0adca40b16a309b927e480178beb8ae96 Mon Sep 17 00:00:00 2001
From: Wilber <jiweibo@baidu.com>
Date: Fri, 12 May 2023 13:29:50 +0800
Subject: [PATCH 303/405] [Inference] Update switch stream logical. (#53589)

---
 .../fluid/inference/api/analysis_predictor.cc | 136 +++++++------
 .../fluid/inference/api/resource_manager.cc   | 137 ++-----------
 paddle/fluid/inference/api/resource_manager.h |  12 +-
 test/cpp/inference/api/CMakeLists.txt         |  16 ++
 .../inference/api/trt_rebind_stream_test.cc   | 183 ++++++++++++++++++
 5 files changed, 295 insertions(+), 189 deletions(-)
 create mode 100644 test/cpp/inference/api/trt_rebind_stream_test.cc

diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc
index 6732f2dbe366c..1ab5366d34c95 100644
--- a/paddle/fluid/inference/api/analysis_predictor.cc
+++ b/paddle/fluid/inference/api/analysis_predictor.cc
@@ -94,6 +94,61 @@
 #endif
 
 namespace paddle {
+namespace {
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+void UpdatePrivateDeviceContext(InferGPUContext *gpu_context,
+                                GPUContextResource *gpu_resource,
+                                Place place_) {
+  gpu_context->SetAllocator(memory::allocation::AllocatorFacade::Instance()
+                                .GetAllocator(place_, gpu_resource->GetStream())
+                                .get());
+  gpu_context->SetPinnedAllocator(
+      memory::allocation::AllocatorFacade::Instance()
+          .GetAllocator(paddle::platform::CUDAPinnedPlace())
+          .get());
+  gpu_context->SetHostAllocator(memory::allocation::AllocatorFacade::Instance()
+                                    .GetAllocator(platform::CPUPlace())
+                                    .get());
+  gpu_context->SetZeroAllocator(memory::allocation::AllocatorFacade::Instance()
+                                    .GetZeroAllocator(place_)
+                                    .get());
+  gpu_context->SetHostZeroAllocator(
+      memory::allocation::AllocatorFacade::Instance()
+          .GetZeroAllocator(platform::CPUPlace())
+          .get());
+  gpu_context->SetGenerator(
+      phi::DefaultCUDAGenerator(place_.GetDeviceId()).get());
+  gpu_context->SetHostGenerator(phi::DefaultCPUGenerator().get());
+
+  gpu_context->SetStream(gpu_resource->GetStream());
+  gpu_context->SetBlasHandle(gpu_resource->GetBlasHandleCreator());
+  gpu_context->SetBlasTensorCoreHandle(
+      gpu_resource->GetBlasTensorCoreHandleCreator());
+  gpu_context->SetBlasTF32Handle(
+      gpu_resource->GetBlasTF32TensorCoreHandleCreator());
+  gpu_context->SetDnnHandle(gpu_resource->GetDnnHandleCreator());
+  gpu_context->SetSolverHandle(gpu_resource->GetSolverDnHandleCreator());
+  gpu_context->SetSparseHandle(gpu_resource->GetSparseHandleCreator());
+  gpu_context->SetEigenDevice(gpu_resource->GetGpuEigenDevice());
+
+  gpu_context->SetComputeCapability(gpu_resource->GetGpuComputeCapability());
+  gpu_context->SetMaxThreadsPerBlock(gpu_resource->GetGpuMaxThreadsPerBlock());
+  gpu_context->SetMaxThreadsPerMultiProcessor(
+      gpu_resource->GetGpuMaxThreadsPerMp());
+  gpu_context->SetMaxGridDimSize(gpu_resource->GetGpuMaxGridDimSize());
+  gpu_context->SetMultiProcessors(gpu_resource->GetGPUMultiProcessors());
+  gpu_context->SetDriverVersion(gpu_resource->GetGpuDriverVersion());
+  gpu_context->SetRuntimeVersion(gpu_resource->GetGpuRuntimeVersion());
+  VLOG(1) << "thread id is " << std::this_thread::get_id() << ", stream id is "
+          << reinterpret_cast<void *>(gpu_resource->GetStream())
+          << ", allotor ptr is "
+          << reinterpret_cast<void *>(
+                 memory::allocation::AllocatorFacade::Instance()
+                     .GetAllocator(place_, gpu_resource->GetStream())
+                     .get());
+}
+#endif
+}  // namespace
 
 using inference::Singleton;
 #ifdef PADDLE_WITH_TENSORRT
@@ -451,60 +506,7 @@ void AnalysisPredictor::InitDeviceContexts() {
           auto *gpu_resource =
               ResourceManager::Instance().GetGPUResource(predictor_stream_);
           auto *gpu_context = new InferGPUContext(place_);
-          gpu_context->SetAllocator(
-              memory::allocation::AllocatorFacade::Instance()
-                  .GetAllocator(place_, gpu_resource->GetStream())
-                  .get());
-          gpu_context->SetPinnedAllocator(
-              memory::allocation::AllocatorFacade::Instance()
-                  .GetAllocator(paddle::platform::CUDAPinnedPlace())
-                  .get());
-          gpu_context->SetHostAllocator(
-              memory::allocation::AllocatorFacade::Instance()
-                  .GetAllocator(platform::CPUPlace())
-                  .get());
-          gpu_context->SetZeroAllocator(
-              memory::allocation::AllocatorFacade::Instance()
-                  .GetZeroAllocator(place_)
-                  .get());
-          gpu_context->SetHostZeroAllocator(
-              memory::allocation::AllocatorFacade::Instance()
-                  .GetZeroAllocator(platform::CPUPlace())
-                  .get());
-          gpu_context->SetGenerator(
-              phi::DefaultCUDAGenerator(place_.GetDeviceId()).get());
-          gpu_context->SetHostGenerator(phi::DefaultCPUGenerator().get());
-
-          gpu_context->SetStream(gpu_resource->GetStream());
-          gpu_context->SetBlasHandle(gpu_resource->GetBlasHandleCreator());
-          gpu_context->SetBlasTensorCoreHandle(
-              gpu_resource->GetBlasTensorCoreHandleCreator());
-          gpu_context->SetBlasTF32Handle(
-              gpu_resource->GetBlasTF32TensorCoreHandleCreator());
-          gpu_context->SetDnnHandle(gpu_resource->GetDnnHandleCreator());
-          gpu_context->SetSolverHandle(
-              gpu_resource->GetSolverDnHandleCreator());
-          gpu_context->SetSparseHandle(gpu_resource->GetSparseHandleCreator());
-          gpu_context->SetEigenDevice(gpu_resource->GetGpuEigenDeviceCreator());
-          gpu_context->SetComputeCapability(
-              gpu_resource->GetGpuComputeCapability());
-          gpu_context->SetMaxThreadsPerBlock(
-              gpu_resource->GetGpuMaxThreadsPerBlock());
-          gpu_context->SetMaxThreadsPerMultiProcessor(
-              gpu_resource->GetGpuMaxThreadsPerMp());
-          gpu_context->SetMaxGridDimSize(gpu_resource->GetGpuMaxGridDimSize());
-          gpu_context->SetMultiProcessors(
-              gpu_resource->GetGPUMultiProcessors());
-          gpu_context->SetDriverVersion(gpu_resource->GetGpuDriverVersion());
-          gpu_context->SetRuntimeVersion(gpu_resource->GetGpuRuntimeVersion());
-          VLOG(1) << "thread id is " << std::this_thread::get_id()
-                  << ", stream id is "
-                  << reinterpret_cast<void *>(gpu_resource->GetStream())
-                  << ", allotor ptr is "
-                  << reinterpret_cast<void *>(
-                         memory::allocation::AllocatorFacade::Instance()
-                             .GetAllocator(place_, gpu_resource->GetStream())
-                             .get());
+          UpdatePrivateDeviceContext(gpu_context, gpu_resource, place_);
           return std::unique_ptr<phi::DeviceContext>(gpu_context);
         }));
   }
@@ -2083,17 +2085,27 @@ bool AnalysisPredictor::ExpRunWithExternalStream(const gpuStream_t stream) {
 #else
     cudaStreamSynchronize(static_cast<gpuStream_t>(predictor_stream_));
 #endif
-    ResourceManager::Instance().GpuResourceReBindStream(predictor_stream_,
+    ResourceManager::Instance().GpuResourceSwitchStream(predictor_stream_,
                                                         stream);
     predictor_stream_ = stream;
 
-    auto *dev_ctxs = reinterpret_cast<const std::map<
-        phi::Place,
-        std::shared_future<std::unique_ptr<phi::DeviceContext>>> *>(
-        this->GetDeviceContexts());
-    auto *dev_ctx =
-        static_cast<InferGPUContext *>(dev_ctxs->at(place_).get().get());
-    dev_ctx->SetStream(stream);
+    auto *dev_ctxs = const_cast<
+        std::map<phi::Place,
+                 std::shared_future<std::unique_ptr<phi::DeviceContext>>> *>(
+        reinterpret_cast<const std::map<
+            phi::Place,
+            std::shared_future<std::unique_ptr<phi::DeviceContext>>> *>(
+            this->GetDeviceContexts()));
+
+    dev_ctxs->erase(place_);
+    dev_ctxs->emplace(
+        place_, std::async(std::launch::deferred, [=] {
+          auto *gpu_resource =
+              ResourceManager::Instance().GetGPUResource(predictor_stream_);
+          auto *gpu_context = new InferGPUContext(place_);
+          UpdatePrivateDeviceContext(gpu_context, gpu_resource, place_);
+          return std::unique_ptr<phi::DeviceContext>(gpu_context);
+        }));
   }
 
   return ZeroCopyRun();
diff --git a/paddle/fluid/inference/api/resource_manager.cc b/paddle/fluid/inference/api/resource_manager.cc
index 0f3612a5e976c..7968b8d7d9d03 100644
--- a/paddle/fluid/inference/api/resource_manager.cc
+++ b/paddle/fluid/inference/api/resource_manager.cc
@@ -154,6 +154,7 @@ void GPUContextResource::InitGPUResource(void* stream) {
   }
 
   InitGpuProperties();
+  InitGpuEigenDevice();
 }
 
 void GPUContextResource::DestroyGPUResource() {
@@ -361,90 +362,6 @@ std::array<int, 3> GPUContextResource::GetGpuMaxGridDimSize() const {
   return max_grid_dim_size_;
 }
 
-void GPUContextResource::ReBindStream(gpuStream_t stream) {
-  owned_stream_ = false;
-  stream_ = stream;
-}
-
-void GPUContextResource::ReBindDnnHandle(gpuStream_t stream) const {
-  if (dnn_handle_) {
-#ifdef PADDLE_WITH_HIP
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        phi::dynload::miopenSetStream(dnn_handle_, stream));
-#else
-    PADDLE_RETRY_CUDA_SUCCESS(
-        phi::dynload::cudnnSetStream(dnn_handle_, stream));
-#endif
-  }
-}
-
-void GPUContextResource::ReBindBlasHandle(gpuStream_t stream) const {
-  if (blas_handle_) {
-#ifdef PADDLE_WITH_HIP
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        phi::dynload::rocblas_set_stream(blas_handle_, stream));
-#else
-    PADDLE_RETRY_CUDA_SUCCESS(
-        phi::dynload::cublasSetStream(blas_handle_, stream));
-#endif
-  }
-}
-
-void GPUContextResource::ReBindBlasTensorCoreHandle(gpuStream_t stream) const {
-  if (blas_tensor_core_handle_) {
-#ifdef PADDLE_WITH_HIP
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        phi::dynload::rocblas_set_stream(blas_tensor_core_handle_, stream));
-#else
-    PADDLE_RETRY_CUDA_SUCCESS(
-        phi::dynload::cublasSetStream(blas_tensor_core_handle_, stream));
-#endif
-  }
-}
-
-void GPUContextResource::ReBindBlasTF32Handle(gpuStream_t stream) const {
-  if (blas_tf32_tensor_core_handle_) {
-#ifdef PADDLE_WITH_HIP
-    PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::rocblas_set_stream(
-        blas_tf32_tensor_core_handle_, stream));
-#else
-    PADDLE_RETRY_CUDA_SUCCESS(
-        phi::dynload::cublasSetStream(blas_tf32_tensor_core_handle_, stream));
-#endif
-  }
-}
-
-void GPUContextResource::ReBindSolverDnHandle(gpuStream_t stream) const {
-  if (solver_handle_) {
-#ifndef PADDLE_WITH_HIP
-    PADDLE_RETRY_CUDA_SUCCESS(
-        phi::dynload::cusolverDnSetStream(solver_handle_, stream));
-#endif
-  }
-}
-
-void GPUContextResource::ReBindSparseHandle(gpuStream_t stream) const {
-  if (sparse_handle_) {
-#if defined(PADDLE_WITH_CUDA)
-// The generic APIs is supported from CUDA10.1
-#if CUDA_VERSION >= 11000
-    PADDLE_RETRY_CUDA_SUCCESS(
-        phi::dynload::cusparseSetStream(sparse_handle_, stream));
-#endif
-#endif
-  }
-}
-
-void GPUContextResource::ReBindEigenDevice(gpuStream_t stream,
-                                           GPUPlace place) const {
-  if (eigen_stream_) {
-    auto* allocator = paddle::memory::allocation::AllocatorFacade::Instance()
-                          .GetAllocator(place_)
-                          .get();
-    eigen_stream_->Reinitialize(stream, allocator, place);
-  }
-}
-
 #endif
 
 void ResourceManager::InitCPUResource() {
@@ -486,24 +403,16 @@ void ResourceManager::DestroyGPUResource(void* stream) {
 }
 
 void ResourceManager::Decrease(void* stream) {
-  PADDLE_ENFORCE_EQ(ref_count_.count(stream),
-                    true,
-                    platform::errors::InvalidArgument(
-                        "The stream[%p] not found in ref_count.", stream));
+  if (ref_count_.count(stream) == 0) return;
   --ref_count_[stream];
+
   if (ref_count_[stream] == 0) {
     ref_count_.erase(stream);
-    gpu_resources_.erase(stream);
+    if (gpu_resources_.count(stream) > 0) gpu_resources_.erase(stream);
   }
 }
 
-void ResourceManager::Increase(void* stream) {
-  PADDLE_ENFORCE_EQ(ref_count_.count(stream),
-                    true,
-                    platform::errors::InvalidArgument(
-                        "The stream[%p] not found in ref_count.", stream));
-  ++ref_count_[stream];
-}
+void ResourceManager::Increase(void* stream) { ++ref_count_[stream]; }
 
 GPUContextResource* ResourceManager::GetGPUResource(void* stream) const {
   PADDLE_ENFORCE_EQ(gpu_resources_.count(stream),
@@ -513,33 +422,29 @@ GPUContextResource* ResourceManager::GetGPUResource(void* stream) const {
   return gpu_resources_.at(stream).get();
 }
 
-void ResourceManager::GpuResourceReBindStream(void* old_stream,
+void ResourceManager::GpuResourceSwitchStream(void* old_stream,
                                               void* new_stream) {
+  // NOTE: add lock to support stream rebind in multi-thread
+  std::lock_guard<std::mutex> lock_gurad(gpu_mutex_);
+  if (old_stream == new_stream) return;
   PADDLE_ENFORCE_EQ(
       gpu_resources_.count(old_stream),
       true,
       platform::errors::InvalidArgument(
           "The stream[%p] not found in gpu_resources.", old_stream));
-  auto gpu_resource = std::move(gpu_resources_.at(old_stream));
-  DestroyGPUResource(old_stream);
-  PADDLE_ENFORCE_EQ(
-      ref_count_.count(old_stream),
-      0,
-      platform::errors::Fatal("gpu resources rebind stream failed."));
-
-  gpu_resource->ReBindStream(static_cast<gpuStream_t>(new_stream));
-  gpu_resource->ReBindDnnHandle(static_cast<gpuStream_t>(new_stream));
-  gpu_resource->ReBindBlasHandle(static_cast<gpuStream_t>(new_stream));
-  gpu_resource->ReBindBlasTensorCoreHandle(
-      static_cast<gpuStream_t>(new_stream));
-  gpu_resource->ReBindBlasTF32Handle(static_cast<gpuStream_t>(new_stream));
-  gpu_resource->ReBindSolverDnHandle(static_cast<gpuStream_t>(new_stream));
-  gpu_resource->ReBindSparseHandle(static_cast<gpuStream_t>(new_stream));
-  gpu_resource->ReBindEigenDevice(static_cast<gpuStream_t>(new_stream),
-                                  gpu_resource->Place());
 
-  ref_count_[new_stream]++;
-  gpu_resources_.emplace(new_stream, std::move(gpu_resource));
+  // NOTE: stream may be used by multiple predictor, skip resource
+  //       operation if resource of new_stream is already exists
+  bool new_stream_existed = gpu_resources_.count(new_stream) > 0;
+  if (!new_stream_existed) {
+    auto place = gpu_resources_.at(old_stream)->Place();
+    std::unique_ptr<GPUContextResource> resource{
+        new GPUContextResource(place, new_stream)};
+    gpu_resources_.emplace(new_stream, std::move(resource));
+  }
+
+  Decrease(old_stream);
+  Increase(new_stream);
 }
 
 int ResourceManager::RefCount(void* stream) const {
diff --git a/paddle/fluid/inference/api/resource_manager.h b/paddle/fluid/inference/api/resource_manager.h
index 119b5d94dc56c..3b4c66d3190c9 100644
--- a/paddle/fluid/inference/api/resource_manager.h
+++ b/paddle/fluid/inference/api/resource_manager.h
@@ -82,16 +82,6 @@ class GPUContextResource {
   int GetGpuMaxThreadsPerBlock() const;
   std::array<int, 3> GetGpuMaxGridDimSize() const;
 
-  // If stream changes, we need to rebind all handle to new stream.
-  void ReBindStream(gpuStream_t stream);
-  void ReBindDnnHandle(gpuStream_t stream) const;
-  void ReBindBlasHandle(gpuStream_t stream) const;
-  void ReBindBlasTensorCoreHandle(gpuStream_t stream) const;
-  void ReBindBlasTF32Handle(gpuStream_t stream) const;
-  void ReBindSolverDnHandle(gpuStream_t stream) const;
-  void ReBindSparseHandle(gpuStream_t stream) const;
-  void ReBindEigenDevice(gpuStream_t stream, GPUPlace place) const;
-
  private:
   void InitGPUResource(void* stream);
   void DestroyGPUResource();
@@ -186,7 +176,7 @@ class ResourceManager {
   void DestroyGPUResource(void* stream);
   GPUContextResource* GetGPUResource(void* stream) const;
   int RefCount(void* stream) const;
-  void GpuResourceReBindStream(void* old_stream, void* new_stream);
+  void GpuResourceSwitchStream(void* old_stream, void* new_stream);
 
  private:
   void Decrease(void* stream);
diff --git a/test/cpp/inference/api/CMakeLists.txt b/test/cpp/inference/api/CMakeLists.txt
index c4045e9ca603a..3c310e68065f0 100644
--- a/test/cpp/inference/api/CMakeLists.txt
+++ b/test/cpp/inference/api/CMakeLists.txt
@@ -1028,6 +1028,18 @@ if(WITH_TESTING AND WITH_INFERENCE_API_TEST)
       target_link_libraries(test_analyzer_capi_exp_xpu paddle_inference_c)
     endif()
 
+    #TODO(inference): windows encounter a SEH error, we need to fix it.
+    if(NOT WIN32)
+      inference_analysis_test(
+        trt_rebind_stream_test
+        SRCS
+        trt_rebind_stream_test.cc
+        EXTRA_DEPS
+        paddle_inference_shared
+        ARGS
+        --infer_model=${TRT_MODEL_INSTALL_DIR}/trt_inference_test_models)
+    endif()
+
     set(TRT_MODEL_QUANT_RESNET_DIR
         "${INFERENCE_DEMO_INSTALL_DIR}/small_quant_model")
     if(NOT EXISTS ${INFERENCE_DEMO_INSTALL_DIR}/small_quant_model.tgz)
@@ -1378,6 +1390,10 @@ if(WITH_TESTING AND WITH_INFERENCE_API_TEST)
   endif()
   if(WITH_GPU AND TENSORRT_FOUND)
     set_tests_properties(trt_mobilenet_test PROPERTIES TIMEOUT 240)
+    if(NOT WIN32)
+      set_tests_properties(trt_rebind_stream_test
+                           PROPERTIES TIMEOUT 360 LABELS "RUN_TYPE=EXCLUSIVE")
+    endif()
     if(WITH_MKLDNN)
       set_tests_properties(test_analyzer_bfloat16_resnet50 PROPERTIES TIMEOUT
                                                                       120)
diff --git a/test/cpp/inference/api/trt_rebind_stream_test.cc b/test/cpp/inference/api/trt_rebind_stream_test.cc
new file mode 100644
index 0000000000000..3a42af93427c5
--- /dev/null
+++ b/test/cpp/inference/api/trt_rebind_stream_test.cc
@@ -0,0 +1,183 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <thread>
+
+#include "gflags/gflags.h"
+#include "paddle/fluid/inference/api/paddle_inference_api.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+// TODO(inference): This case failed in windows with a SEH error, we need to fix
+// it.
+TEST(ReBindStream_single, use_gpu) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  AnalysisConfig config;
+  config.EnableUseGpu(100, 0);
+  config.SetModel(model_dir);
+  config.EnableTensorRtEngine();
+
+  cudaStream_t stream1, stream2, stream3;
+  cudaStreamCreate(&stream1);
+  cudaStreamCreate(&stream2);
+  cudaStreamCreate(&stream3);
+
+  config.SetExecStream(stream1);
+  auto predictor = paddle_infer::CreatePredictor(config);
+  auto x_t = predictor->GetInputHandle("x");
+  x_t->Reshape({1, 3, 224, 224});
+  float x_data[3 * 224 * 224] = {0};
+  x_t->CopyFromCpu(x_data);
+  ASSERT_TRUE(predictor->Run());
+  cudaDeviceSynchronize();
+  ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
+      predictor.get(), stream2));
+  cudaDeviceSynchronize();
+  ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
+      predictor.get(), stream3));
+  cudaDeviceSynchronize();
+}
+
+TEST(ReBindStream_multi, use_gpu) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  AnalysisConfig config1;
+  config1.EnableUseGpu(100, 0);
+  config1.SetModel(model_dir);
+  config1.EnableTensorRtEngine();
+  AnalysisConfig config2;
+  config2.EnableUseGpu(100, 0);
+  config2.EnableTensorRtEngine();
+  config2.SetModel(model_dir);
+
+  cudaStream_t stream1, stream2, stream3;
+  cudaStreamCreate(&stream1);
+  cudaStreamCreate(&stream2);
+  cudaStreamCreate(&stream3);
+
+  config1.SetExecStream(stream1);
+  config2.SetExecStream(stream1);
+  auto predictor1 = paddle_infer::CreatePredictor(config1);
+  auto predictor2 = paddle_infer::CreatePredictor(config2);
+
+  std::vector<float> x1(3 * 224 * 224, 1.0);
+  auto x_t1 = predictor1->GetInputHandle("x");
+  x_t1->Reshape({1, 3, 224, 224});
+  x_t1->CopyFromCpu(x1.data());
+  std::vector<float> x2(3 * 224 * 224, 2.0);
+  auto x_t2 = predictor2->GetInputHandle("x");
+  x_t2->Reshape({1, 3, 224, 224});
+  x_t2->CopyFromCpu(x2.data());
+
+  ASSERT_TRUE(predictor1->Run());
+  cudaStreamSynchronize(stream1);
+  ASSERT_TRUE(predictor2->Run());
+  cudaStreamSynchronize(stream1);
+
+  ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
+      predictor1.get(), stream2));
+  cudaDeviceSynchronize();
+  ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
+      predictor2.get(), stream2));
+  cudaDeviceSynchronize();
+
+  ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
+      predictor1.get(), stream3));
+  cudaStreamSynchronize(stream3);
+  ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
+      predictor2.get(), stream3));
+  cudaStreamSynchronize(stream3);
+}
+
+TEST(SwitchStream_multi, use_gpu) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  AnalysisConfig config1;
+  config1.EnableUseGpu(100, 0);
+  config1.SetModel(model_dir);
+  AnalysisConfig config2;
+  config2.EnableUseGpu(100, 0);
+  config2.SetModel(model_dir);
+  AnalysisConfig config3;
+  config3.EnableUseGpu(100, 0);
+  config3.SetModel(model_dir);
+
+  // config1.EnableTensorRtEngine();
+  // config2.EnableTensorRtEngine();
+  // config3.EnableTensorRtEngine();
+
+  cudaStream_t stream1, stream2, stream3;
+  cudaStreamCreate(&stream1);
+  cudaStreamCreate(&stream2);
+  cudaStreamCreate(&stream3);
+
+  config1.SetExecStream(stream1);
+  config2.SetExecStream(stream1);
+  config3.SetExecStream(stream1);
+  auto predictor1 = paddle_infer::CreatePredictor(config1);
+  auto predictor2 = paddle_infer::CreatePredictor(config2);
+  auto predictor3 = paddle_infer::CreatePredictor(config3);
+
+  std::vector<float> x1(3 * 224 * 224, 1.0);
+  auto x_t1 = predictor1->GetInputHandle("x");
+  x_t1->Reshape({1, 3, 224, 224});
+  x_t1->CopyFromCpu(x1.data());
+  std::vector<float> x2(3 * 224 * 224, 2.0);
+  auto x_t2 = predictor2->GetInputHandle("x");
+  x_t2->Reshape({1, 3, 224, 224});
+  x_t2->CopyFromCpu(x2.data());
+  std::vector<float> x3(3 * 224 * 224, 2.5);
+  auto x_t3 = predictor3->GetInputHandle("x");
+  x_t3->Reshape({1, 3, 224, 224});
+  x_t3->CopyFromCpu(x3.data());
+
+  // TODO(wilber): fix.
+  // NOTE: Must run once on master thread, but why?
+  // if remove the code, the unit test fail.
+  ASSERT_TRUE(predictor1->Run());
+  cudaStreamSynchronize(stream1);
+  ASSERT_TRUE(predictor2->Run());
+  cudaStreamSynchronize(stream1);
+  ASSERT_TRUE(predictor3->Run());
+  cudaStreamSynchronize(stream1);
+
+  auto Run = [&](paddle_infer::Predictor* p,
+                 std::vector<cudaStream_t> streams) {
+    for (auto s : streams) {
+      paddle_infer::experimental::InternalUtils::RunWithExternalStream(p, s);
+    }
+  };
+
+  std::thread p1(Run,
+                 predictor1.get(),
+                 std::vector<cudaStream_t>{
+                     stream1, stream2, stream3, stream3, stream2, stream2});
+  std::thread p2(Run,
+                 predictor2.get(),
+                 std::vector<cudaStream_t>{
+                     stream1, stream3, stream1, stream2, stream1, stream3});
+  std::thread p3(Run,
+                 predictor3.get(),
+                 std::vector<cudaStream_t>{
+                     stream1, stream1, stream2, stream3, stream3, stream2});
+  p1.join();
+  p2.join();
+  p3.join();
+  cudaDeviceSynchronize();
+}
+
+}  // namespace inference
+}  // namespace paddle

From df8c302765b687ff203a00d3c110224bf1b097a2 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Fri, 12 May 2023 13:45:31 +0800
Subject: [PATCH 304/405] Remove the softmax_out argument.

---
 paddle/fluid/operators/fused/fused_gate_attention.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 19659a46e1cc2..cd3d5f96cc6cb 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -944,7 +944,6 @@ class FlashAttnWithGating {
         is_bf16,
         fa_num_splits_,
         softmax_lse->data(),
-        nullptr,  // softmax out,
         nullptr,
         &workspace_size,
         stream,
@@ -981,7 +980,6 @@ class FlashAttnWithGating {
         is_bf16,
         fa_num_splits_,
         softmax_lse->data(),
-        nullptr,  // softmax out
         (workspace_size > 0) ? static_cast<void*>(workspace.data()) : nullptr,
         &workspace_size,
         stream,

From d03bbefa96ee41100a3e1d9a26e6986845bfc918 Mon Sep 17 00:00:00 2001
From: ronnywang <ronny1996@163.com>
Date: Fri, 12 May 2023 14:43:38 +0800
Subject: [PATCH 305/405] [CustomDevice] add inference MP support, PART0
 (#53719)

* [CustomDevice] add inference MP support, PART0

* update
---
 paddle/fluid/imperative/CMakeLists.txt       |  20 +-
 paddle/fluid/imperative/xccl_context.cc      | 282 +++++++++++++++++++
 paddle/fluid/imperative/xccl_context.h       |  71 +++++
 paddle/fluid/platform/collective_helper.cc   | 230 +++++++++++++++
 paddle/fluid/platform/collective_helper.h    | 108 +++++++
 paddle/fluid/platform/gen_comm_id_helper.cc  |  60 +++-
 paddle/fluid/platform/gen_comm_id_helper.h   |   2 +-
 paddle/phi/backends/c_comm_lib.h             |   4 +-
 paddle/phi/backends/custom/custom_context.cc |  13 +
 paddle/phi/backends/custom/custom_context.h  |   7 +
 10 files changed, 791 insertions(+), 6 deletions(-)
 create mode 100644 paddle/fluid/imperative/xccl_context.cc
 create mode 100644 paddle/fluid/imperative/xccl_context.h

diff --git a/paddle/fluid/imperative/CMakeLists.txt b/paddle/fluid/imperative/CMakeLists.txt
index 8ef07cfa76e8c..f6fe845b30cab 100644
--- a/paddle/fluid/imperative/CMakeLists.txt
+++ b/paddle/fluid/imperative/CMakeLists.txt
@@ -123,10 +123,26 @@ if(NOT WIN32)
       SRCS reducer.cc
       DEPS layer)
   endif()
-
+  if(WITH_CUSTOM_DEVICE)
+    cc_library(
+      xccl_context
+      SRCS xccl_context.cc
+      DEPS collective_helper device_context tensor var_type_traits)
+    if(NOT
+       (WITH_NCCL
+        OR WITH_RCCL
+        OR WITH_XPU_BKCL
+        OR WITH_GLOO))
+      cc_library(
+        reducer
+        SRCS reducer.cc
+        DEPS layer)
+    endif()
+  endif()
   if(WITH_NCCL
      OR WITH_RCCL
-     OR WITH_XPU_BKCL)
+     OR WITH_XPU_BKCL
+     OR WITH_CUSTOM_DEVICE)
     cc_library(
       heter_ccl_context
       SRCS heter_ccl_context.cc
diff --git a/paddle/fluid/imperative/xccl_context.cc b/paddle/fluid/imperative/xccl_context.cc
new file mode 100644
index 0000000000000..dc7a4b939b7bd
--- /dev/null
+++ b/paddle/fluid/imperative/xccl_context.cc
@@ -0,0 +1,282 @@
+//   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/imperative/xccl_context.h"
+
+#if defined(PADDLE_WITH_CUSTOM_DEVICE)
+#include "paddle/fluid/platform/collective_helper.h"
+#include "paddle/fluid/platform/gen_comm_id_helper.h"
+#endif
+
+#include "paddle/fluid/framework/convert_utils.h"
+#include "paddle/fluid/framework/scope.h"
+#include "paddle/fluid/framework/variable.h"
+#include "paddle/fluid/platform/device_context.h"
+#include "paddle/fluid/platform/place.h"
+
+namespace paddle {
+namespace framework {
+class Variable;
+}  // namespace framework
+}  // namespace paddle
+
+namespace paddle {
+namespace imperative {
+
+static void XcclAllReduce(const phi::DenseTensor &src,
+                          phi::DenseTensor *dst,
+                          const phi::stream::Stream &stream,
+                          const phi::ccl::CCLComm &comm) {
+  const auto &place = src.place();
+  PADDLE_ENFORCE_EQ(
+      platform::is_custom_place(place),
+      true,
+      platform::errors::Unimplemented(
+          "Dynamic graph mode does not support multi-CPU training yet."));
+
+  void *src_ptr = const_cast<void *>(src.data());
+  dst->Resize(src.dims());
+  auto *dst_ptr = phi::DeviceContextPool::Instance()
+                      .Get(src.place())
+                      ->Alloc(dst, src.dtype());
+  auto xccl_dtype = phi::ccl::ToCCLDataType(src.dtype());
+
+  phi::DeviceManager::CCLAllReduce(place.GetDeviceType(),
+                                   src_ptr,
+                                   dst_ptr,
+                                   src.numel(),
+                                   xccl_dtype,
+                                   phi::ccl::CCLReduceOp::SUM,
+                                   comm,
+                                   stream);
+}
+
+void XCCLParallelContext::BcastXCCLId(
+    std::vector<phi::ccl::CCLRootId> &xccl_ids,  // NOLINT
+    int root,
+    int server_fd) {
+  if (strategy_.local_rank_ == root) {
+    std::vector<std::string> other_trainers;
+    for (auto &ep : strategy_.trainer_endpoints_) {
+      if (ep != strategy_.current_endpoint_) {
+        other_trainers.push_back(ep);
+      }
+    }
+    platform::SendBroadCastCommID(other_trainers, &xccl_ids);
+  } else {
+    platform::RecvBroadCastCommID(
+        server_fd, strategy_.current_endpoint_, &xccl_ids);
+  }
+}
+
+void XCCLParallelContext::Init() {
+  int server_fd = -1;
+
+  std::vector<phi::ccl::CCLRootId> xccl_ids;
+  xccl_ids.resize(strategy_.nrings_);
+
+  if (strategy_.local_rank_ == 0) {
+    // generate the unique ncclid on the root worker
+    for (size_t i = 0; i < xccl_ids.size(); ++i) {
+      phi::DeviceManager::CCLGetUniqueId(place_.GetDeviceType(), &xccl_ids[i]);
+    }
+  } else {
+    // FIXME(wangxi): gloo will use rank0 endpoint, so not create socket server
+    // on rank0.
+    server_fd = platform::SocketServer::GetInstance(strategy_.current_endpoint_)
+                    .socket();
+  }
+  BcastXCCLId(xccl_ids, 0, server_fd);
+
+  int dev_id = place_.device;
+  for (int ring_id = 0; ring_id < strategy_.nrings_; ring_id++) {
+    VLOG(0) << "init nccl context nranks: " << strategy_.nranks_
+            << " local rank: " << strategy_.local_rank_ << " dev id: " << dev_id
+            << " ring id: " << ring_id;
+    // it will assign nccl_comm in phi::CustomContext within ring_id
+    platform::XCCLCommContext::Instance(place_.GetDeviceType())
+        .CreateComm(&xccl_ids[ring_id],
+                    strategy_.nranks_,
+                    strategy_.local_rank_,
+                    dev_id,
+                    ring_id);
+    auto compute_event = std::make_shared<phi::event::Event>();
+    auto comm_event = std::make_shared<phi::event::Event>();
+    compute_event->Init(place_);
+    comm_event->Init(place_);
+    compute_events_.emplace_back(compute_event);
+    comm_events_.emplace_back(comm_event);
+  }
+}
+
+void XCCLParallelContext::InitWithRingID(int ring_id) {
+  int server_fd = -1;
+  std::vector<phi::ccl::CCLRootId> xccl_ids;
+  xccl_ids.resize(1);
+
+  if (strategy_.local_rank_ == 0) {
+    // generate the unique ncclid on the root worker
+    phi::DeviceManager::CCLGetUniqueId(place_.GetDeviceType(), &xccl_ids[0]);
+  } else {
+    // FIXME(wangxi): gloo will use rank0 endpoint, so not create socket server
+    // on rank0.
+    server_fd = platform::SocketServer::GetInstance(strategy_.current_endpoint_)
+                    .socket();
+  }
+  BcastXCCLId(xccl_ids, 0, server_fd);
+
+  int dev_id = place_.device;
+  VLOG(0) << "init xccl context nranks: " << strategy_.nranks_
+          << " local rank: " << strategy_.local_rank_ << " dev id: " << dev_id
+          << " ring id: " << ring_id;
+  // it will assign xccl_comm in phi::CustomContext within ring_id
+  platform::XCCLCommContext::Instance(place_.GetDeviceType())
+      .CreateComm(&xccl_ids[0],
+                  strategy_.nranks_,
+                  strategy_.local_rank_,
+                  dev_id,
+                  ring_id);
+
+  auto compute_event = std::make_shared<phi::event::Event>();
+  auto comm_event = std::make_shared<phi::event::Event>();
+  compute_event->Init(place_);
+  comm_event->Init(place_);
+  compute_events_.emplace_back(compute_event);
+  comm_events_.emplace_back(comm_event);
+}
+
+void XCCLParallelContext::AllReduceByStream(const framework::Variable &src,
+                                            framework::Variable *dst,
+                                            int ring_id,
+                                            bool use_calc_stream) {
+  PADDLE_ENFORCE_EQ(
+      platform::is_custom_place(place_),
+      true,
+      platform::errors::Unimplemented(
+          "Dynamic graph mode does not support multi-CPU training yet."));
+  auto place = place_;
+
+  auto *dev_ctx = static_cast<platform::CustomDeviceContext *>(
+      platform::DeviceContextPool::Instance().Get(place));
+  platform::XCCLComm *comm =
+      platform::XCCLCommContext::Instance(place.GetDeviceType())
+          .Get(ring_id, place);
+  auto stream = use_calc_stream ? dev_ctx->GetStream() : comm->stream();
+
+  if (src.IsType<phi::DenseTensor>()) {
+    if (!dst->IsType<phi::DenseTensor>()) {
+      dst->Clear();
+    }
+    XcclAllReduce(src.Get<phi::DenseTensor>(),
+                  dst->GetMutable<phi::DenseTensor>(),
+                  *stream,
+                  comm->comm());
+  } else {
+    PADDLE_THROW(platform::errors::InvalidArgument(
+        "custom device unsupported variable type %s for imperative allreduce, "
+        "only "
+        "LoDTensor are supported.",
+        platform::demangle(framework::ToTypeName(src.Type()))));
+  }
+}
+
+void XCCLParallelContext::Broadcast(framework::Variable *src, int ring_id) {
+  VLOG(3) << "/// DEBUG /// start inter broadcast with ring_id: " << ring_id;
+  phi::DenseTensor *src_tensor = src->GetMutable<phi::DenseTensor>();
+  const auto &place = src_tensor->place();
+  platform::XCCLComm *comm =
+      platform::XCCLCommContext::Instance(place_.GetDeviceType())
+          .Get(ring_id, place);
+  auto stream = comm->stream();
+
+  void *src_ptr = src_tensor->data();
+  auto xccl_dtype = phi::ccl::ToCCLDataType(src_tensor->dtype());
+
+  phi::DeviceManager::CCLBroadcast(place_.GetDeviceType(),
+                                   src_ptr,
+                                   src_tensor->numel(),
+                                   xccl_dtype,
+                                   0,
+                                   comm->comm(),
+                                   *stream);
+}
+
+paddle::platform::DeviceContext *XCCLParallelContext::GetDeviceContext(
+    int ring_id) {
+  return static_cast<platform::DeviceContext *>(
+      platform::XCCLCommContext::Instance(place_.GetDeviceType())
+          .Get(ring_id, place_)
+          ->dev_context());
+}
+
+void XCCLParallelContext::WaitCompute(int ring_id) {
+  PADDLE_ENFORCE_GE(
+      ring_id,
+      0,
+      platform::errors::OutOfRange("ring id must >= 0, but got %d", ring_id));
+  PADDLE_ENFORCE_LT(ring_id,
+                    compute_events_.size(),
+                    platform::errors::OutOfRange(
+                        "ring id must < compute events size,"
+                        "but got ring id = %d, compute events size = %d",
+                        ring_id,
+                        compute_events_.size()));
+
+  auto compute_stream = static_cast<phi::CustomContext *>(
+                            platform::DeviceContextPool::Instance().Get(place_))
+                            ->GetStream();
+  auto comm_stream = platform::XCCLCommContext::Instance(place_.GetDeviceType())
+                         .Get(ring_id, place_)
+                         ->stream();
+  auto event = compute_events_[ring_id].get();
+
+  // compute_stream-->event-->comm_stream
+  event->Record(compute_stream.get());
+  comm_stream->WaitEvent(event);
+}
+
+void XCCLParallelContext::WaitComm(int ring_id) {
+  PADDLE_ENFORCE_GE(
+      ring_id,
+      0,
+      platform::errors::OutOfRange("ring id must >= 0, but got %d", ring_id));
+  PADDLE_ENFORCE_LT(ring_id,
+                    comm_events_.size(),
+                    platform::errors::OutOfRange(
+                        "ring id must < comm events size,"
+                        "but got ring id = %d, comm events size = %d",
+                        ring_id,
+                        comm_events_.size()));
+
+  auto compute_stream = static_cast<phi::CustomContext *>(
+                            platform::DeviceContextPool::Instance().Get(place_))
+                            ->GetStream();
+  auto comm_stream = platform::XCCLCommContext::Instance(place_.GetDeviceType())
+                         .Get(ring_id, place_)
+                         ->stream();
+  auto event = comm_events_[ring_id].get();
+
+  // comm_stream-->event-->compute_stream
+  event->Record(comm_stream.get());
+  compute_stream->WaitEvent(event);
+}
+
+void XCCLParallelContext::SynchronizeCompute() {
+  auto *compute_dev_ctx = static_cast<phi::CustomContext *>(
+      platform::DeviceContextPool::Instance().Get(place_));
+  compute_dev_ctx->Wait();
+}
+
+}  //  namespace imperative
+}  //  namespace paddle
diff --git a/paddle/fluid/imperative/xccl_context.h b/paddle/fluid/imperative/xccl_context.h
new file mode 100644
index 0000000000000..4426a25372570
--- /dev/null
+++ b/paddle/fluid/imperative/xccl_context.h
@@ -0,0 +1,71 @@
+//   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paddle/fluid/imperative/parallel_context.h"
+
+namespace paddle {
+namespace framework {
+class Variable;
+}  // namespace framework
+}  // namespace paddle
+
+namespace paddle {
+namespace imperative {
+#ifdef PADDLE_WITH_CUSTOM_DEVICE
+class XCCLParallelContext : public ParallelContext {
+ public:
+  explicit XCCLParallelContext(const ParallelStrategy& strategy,
+                               const platform::Place& place)
+      : ParallelContext(strategy, place) {}
+
+  ~XCCLParallelContext() override = default;
+
+  void BcastXCCLId(std::vector<phi::ccl::CCLRootId>& xccl_ids,  // NOLINT
+                   int root,
+                   int server_fd);
+
+  void Init() override;
+
+  void InitWithRingID(int ring_id) override;
+
+  void AllReduceByStream(const framework::Variable& src,
+                         framework::Variable* dst,
+                         int ring_id,
+                         bool use_calc_stream) override;
+
+  void Broadcast(framework::Variable* src, int ring_id) override;
+
+  paddle::platform::DeviceContext* GetDeviceContext(int ring_id) override;
+
+  void WaitCompute(int ring_id) override;
+
+  void WaitComm(int ring_id) override;
+
+  void SynchronizeCompute() override;
+
+ private:
+  // used for comm wait compute, compute_stream-->event-->comm_stream[ring_id]
+  std::vector<std::shared_ptr<phi::event::Event>> compute_events_;
+
+  // used for compute wait comm, comm_stream[ring_id]-->event-->compute_stream
+  std::vector<std::shared_ptr<phi::event::Event>> comm_events_;
+};
+#endif
+}  //  namespace imperative
+}  //  namespace paddle
diff --git a/paddle/fluid/platform/collective_helper.cc b/paddle/fluid/platform/collective_helper.cc
index 948fa300c1f47..b133a57d523ac 100644
--- a/paddle/fluid/platform/collective_helper.cc
+++ b/paddle/fluid/platform/collective_helper.cc
@@ -404,5 +404,235 @@ void BKCLCommContext::ReleaseBKCLComms() {
 
 #endif
 
+#if defined(PADDLE_WITH_CUSTOM_DEVICE)
+class XCCLCommImpl : public XCCLComm {
+ public:
+  void set_ring_id(int ring_id) { ring_id_ = ring_id; }
+  int ring_id() const override { return ring_id_; }
+
+  void set_nranks(int nranks) { nranks_ = nranks; }
+  int nranks() const override { return nranks_; }
+
+  void set_rank(int rank) { rank_ = rank; }
+  int rank() const override { return rank_; }
+
+  int device_id() const override { return dev_ctx_->GetPlace().device; }
+
+  void set_comm(phi::ccl::CCLComm comm) { comm_ = comm; }
+  phi::ccl::CCLComm comm() const override { return comm_; }
+
+  std::shared_ptr<phi::stream::Stream> stream() const override {
+    return dev_ctx_->GetStream();
+  }
+
+  void set_dev_ctx(std::unique_ptr<phi::CustomContext>&& dev_ctx) {
+    dev_ctx_ = std::move(dev_ctx);
+  }
+  phi::CustomContext* dev_context() const override { return dev_ctx_.get(); }
+
+  std::shared_ptr<phi::event::Event> compute_event() const override {
+    return compute_event_;
+  }
+
+  std::shared_ptr<phi::event::Event> comm_event() const override {
+    return comm_event_;
+  }
+
+  void set_compute_event(std::shared_ptr<phi::event::Event>&& compute_event) {
+    compute_event_ = std::move(compute_event);
+  }
+
+  void set_comm_event(std::shared_ptr<phi::event::Event>&& comm_event) {
+    comm_event_ = std::move(comm_event);
+  }
+
+ private:
+  int ring_id_;
+  int nranks_;
+  int rank_;
+  phi::ccl::CCLComm comm_;
+  std::unique_ptr<phi::CustomContext> dev_ctx_;
+
+  // used for comm wait compute, compute_stream-->event-->comm_stream
+  std::shared_ptr<phi::event::Event> compute_event_;
+
+  // used for compute wait comm, comm_stream-->event-->compute_stream
+  std::shared_ptr<phi::event::Event> comm_event_;
+};
+
+static std::unordered_map<std::string, std::unique_ptr<XCCLCommContext>>
+    g_xccl_comm_ctx_map;
+
+void XCCLCommContext::Release() {
+  for (auto& it : g_xccl_comm_ctx_map) {
+    it.second->ReleaseXCCLComms();
+  }
+  g_xccl_comm_ctx_map.clear();
+}
+
+XCCLCommContext& XCCLCommContext::Instance(const std::string& device_type) {
+  if (g_xccl_comm_ctx_map.find(device_type) == g_xccl_comm_ctx_map.end()) {
+    g_xccl_comm_ctx_map.insert(
+        {device_type,
+         std::unique_ptr<XCCLCommContext>(new XCCLCommContext(device_type))});
+  }
+  return *g_xccl_comm_ctx_map[device_type];
+}
+
+XCCLComm* XCCLCommContext::CreateComm(phi::ccl::CCLRootId* xccl_id,
+                                      int nranks,
+                                      int rank,
+                                      int dev_id,
+                                      int ring_id) {
+  PADDLE_ENFORCE_NOT_NULL(xccl_id,
+                          platform::errors::InvalidArgument(
+                              "The xccl unique id should not be null."));
+  PADDLE_ENFORCE_GT(
+      nranks,
+      1,
+      platform::errors::InvalidArgument(
+          "Expected nranks > 1. But received nranks is %d.", nranks));
+  PADDLE_ENFORCE_GE(rank,
+                    0,
+                    platform::errors::InvalidArgument(
+                        "Expected rank >= 0. But received rank is %d.", rank));
+  PADDLE_ENFORCE_LT(
+      rank,
+      nranks,
+      platform::errors::InvalidArgument(
+          "Expected rank < nranks. But received rank is %d, nranks is %d.",
+          rank,
+          nranks));
+  PADDLE_ENFORCE_GE(
+      dev_id,
+      0,
+      platform::errors::InvalidArgument(
+          "Expected dev_id >= 0. But received dev_id is %d.", dev_id));
+
+  phi::ccl::CCLComm comm = nullptr;
+  phi::DeviceManager::SetDevice(device_type_, dev_id);
+  phi::DeviceManager::CCLCommInitRank(
+      device_type_, nranks, xccl_id, rank, &comm);
+
+  auto* comm_wrapper = AssignXCCLComm(comm, nranks, rank, dev_id, ring_id);
+
+  VLOG(1) << "xccl communicator of rank " << rank << " in ring " << ring_id
+          << " has been created on device " << dev_id;
+
+  return comm_wrapper;
+}
+
+void XCCLCommContext::CreateXCCLCommMultiTrainer(
+    const std::vector<int>& dev_ids,
+    phi::ccl::CCLRootId* xccl_id,
+    int ntrainers,
+    int train_id,
+    int ring_id) {
+  PADDLE_ENFORCE_GT(
+      dev_ids.size(),
+      0,
+      paddle::platform::errors::InvalidArgument(
+          "dev ids = [%d], it should greater than 0.", dev_ids.size()));
+  const int kDevices = dev_ids.size();
+  VLOG(1) << "Begin CreateXCCLCommMultiTrainer. device number: " << kDevices
+          << ", ntrainers: " << ntrainers << ", train_id: " << train_id
+          << ", rind_id: " << ring_id;
+  phi::ccl::CCLComm comms[kDevices];
+  {
+    for (int i = 0; i < kDevices; i++) {
+      phi::DeviceManager::SetDevice(device_type_, i);
+      phi::DeviceManager::CCLCommInitRank(device_type_,
+                                          kDevices * ntrainers,
+                                          xccl_id,
+                                          train_id * kDevices + i,
+                                          comms + i);
+      VLOG(1) << "CCLCommInitRank: " << i;
+    }
+  }
+  PADDLE_ENFORCE_EQ(comm_map_.count(ring_id),
+                    0,
+                    platform::errors::InvalidArgument(
+                        "comm_map_ of ring_id: %s should be 0. %s is provided",
+                        ring_id,
+                        comm_map_.count(ring_id)));
+  for (int i = 0; i < kDevices; ++i) {
+    AssignXCCLComm(comms[i],
+                   kDevices * ntrainers,
+                   train_id * kDevices + i,
+                   dev_ids[i],
+                   ring_id);
+    VLOG(1) << "xccl communicator of train_id " << train_id * kDevices + i
+            << " in ring " << ring_id << " has been created on device "
+            << dev_ids[i];
+  }
+}
+
+XCCLComm* XCCLCommContext::AssignXCCLComm(
+    phi::ccl::CCLComm comm, int nranks, int rank, int dev_id, int ring_id) {
+  auto place = CustomPlace(device_type_, dev_id);
+  std::unique_ptr<phi::CustomContext> dev_ctx(new phi::CustomContext(place));
+  dev_ctx->SetAllocator(paddle::memory::allocation::AllocatorFacade::Instance()
+                            .GetAllocator(place)
+                            .get());
+  dev_ctx->SetHostAllocator(
+      paddle::memory::allocation::AllocatorFacade::Instance()
+          .GetAllocator(paddle::platform::CPUPlace())
+          .get());
+  dev_ctx->SetZeroAllocator(
+      paddle::memory::allocation::AllocatorFacade::Instance()
+          .GetZeroAllocator(place)
+          .get());
+  dev_ctx->SetHostZeroAllocator(
+      paddle::memory::allocation::AllocatorFacade::Instance()
+          .GetZeroAllocator(paddle::platform::CPUPlace())
+          .get());
+  dev_ctx->SetPinnedAllocator(
+      paddle::memory::allocation::AllocatorFacade::Instance()
+          .GetAllocator(paddle::platform::CPUPlace())
+          .get());
+  // dev_ctx->PartialInitWithAllocator();
+
+  auto compute_event = std::make_shared<phi::event::Event>();
+  auto comm_event = std::make_shared<phi::event::Event>();
+  compute_event->Init(place);
+  comm_event->Init(place);
+
+  auto* c = new XCCLCommImpl;
+  c->set_ring_id(ring_id);
+  c->set_nranks(nranks);
+  c->set_rank(rank);
+  c->set_comm(comm);
+  c->set_dev_ctx(std::move(dev_ctx));
+  c->set_compute_event(std::move(compute_event));
+  c->set_comm_event(std::move(comm_event));
+
+  comm_map_mutex_.lock();
+  if (comm_map_.count(ring_id) == 0) {
+    comm_map_.emplace(ring_id, std::map<int, std::unique_ptr<XCCLComm>>());
+  }
+  auto& dev2comm = comm_map_[ring_id];
+
+  dev2comm.emplace(dev_id, std::unique_ptr<XCCLComm>(c));
+  comm_map_mutex_.unlock();
+
+  if (ring_id == 0) {
+    auto* dev_ctx = static_cast<phi::CustomContext*>(
+        platform::DeviceContextPool::Instance().Get(place));
+    dev_ctx->set_xccl_comm(comm);
+  }
+  VLOG(4) << "add xccl comm: " << comm_map_[ring_id][dev_id].get()
+          << ", ring_id:" << ring_id << ", dev_id:" << dev_id;
+  return comm_map_[ring_id][dev_id].get();
+}
+
+void XCCLCommContext::ReleaseXCCLComms() {
+  for (auto& p : comm_map_) {
+    for (auto& q : p.second) {
+      q.second.reset();
+    }
+  }
+}
+
+#endif
 }  // namespace platform
 }  // namespace paddle
diff --git a/paddle/fluid/platform/collective_helper.h b/paddle/fluid/platform/collective_helper.h
index c0a7f2f37b07e..6636856a0eb6c 100644
--- a/paddle/fluid/platform/collective_helper.h
+++ b/paddle/fluid/platform/collective_helper.h
@@ -22,6 +22,7 @@
 #include "paddle/fluid/framework/data_type.h"
 #include "paddle/fluid/platform/device_context.h"
 #include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/backends/device_manager.h"
 #include "paddle/utils/variant.h"
 
 namespace paddle {
@@ -243,5 +244,112 @@ class BKCLCommContext {
   DISABLE_COPY_AND_ASSIGN(BKCLCommContext);
 };
 #endif
+
+#if defined(PADDLE_WITH_CUSTOM_DEVICE)
+class XCCLComm {
+ public:
+  virtual int ring_id() const = 0;
+  virtual int nranks() const = 0;
+  virtual int rank() const = 0;
+  virtual int device_id() const = 0;
+  virtual phi::ccl::CCLComm comm() const = 0;
+  virtual std::shared_ptr<phi::stream::Stream> stream() const = 0;
+  virtual std::shared_ptr<phi::event::Event> compute_event() const = 0;
+  virtual std::shared_ptr<phi::event::Event> comm_event() const = 0;
+  virtual phi::CustomContext* dev_context() const = 0;
+  virtual ~XCCLComm() = default;
+};
+
+// A singleton XCCL communicator context reserves communication ring ids
+class XCCLCommContext {
+ public:
+  static XCCLCommContext& Instance(const std::string& device_type);
+  static void Release();
+
+  XCCLComm* CreateComm(phi::ccl::CCLRootId* nccl_id,
+                       int nranks,
+                       int rank,
+                       int dev_id,
+                       int ring_id = 0);
+
+  void CreateAllXCCLComms(const std::vector<int>& dev_ids, int ring_id = 0);
+
+  void CreateXCCLCommMultiTrainer(const std::vector<int>& dev_ids,
+                                  phi::ccl::CCLRootId* xccl_id,
+                                  int nranks,
+                                  int rank,
+                                  int ring_id);
+
+  // a latter comm with the same dev_id and the same ring_id
+  // will override the former
+  XCCLComm* AssignXCCLComm(phi::ccl::CCLComm comm,
+                           int nranks,
+                           int rank,
+                           int dev_id,
+                           int ring_id = 0);
+
+  // retrieve a communicator by the ring id in multiprocessing mode
+  XCCLComm* Get(int ring_id) const {
+    PADDLE_ENFORCE_GT(
+        comm_map_.count(ring_id),
+        0,
+        platform::errors::InvalidArgument(
+            "Communicator in ring id %d has not been initialized.", ring_id));
+    PADDLE_ENFORCE_EQ(comm_map_.at(ring_id).size(),
+                      1,
+                      platform::errors::InvalidArgument(
+                          "One device id should be specified to retrieve from "
+                          "multiple communicators."));
+    return comm_map_.at(ring_id).begin()->second.get();
+  }
+
+  int GetRingId(phi::ccl::CCLComm comm) const {
+    for (const auto& pair : comm_map_) {
+      for (const auto& p : pair.second) {
+        if (p.second.get()->comm() == comm) {
+          return pair.first;
+        }
+      }
+    }
+    return -1;
+  }
+
+  // retrieve a communicator by the ring id and the device id
+  XCCLComm* Get(int ring_id, int dev_id) const {
+    PADDLE_ENFORCE_GT(
+        comm_map_.count(ring_id),
+        0,
+        platform::errors::InvalidArgument(
+            "Communicator of ring id %d has not been initialized.", ring_id));
+    PADDLE_ENFORCE_GT(
+        comm_map_.at(ring_id).count(dev_id),
+        0,
+        platform::errors::InvalidArgument(
+            "Communicator at device id %d has not been initialized in ring %d.",
+            dev_id,
+            ring_id));
+    return comm_map_.at(ring_id).at(dev_id).get();
+  }
+
+  // retrieve a communicator by the ring id and place
+  XCCLComm* Get(int ring_id, Place place) const {
+    return Get(ring_id, place.device);
+  }
+
+ private:
+  std::string device_type_;
+  std::once_flag once_flag_;
+  std::mutex comm_map_mutex_;
+  // ring id to dev-XCCLComm
+  std::map<int, std::map<int, std::unique_ptr<XCCLComm>>> comm_map_;
+
+  void ReleaseXCCLComms();
+
+  XCCLCommContext() = default;
+  explicit XCCLCommContext(const std::string& device_type)
+      : device_type_(device_type) {}
+  DISABLE_COPY_AND_ASSIGN(XCCLCommContext);
+};
+#endif
 }  // namespace platform
 }  // namespace paddle
diff --git a/paddle/fluid/platform/gen_comm_id_helper.cc b/paddle/fluid/platform/gen_comm_id_helper.cc
index ca9f9d7c4f806..0237d28e52c27 100644
--- a/paddle/fluid/platform/gen_comm_id_helper.cc
+++ b/paddle/fluid/platform/gen_comm_id_helper.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) || \
-    defined(PADDLE_WITH_XPU_BKCL)
+    defined(PADDLE_WITH_XPU_BKCL) || defined(PADDLE_WITH_CUSTOM_DEVICE)
 #include "paddle/fluid/platform/gen_comm_id_helper.h"
 
 #include <arpa/inet.h>
@@ -33,6 +33,9 @@ limitations under the License. */
 #if defined(PADDLE_WITH_XPU_BKCL)
 #include "xpu/bkcl.h"
 #endif
+#if defined(PADDLE_WITH_CUSTOM_DEVICE)
+#include "paddle/phi/backends/c_comm_lib.h"
+#endif
 
 PHI_DECLARE_int32(get_host_by_name_time);
 
@@ -348,6 +351,58 @@ static void SendCommID(int conn, CommUniqueId* nccl_id) {
                  "send comm unique id");
 }
 
+#ifdef PADDLE_WITH_CUSTOM_DEVICE
+template <>
+void RecvCommID<phi::ccl::CCLRootId>(int conn, phi::ccl::CCLRootId* nccl_id) {
+  char buffer[MAX_COMMUNIQUEID_LEN] = {0};
+  CHECK_SYS_CALL(SocketRecv(conn, buffer, sizeof(size_t)),
+                 "recv comm unique id size");
+  size_t unique_id_size = *reinterpret_cast<size_t*>(buffer);
+  VLOG(6) << "RecvCommID size: " << unique_id_size;
+  nccl_id->resize(unique_id_size);
+
+  size_t n_repeat = unique_id_size / MAX_COMMUNIQUEID_LEN;
+  size_t n_remain = unique_id_size % MAX_COMMUNIQUEID_LEN;
+  for (size_t i = 0; i < n_repeat; ++i) {
+    CHECK_SYS_CALL(SocketRecv(conn, buffer, MAX_COMMUNIQUEID_LEN),
+                   "recv comm unique id");
+    memcpy(nccl_id->data() + i * MAX_COMMUNIQUEID_LEN,
+           buffer,
+           MAX_COMMUNIQUEID_LEN);
+  }
+  if (n_remain) {
+    CHECK_SYS_CALL(SocketRecv(conn, buffer, n_remain), "recv comm unique id");
+    memcpy(nccl_id->data() + n_repeat * MAX_COMMUNIQUEID_LEN, buffer, n_remain);
+  }
+  VLOG(6) << "RecvCommID done";
+}
+
+template <>
+void SendCommID<phi::ccl::CCLRootId>(int conn, phi::ccl::CCLRootId* nccl_id) {
+  char buffer[MAX_COMMUNIQUEID_LEN] = {0};
+  size_t unique_id_size = nccl_id->size();
+  VLOG(6) << "SendCommID size: " << unique_id_size;
+  memcpy(buffer, &unique_id_size, sizeof(size_t));
+  CHECK_SYS_CALL(SocketSend(conn, buffer, sizeof(size_t)),
+                 "send comm unique id size");
+
+  size_t n_repeat = unique_id_size / MAX_COMMUNIQUEID_LEN;
+  size_t n_remain = unique_id_size % MAX_COMMUNIQUEID_LEN;
+  for (size_t i = 0; i < n_repeat; ++i) {
+    memcpy(buffer,
+           nccl_id->data() + i * MAX_COMMUNIQUEID_LEN,
+           MAX_COMMUNIQUEID_LEN);
+    CHECK_SYS_CALL(SocketSend(conn, buffer, MAX_COMMUNIQUEID_LEN),
+                   "send comm unique id");
+  }
+  if (n_remain) {
+    memcpy(buffer, nccl_id->data() + n_repeat * MAX_COMMUNIQUEID_LEN, n_remain);
+    CHECK_SYS_CALL(SocketSend(conn, buffer, n_remain), "send comm unique id");
+  }
+  VLOG(6) << "SendCommID done";
+}
+#endif
+
 template <typename CommUniqueId>
 void SendBroadCastCommID(std::vector<std::string> servers,
                          std::vector<CommUniqueId>* nccl_ids,
@@ -444,6 +499,9 @@ INSTANT_TEMPLATE(ncclUniqueId)
 #ifdef PADDLE_WITH_XPU_BKCL
 INSTANT_TEMPLATE(BKCLUniqueId)
 #endif
+#ifdef PADDLE_WITH_CUSTOM_DEVICE
+INSTANT_TEMPLATE(phi::ccl::CCLRootId)
+#endif
 }  // namespace platform
 }  // namespace paddle
 
diff --git a/paddle/fluid/platform/gen_comm_id_helper.h b/paddle/fluid/platform/gen_comm_id_helper.h
index 0aef760fd4e6f..d97b41311995e 100644
--- a/paddle/fluid/platform/gen_comm_id_helper.h
+++ b/paddle/fluid/platform/gen_comm_id_helper.h
@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 
 #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) || \
-    defined(PADDLE_WITH_XPU_BKCL)
+    defined(PADDLE_WITH_XPU_BKCL) || defined(PADDLE_WITH_CUSTOM_DEVICE)
 #include <functional>
 #include <memory>
 #include <mutex>
diff --git a/paddle/phi/backends/c_comm_lib.h b/paddle/phi/backends/c_comm_lib.h
index c93a1c1d18d71..e67530add58da 100644
--- a/paddle/phi/backends/c_comm_lib.h
+++ b/paddle/phi/backends/c_comm_lib.h
@@ -23,8 +23,8 @@
 
 namespace phi {
 namespace ccl {
-using CCLComm = void*;
-using CCLRootId = std::vector<uint8_t>;
+typedef void* CCLComm;
+typedef std::vector<uint8_t> CCLRootId;
 
 enum CCLReduceOp { SUM = 0, AVG, MAX, MIN, PRODUCT };
 enum CCLDataType {
diff --git a/paddle/phi/backends/custom/custom_context.cc b/paddle/phi/backends/custom/custom_context.cc
index c5b7676df48dd..ddba0baea7e06 100644
--- a/paddle/phi/backends/custom/custom_context.cc
+++ b/paddle/phi/backends/custom/custom_context.cc
@@ -44,9 +44,15 @@ struct CustomContext::Impl {
 
   void Wait() const { stream_->Wait(); }
 
+  phi::ccl::CCLComm xccl_comm() const { return comm_; }
+
+  void set_xccl_comm(phi::ccl::CCLComm comm) { comm_ = comm; }
+
   Place place_;
 
   std::shared_ptr<phi::stream::Stream> stream_;
+
+  phi::ccl::CCLComm comm_;
 };
 
 void CustomContext::Init() { impl_->Init(); }
@@ -72,4 +78,11 @@ CustomContext::CustomContext(const CustomPlace& place)
 
 CustomContext::~CustomContext() { impl_->Init(); }
 
+phi::ccl::CCLComm CustomContext::xccl_comm() const {
+  return impl_->xccl_comm();
+}
+
+void CustomContext::set_xccl_comm(phi::ccl::CCLComm comm) {
+  impl_->set_xccl_comm(comm);
+}
 }  // namespace phi
diff --git a/paddle/phi/backends/custom/custom_context.h b/paddle/phi/backends/custom/custom_context.h
index d12ba20d4b6a6..68abfeae36653 100644
--- a/paddle/phi/backends/custom/custom_context.h
+++ b/paddle/phi/backends/custom/custom_context.h
@@ -16,6 +16,7 @@ limitations under the License. */
 
 #include <memory>
 
+#include "paddle/phi/backends/c_comm_lib.h"
 #include "paddle/phi/backends/stream.h"
 #include "paddle/phi/common/place.h"
 #include "paddle/phi/core/device_context.h"
@@ -63,6 +64,12 @@ class CustomContext : public DeviceContext,
   // all resources and delete them when destructing.
   void Init();
 
+  /*! \brief  Return xccl communicators. */
+  phi::ccl::CCLComm xccl_comm() const;
+
+  /*! \brief  Set nccl communicators. */
+  void set_xccl_comm(phi::ccl::CCLComm comm);
+
  private:
   CustomContext();
 

From 3846111defd9815cf362c52f1bd8d1879c15f5a9 Mon Sep 17 00:00:00 2001
From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com>
Date: Fri, 12 May 2023 14:54:52 +0800
Subject: [PATCH 306/405] =?UTF-8?q?=E3=80=90prim=E3=80=91add=20forward=20o?=
 =?UTF-8?q?utput=20for=20Silu=20grad=20signature=20(#53632)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add rules

* modify silu_grad input

* modify kernel signature

* modify kernel signature

* code style

* review
---
 .../api/composite_backward/composite_backward_api.h  |  7 +++++--
 paddle/phi/api/yaml/backward.yaml                    |  4 ++--
 paddle/phi/kernels/activation_grad_kernel.h          |  7 ++++++-
 paddle/phi/kernels/cpu/activation_grad_kernel.cc     | 12 +++++++++++-
 paddle/phi/kernels/gpu/activation_grad_kernel.cu     | 11 ++++++++++-
 paddle/phi/kernels/xpu/activation_grad_kernel.cc     | 11 ++++++++++-
 6 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
index a9d91b6e85489..cd88139af23a7 100644
--- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
+++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
@@ -60,9 +60,12 @@ void leaky_relu_grad(const Tensor& out,
 }
 
 template <typename T>
-void silu_grad(const Tensor& x, const Tensor& out_grad, Tensor* x_grad) {
+void silu_grad(const Tensor& x,
+               const Tensor& out,
+               const Tensor& out_grad,
+               Tensor* x_grad) {
   if (x_grad) {
-    auto sigmoid = 1.0 / (1.0 + exp<T>(-x));
+    auto sigmoid = out / x;
     auto res = out_grad * sigmoid * (1.0 + x * (1.0 - sigmoid));
     set_output<T>(res, x_grad);
   }
diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index 6ae8c19006410..27bdf904eee6e 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -1599,14 +1599,14 @@
 
 - backward_op : silu_grad
   forward : silu (Tensor x) -> Tensor(out)
-  args : (Tensor x, Tensor out_grad)
+  args : (Tensor x, Tensor out, Tensor out_grad)
   output : Tensor(x_grad)
   infer_meta :
     func : UnchangedInferMeta
     param : [x]
   kernel :
     func : silu_grad
-  composite : silu_grad(x, out_grad, x_grad)
+  composite : silu_grad(x, out, out_grad, x_grad)
   inplace : (out_grad -> x_grad)
 
 - backward_op : sin_double_grad
diff --git a/paddle/phi/kernels/activation_grad_kernel.h b/paddle/phi/kernels/activation_grad_kernel.h
index ca75a6e0b24a4..302e0f6ba9080 100644
--- a/paddle/phi/kernels/activation_grad_kernel.h
+++ b/paddle/phi/kernels/activation_grad_kernel.h
@@ -75,6 +75,12 @@ namespace phi {
                         DenseTensor* dx);
 
 template <typename T, typename Context>
+void SiluGradKernel(const Context& dev_ctx,
+                    const DenseTensor& x,
+                    const DenseTensor& out,
+                    const DenseTensor& dout,
+                    DenseTensor* dx);
+template <typename T, typename Context>
 void ReluDoubleGradKernel(const Context& dev_ctx,
                           const DenseTensor& out,
                           const DenseTensor& ddx,
@@ -277,7 +283,6 @@ DECLARE_ACTIVATION_GRAD_KERNEL_DEPX(Asinh);
 DECLARE_ACTIVATION_GRAD_KERNEL_DEPX(Acosh);
 DECLARE_ACTIVATION_GRAD_KERNEL_DEPX(Atanh);
 DECLARE_ACTIVATION_GRAD_KERNEL_DEPX(TanhShrink);
-DECLARE_ACTIVATION_GRAD_KERNEL_DEPX(Silu);
 DECLARE_ACTIVATION_GRAD_KERNEL_DEPX(Square);
 DECLARE_ACTIVATION_GRAD_KERNEL_DEPX(Softsign);
 DECLARE_ACTIVATION_GRAD_KERNEL_DEPX(LogSigmoid);
diff --git a/paddle/phi/kernels/cpu/activation_grad_kernel.cc b/paddle/phi/kernels/cpu/activation_grad_kernel.cc
index 9273f8393b5b3..7b9074ffa92f3 100644
--- a/paddle/phi/kernels/cpu/activation_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/activation_grad_kernel.cc
@@ -128,7 +128,6 @@ DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Asinh, AsinhGradFunctor);
 DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Acosh, AcoshGradFunctor);
 DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Atanh, AtanhGradFunctor);
 DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(TanhShrink, TanhShrinkGradFunctor);
-DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Silu, SiluGradFunctor);
 DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Square, SquareGradFunctor);
 
 DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Exp, ExpGradFunctor);
@@ -190,6 +189,17 @@ DEFINE_CPU_ACT_GRAD_KERNEL_WITH_TWO_ATTRS_DEPOUT(HardSigmoid,
                                                  slope,
                                                  offset);
 
+template <typename T, typename Context>
+void SiluGradKernel(const Context& dev_ctx,
+                    const DenseTensor& x,
+                    const DenseTensor& out,
+                    const DenseTensor& dout,
+                    DenseTensor* dx) {
+  funcs::SiluGradFunctor<T> functor;
+  ActivationGradImpl<T, Context, funcs::SiluGradFunctor<T>>(
+      dev_ctx, &x, &out, &dout, dx, functor);
+}
+
 template <typename T, typename Context>
 void EluGradKernel(const Context& dev_ctx,
                    const DenseTensor& x,
diff --git a/paddle/phi/kernels/gpu/activation_grad_kernel.cu b/paddle/phi/kernels/gpu/activation_grad_kernel.cu
index c0fb7342a80b6..aa703ede3bad6 100644
--- a/paddle/phi/kernels/gpu/activation_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/activation_grad_kernel.cu
@@ -190,7 +190,6 @@ DEFINE_GPU_ACTIVATION_GRAD_KERNEL_DEPX(Asinh, CudaAsinhGradFunctor);
 DEFINE_GPU_ACTIVATION_GRAD_KERNEL_DEPX(Acosh, CudaAcoshGradFunctor);
 DEFINE_GPU_ACTIVATION_GRAD_KERNEL_DEPX(Atanh, CudaAtanhGradFunctor);
 DEFINE_GPU_ACTIVATION_GRAD_KERNEL_DEPX(TanhShrink, CudaTanhShrinkGradFunctor);
-DEFINE_GPU_ACTIVATION_GRAD_KERNEL_DEPX(Silu, CudaSiluGradFunctor);
 DEFINE_GPU_ACTIVATION_GRAD_KERNEL_DEPX(Square, CudaSquareGradFunctor);
 
 DEFINE_GPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Exp, CudaExpGradFunctor);
@@ -249,6 +248,16 @@ DEFINE_GPU_ACT_GRAD_KERNEL_WITH_TWO_ATTRS_DEPOUT(HardSigmoid,
                                                  slope,
                                                  offset);
 
+template <typename T, typename Context>
+void SiluGradKernel(const Context& dev_ctx,
+                    const DenseTensor& x,
+                    const DenseTensor& out,
+                    const DenseTensor& dout,
+                    DenseTensor* dx) {
+  funcs::CudaSiluGradFunctor<T> functor;
+  ActivationGradGPUImpl<T, Context, funcs::CudaSiluGradFunctor<T>>(
+      dev_ctx, &x, &out, &dout, dx, functor);
+}
 template <typename T, typename Context>
 void EluGradKernel(const Context& dev_ctx,
                    const DenseTensor& x,
diff --git a/paddle/phi/kernels/xpu/activation_grad_kernel.cc b/paddle/phi/kernels/xpu/activation_grad_kernel.cc
index 02243215f9588..8aae57b1e498b 100644
--- a/paddle/phi/kernels/xpu/activation_grad_kernel.cc
+++ b/paddle/phi/kernels/xpu/activation_grad_kernel.cc
@@ -567,7 +567,6 @@ DEFINE_XPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Tanh, XPUTanhGradFunctor);
 DEFINE_XPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Relu, XPUReluGradFunctor);
 DEFINE_XPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Relu6, XPURelu6GradFunctor);
 
-DEFINE_XPU_ACTIVATION_GRAD_KERNEL_DEPX(Silu, XPUSiluGradFunctor);
 DEFINE_XPU_ACTIVATION_GRAD_KERNEL_DEPX(Log, XPULogGradFunctor);
 DEFINE_XPU_ACTIVATION_GRAD_KERNEL_DEPX(Square, XPUSquareGradFunctor);
 DEFINE_XPU_ACTIVATION_GRAD_KERNEL_DEPX(Swish, XPUSwishGradFunctor);
@@ -605,6 +604,16 @@ void HardSwishGradKernel(const Context& dev_ctx,
       dev_ctx, &x, nullptr, &dout, dx, functor);
 }
 
+template <typename T, typename Context>
+void SiluGradKernel(const Context& dev_ctx,
+                    const DenseTensor& x,
+                    const DenseTensor& out,
+                    const DenseTensor& dout,
+                    DenseTensor* dx) {
+  XPUSiluGradFunctor<T> functor;
+  ActivationGradXPUImpl<T, Context, XPUSiluGradFunctor<T>>(
+      dev_ctx, &x, &out, &dout, dx, functor);
+}
 }  // namespace phi
 
 PD_REGISTER_KERNEL(relu_grad,

From d01c89c3de7b097fda7c8186efe321d0b8b4f68a Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Fri, 12 May 2023 15:56:52 +0800
Subject: [PATCH 307/405] Remove is_causal.

---
 paddle/fluid/operators/fused/fused_gate_attention.h | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index cd3d5f96cc6cb..3d8ab898e35e0 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -940,7 +940,6 @@ class FlashAttnWithGating {
         fa_dropout_prob_,
         fa_softmax_scale_,
         fa_zero_tensors_,
-        fa_is_causal_,
         is_bf16,
         fa_num_splits_,
         softmax_lse->data(),
@@ -976,7 +975,6 @@ class FlashAttnWithGating {
         fa_dropout_prob_,
         fa_softmax_scale_,
         fa_zero_tensors_,
-        fa_is_causal_,
         is_bf16,
         fa_num_splits_,
         softmax_lse->data(),
@@ -1099,7 +1097,6 @@ class FlashAttnWithGating {
         fa_dropout_prob_,
         fa_softmax_scale_,
         fa_zero_tensors_,
-        fa_is_causal_,
         is_bf16,
         fa_num_splits_,
         softmax_lse->data(),
@@ -1140,7 +1137,6 @@ class FlashAttnWithGating {
         fa_dropout_prob_,
         fa_softmax_scale_,
         fa_zero_tensors_,
-        fa_is_causal_,
         is_bf16,
         fa_num_splits_,
         softmax_lse->data(),
@@ -1264,7 +1260,6 @@ class FlashAttnWithGating {
     fa_offset_ = seed_offset_pair.second;
 
     fa_zero_tensors_ = false;
-    fa_is_causal_ = false;
 
     if (VLOG_IS_ON(6)) {
       VLOG(6) << "fa_batch_size    : " << fa_batch_size_;
@@ -1315,7 +1310,6 @@ class FlashAttnWithGating {
   uint64_t fa_seed_{0};
   uint64_t fa_offset_{0};
   bool fa_zero_tensors_{false};
-  bool fa_is_causal_{false};
 };
 
 }  // namespace operators

From 1019b264cb8d6ae9d640cbdf33e410c1de437978 Mon Sep 17 00:00:00 2001
From: houj04 <35131887+houj04@users.noreply.github.com>
Date: Fri, 12 May 2023 16:09:10 +0800
Subject: [PATCH 308/405] [XPU] remove clip of c_softmax_with_cross_entropy_op
 (#53734)

---
 .../c_softmax_with_cross_entropy_op_xpu.cc         | 14 --------------
 ...collective_softmax_with_cross_entropy_op_xpu.py |  2 +-
 ...st_collective_softmax_with_cross_entropy_xpu.py |  6 +++---
 3 files changed, 4 insertions(+), 18 deletions(-)

diff --git a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op_xpu.cc b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op_xpu.cc
index 695c28d77554a..fa17b99e79f2a 100644
--- a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op_xpu.cc
+++ b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op_xpu.cc
@@ -131,13 +131,6 @@ struct CSoftmaxWithCrossEntropyProcessGroupFunctor<phi::XPUContext, T> {
       };
       phi::XPUElementwise<T, XPUType>(
           dev_ctx, logits_2d, logits_max, axis, &softmax_2d, f);
-      ret = xpu::clip<XPUType>(dev_ctx.x_context(),
-                               reinterpret_cast<XPUType*>(softmax_2d.data<T>()),
-                               reinterpret_cast<XPUType*>(softmax_2d.data<T>()),
-                               N * D,
-                               -64.,
-                               0.);
-      PADDLE_ENFORCE_XDNN_SUCCESS(ret, "clip");
     }
 
     // step 3, obtain predict target
@@ -335,13 +328,6 @@ struct CSoftmaxWithCrossEntropyFunctor<phi::XPUContext, T> {
       };
       phi::XPUElementwise<T, XPUType>(
           dev_ctx, logits_2d, logits_max, axis, &softmax_2d, f);
-      ret = xpu::clip<XPUType>(dev_ctx.x_context(),
-                               reinterpret_cast<XPUType*>(softmax_2d.data<T>()),
-                               reinterpret_cast<XPUType*>(softmax_2d.data<T>()),
-                               N * D,
-                               -64.,
-                               0.);
-      PADDLE_ENFORCE_XDNN_SUCCESS(ret, "clip");
     }
 
     // step 3, obtain predict target
diff --git a/test/xpu/collective_softmax_with_cross_entropy_op_xpu.py b/test/xpu/collective_softmax_with_cross_entropy_op_xpu.py
index e450811479412..abb94cc6ad15b 100644
--- a/test/xpu/collective_softmax_with_cross_entropy_op_xpu.py
+++ b/test/xpu/collective_softmax_with_cross_entropy_op_xpu.py
@@ -133,7 +133,7 @@ def run_trainer(self, args):
         # each xpu uses own half of logits
         np.random.seed(os.getpid())
         logits = np.random.uniform(
-            low=-10.0, high=10.0, size=(self.batch_size, self.local_elements)
+            low=-40.0, high=40.0, size=(self.batch_size, self.local_elements)
         ).astype(np_data_type)
         out = exe.run(
             train_prog,
diff --git a/test/xpu/test_collective_softmax_with_cross_entropy_xpu.py b/test/xpu/test_collective_softmax_with_cross_entropy_xpu.py
index 0bc75c7a4930b..9346f004f83ad 100644
--- a/test/xpu/test_collective_softmax_with_cross_entropy_xpu.py
+++ b/test/xpu/test_collective_softmax_with_cross_entropy_xpu.py
@@ -33,7 +33,7 @@ def stable_softmax(x):
     """Compute the softmax of vector x in a numerically stable way."""
     # clip to shiftx, otherwise, when calc loss with
     # log(exp(shiftx)), may get log(0)=INF
-    shiftx = (x - np.max(x)).clip(-64.0)
+    shiftx = x - np.max(x)
     exps = np.exp(shiftx)
     return exps / np.sum(exps)
 
@@ -131,13 +131,13 @@ def check_with_place(
             # get input data for rank 0
             np.random.seed(pid0)
             input0 = np.random.uniform(
-                low=-10.0, high=10.0, size=(self.batch_size, local_elements)
+                low=-40.0, high=40.0, size=(self.batch_size, local_elements)
             ).astype(np_data_type)
 
             # get input data for rank 1
             np.random.seed(pid1)
             input1 = np.random.uniform(
-                low=-10.0, high=10.0, size=(self.batch_size, local_elements)
+                low=-40.0, high=40.0, size=(self.batch_size, local_elements)
             ).astype(np_data_type)
 
             # get combined input data

From 772b490656f60064f796ad1fa1bd423f8d03abac Mon Sep 17 00:00:00 2001
From: Zhang Ting <zhangting_2017@163.com>
Date: Fri, 12 May 2023 16:11:35 +0800
Subject: [PATCH 309/405] fix dtype missmatch error (#53712)

---
 python/paddle/static/amp/fp16_utils.py | 27 +++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/python/paddle/static/amp/fp16_utils.py b/python/paddle/static/amp/fp16_utils.py
index c2f8a12d33b46..333d652f9f59d 100644
--- a/python/paddle/static/amp/fp16_utils.py
+++ b/python/paddle/static/amp/fp16_utils.py
@@ -629,12 +629,15 @@ def cast_model_to_fp16(
 
     def need_process(op):
         need_process = True
-        if op.type in ["cast", "create_py_reader", "read"]:
+        if op.type in ["create_py_reader", "read"]:
             need_process = False
         else:
             for attr_name in ['out_dtype', 'dtype']:
-                if op.has_attr(attr_name) and is_float_dtype(
-                    op.attr(attr_name)
+                # output type of some operators such as fill_constant will be determined by the attribute value.
+                #
+                if not op.has_attr('in_dtype') and (
+                    op.has_attr(attr_name)
+                    and is_float_dtype(op.attr(attr_name))
                 ):
                     need_process = False
 
@@ -667,6 +670,24 @@ def need_process(op):
                     "---- Add into keep_fp16_ops because the op in white_list ----"
                 )
             else:
+                # if cast in orgin program, we only modifiy attr and output's dtype to avoid dtype mismatch errors.
+                if op.type == 'cast':
+                    in_var = block._find_var_recursive(op.input('X')[0])
+                    out_var = block._find_var_recursive(op.output('Out')[0])
+                    op._set_attr('in_dtype', in_var.dtype)
+                    out_var.desc.set_dtype(paddle.dtype(op.attr('out_dtype')))
+                    _logger.debug(
+                        "---- op type: {}, in var [name: {} dtype: {}], out var [name: {} dtype: {}], attr [in_dtype {} out_dtype {}] ----".format(
+                            op.type,
+                            op.input('X')[0],
+                            in_var.dtype,
+                            op.output('Out')[0],
+                            out_var.dtype,
+                            op.attr('in_dtype'),
+                            op.attr('out_dtype'),
+                        )
+                    )
+                    continue
                 # divide others ops into fp16/fp32 sets according to promoting principle.
                 dst_dtype = dest_type
                 if not use_promote:

From 60cf9b508e2f22447737dba5a5ee171a6ebd49de Mon Sep 17 00:00:00 2001
From: 6clc <569519574@qq.com>
Date: Fri, 12 May 2023 17:27:59 +0800
Subject: [PATCH 310/405] test(prim-cinn): split test_resnet and test_bert into
 three tests (#53723)

* test(prim-cinn): split test_resnet and test_bert into three tests

* test(prim-cinn): fix cmake file to run prim test in CINN-CI
---
 test/prim/model/CMakeLists.txt           |  13 +-
 test/prim/model/test_bert_cinn.py        | 144 +++++++++++++++++
 test/prim/model/test_bert_prim.py        | 142 +++++++++++++++++
 test/prim/model/test_bert_prim_cinn.py   |  42 -----
 test/prim/model/test_resnet_cinn.py      | 189 +++++++++++++++++++++++
 test/prim/model/test_resnet_prim.py      | 189 +++++++++++++++++++++++
 test/prim/model/test_resnet_prim_cinn.py |  65 --------
 7 files changed, 675 insertions(+), 109 deletions(-)
 create mode 100644 test/prim/model/test_bert_cinn.py
 create mode 100644 test/prim/model/test_bert_prim.py
 create mode 100644 test/prim/model/test_resnet_cinn.py
 create mode 100644 test/prim/model/test_resnet_prim.py

diff --git a/test/prim/model/CMakeLists.txt b/test/prim/model/CMakeLists.txt
index 908c5ce625447..c37a25924aa97 100644
--- a/test/prim/model/CMakeLists.txt
+++ b/test/prim/model/CMakeLists.txt
@@ -8,12 +8,21 @@ foreach(TEST_OP ${TEST_OPS})
   py_test_modules(${TEST_OP} MODULES ${TEST_OP} ENVS ${GC_ENVS})
 endforeach()
 
-set_tests_properties(test_resnet_prim_cinn PROPERTIES TIMEOUT 850)
-set_tests_properties(test_bert_prim_cinn PROPERTIES TIMEOUT 500)
+set_tests_properties(test_resnet_prim PROPERTIES TIMEOUT 850)
+set_tests_properties(test_bert_prim PROPERTIES TIMEOUT 500)
 set_tests_properties(test_prim_simplenet_cinn PROPERTIES TIMEOUT 120)
 
 if(WITH_CINN)
+  set_tests_properties(test_resnet_cinn PROPERTIES TIMEOUT 850)
+  set_tests_properties(test_resnet_prim_cinn PROPERTIES TIMEOUT 850)
+  set_tests_properties(test_bert_cinn PROPERTIES TIMEOUT 500)
+  set_tests_properties(test_bert_prim_cinn PROPERTIES TIMEOUT 500)
+
+  set_tests_properties(test_resnet_prim PROPERTIES LABELS "RUN_TYPE=CINN")
+  set_tests_properties(test_resnet_cinn PROPERTIES LABELS "RUN_TYPE=CINN")
   set_tests_properties(test_resnet_prim_cinn PROPERTIES LABELS "RUN_TYPE=CINN")
+  set_tests_properties(test_bert_prim PROPERTIES LABELS "RUN_TYPE=CINN")
+  set_tests_properties(test_bert_cinn PROPERTIES LABELS "RUN_TYPE=CINN")
   set_tests_properties(test_bert_prim_cinn PROPERTIES LABELS "RUN_TYPE=CINN")
   set_tests_properties(test_prim_simplenet_cinn PROPERTIES LABELS
                                                            "RUN_TYPE=CINN")
diff --git a/test/prim/model/test_bert_cinn.py b/test/prim/model/test_bert_cinn.py
new file mode 100644
index 0000000000000..c647f72fdd68c
--- /dev/null
+++ b/test/prim/model/test_bert_cinn.py
@@ -0,0 +1,144 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+import unittest
+
+import numpy as np
+from bert import Bert, BertPretrainingCriterion, create_pretraining_dataset
+
+import paddle
+from paddle import fluid
+from paddle.dataset.common import DATA_HOME, download
+from paddle.fluid import core
+
+SEED = 2023
+BATCH_SIZE = 2
+
+URL = 'https://paddle-ci.gz.bcebos.com/prim_cinn/bert_training_data.npz'
+MODULE_NAME = 'test_bert_prim_cinn'
+MD5SUM = '71e730ee8d7aa77a215b7e898aa089af'
+SAVE_NAME = 'bert_training_data.npz'
+
+
+DY2ST_CINN_GT = [
+    10.649632453918457,
+    10.333406448364258,
+    10.33541202545166,
+    10.260543823242188,
+    10.219606399536133,
+    10.176884651184082,
+    10.124699592590332,
+    10.072620391845703,
+    10.112163543701172,
+    9.969393730163574,
+]
+
+
+if core.is_compiled_with_cuda():
+    paddle.set_flags({'FLAGS_cudnn_deterministic': True})
+
+
+def train(to_static, enable_prim, enable_cinn):
+    if core.is_compiled_with_cuda():
+        paddle.set_device('gpu')
+    else:
+        paddle.set_device('cpu')
+    fluid.core._set_prim_all_enabled(enable_prim)
+
+    np.random.seed(SEED)
+    paddle.seed(SEED)
+    # paddle.framework.random._manual_program_seed(SEED)
+
+    train_data_loader = create_pretraining_dataset(
+        os.path.join(DATA_HOME, MODULE_NAME, SAVE_NAME),
+        20,
+        {},
+        batch_size=BATCH_SIZE,
+        worker_init=None,
+    )
+
+    # Now only apply dy2st for encoder
+    bert = Bert(to_static, enable_cinn)
+    criterion = BertPretrainingCriterion()
+
+    optimizer = fluid.optimizer.Adam(parameter_list=bert.parameters())
+
+    losses = []
+    for step, batch in enumerate(train_data_loader):
+        start_time = time.time()
+        (
+            input_ids,
+            segment_ids,
+            input_mask,
+            masked_lm_positions,
+            masked_lm_labels,
+            next_sentence_labels,
+            masked_lm_scale,
+        ) = batch
+
+        prediction_scores, seq_relationship_score = bert(
+            input_ids=input_ids,
+            token_type_ids=segment_ids,
+            attention_mask=input_mask,
+            masked_positions=masked_lm_positions,
+        )
+
+        loss = criterion(
+            prediction_scores,
+            seq_relationship_score,
+            masked_lm_labels,
+            next_sentence_labels,
+            masked_lm_scale,
+        )
+
+        loss.backward()
+        optimizer.minimize(loss)
+        bert.clear_gradients()
+        losses.append(loss.numpy().item())
+
+        print(
+            "step: {}, loss: {}, batch_cost: {:.5}".format(
+                step,
+                loss.numpy(),
+                time.time() - start_time,
+            )
+        )
+        if step >= 9:
+            break
+    print(losses)
+    return losses
+
+
+class TestBert(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        download(URL, MODULE_NAME, MD5SUM, SAVE_NAME)
+
+    def tearDown(self):
+        paddle.set_flags({'FLAGS_deny_cinn_ops': ''})
+
+    @unittest.skipIf(
+        not (paddle.is_compiled_with_cinn() and paddle.is_compiled_with_cuda()),
+        "paddle is not compiled with CINN and CUDA",
+    )
+    def test_cinn(self):
+        paddle.set_flags({'FLAGS_deny_cinn_ops': "dropout"})
+        dy2st_cinn = train(to_static=True, enable_prim=False, enable_cinn=True)
+        np.testing.assert_allclose(dy2st_cinn, DY2ST_CINN_GT, rtol=1e-5)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/prim/model/test_bert_prim.py b/test/prim/model/test_bert_prim.py
new file mode 100644
index 0000000000000..4044620f7e8fc
--- /dev/null
+++ b/test/prim/model/test_bert_prim.py
@@ -0,0 +1,142 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+import unittest
+
+import numpy as np
+from bert import Bert, BertPretrainingCriterion, create_pretraining_dataset
+
+import paddle
+from paddle import fluid
+from paddle.dataset.common import DATA_HOME, download
+from paddle.fluid import core
+
+SEED = 2023
+BATCH_SIZE = 2
+
+URL = 'https://paddle-ci.gz.bcebos.com/prim_cinn/bert_training_data.npz'
+MODULE_NAME = 'test_bert_prim_cinn'
+MD5SUM = '71e730ee8d7aa77a215b7e898aa089af'
+SAVE_NAME = 'bert_training_data.npz'
+
+
+DY2ST_PRIM_GT = [
+    11.144556999206543,
+    10.343620300292969,
+    10.330279350280762,
+    10.276118278503418,
+    10.222086906433105,
+    10.194628715515137,
+    10.14902114868164,
+    10.096250534057617,
+    10.104615211486816,
+    9.985644340515137,
+]
+
+if core.is_compiled_with_cuda():
+    paddle.set_flags({'FLAGS_cudnn_deterministic': True})
+
+
+def train(to_static, enable_prim, enable_cinn):
+    if core.is_compiled_with_cuda():
+        paddle.set_device('gpu')
+    else:
+        paddle.set_device('cpu')
+    fluid.core._set_prim_all_enabled(enable_prim)
+
+    np.random.seed(SEED)
+    paddle.seed(SEED)
+    # paddle.framework.random._manual_program_seed(SEED)
+
+    train_data_loader = create_pretraining_dataset(
+        os.path.join(DATA_HOME, MODULE_NAME, SAVE_NAME),
+        20,
+        {},
+        batch_size=BATCH_SIZE,
+        worker_init=None,
+    )
+
+    # Now only apply dy2st for encoder
+    bert = Bert(to_static, enable_cinn)
+    criterion = BertPretrainingCriterion()
+
+    optimizer = fluid.optimizer.Adam(parameter_list=bert.parameters())
+
+    losses = []
+    for step, batch in enumerate(train_data_loader):
+        start_time = time.time()
+        (
+            input_ids,
+            segment_ids,
+            input_mask,
+            masked_lm_positions,
+            masked_lm_labels,
+            next_sentence_labels,
+            masked_lm_scale,
+        ) = batch
+
+        prediction_scores, seq_relationship_score = bert(
+            input_ids=input_ids,
+            token_type_ids=segment_ids,
+            attention_mask=input_mask,
+            masked_positions=masked_lm_positions,
+        )
+
+        loss = criterion(
+            prediction_scores,
+            seq_relationship_score,
+            masked_lm_labels,
+            next_sentence_labels,
+            masked_lm_scale,
+        )
+
+        loss.backward()
+        optimizer.minimize(loss)
+        bert.clear_gradients()
+        losses.append(loss.numpy().item())
+
+        print(
+            "step: {}, loss: {}, batch_cost: {:.5}".format(
+                step,
+                loss.numpy(),
+                time.time() - start_time,
+            )
+        )
+        if step >= 9:
+            break
+    print(losses)
+    return losses
+
+
+class TestBert(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        download(URL, MODULE_NAME, MD5SUM, SAVE_NAME)
+
+    def tearDown(self):
+        paddle.set_flags({'FLAGS_deny_cinn_ops': ''})
+
+    @unittest.skipIf(
+        not (paddle.is_compiled_with_cinn() and paddle.is_compiled_with_cuda()),
+        "paddle is not compiled with CINN and CUDA",
+    )
+    def test_prim(self):
+        dy2st_prim = train(to_static=True, enable_prim=True, enable_cinn=False)
+        np.testing.assert_allclose(dy2st_prim, DY2ST_PRIM_GT, rtol=1e-5)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/prim/model/test_bert_prim_cinn.py b/test/prim/model/test_bert_prim_cinn.py
index c6438536673f6..3170dba893cf1 100644
--- a/test/prim/model/test_bert_prim_cinn.py
+++ b/test/prim/model/test_bert_prim_cinn.py
@@ -33,31 +33,6 @@
 SAVE_NAME = 'bert_training_data.npz'
 
 
-DY2ST_PRIM_GT = [
-    11.144556999206543,
-    10.343620300292969,
-    10.330279350280762,
-    10.276118278503418,
-    10.222086906433105,
-    10.194628715515137,
-    10.14902114868164,
-    10.096250534057617,
-    10.104615211486816,
-    9.985644340515137,
-]
-DY2ST_CINN_GT = [
-    10.649632453918457,
-    10.333406448364258,
-    10.33541202545166,
-    10.260543823242188,
-    10.219606399536133,
-    10.176884651184082,
-    10.124699592590332,
-    10.072620391845703,
-    10.112163543701172,
-    9.969393730163574,
-]
-
 DY2ST_PRIM_CINN_GT = [
     10.976988792419434,
     10.345282554626465,
@@ -155,23 +130,6 @@ def setUpClass(cls):
     def tearDown(self):
         paddle.set_flags({'FLAGS_deny_cinn_ops': ''})
 
-    @unittest.skipIf(
-        not (paddle.is_compiled_with_cinn() and paddle.is_compiled_with_cuda()),
-        "paddle is not compiled with CINN and CUDA",
-    )
-    def test_prim(self):
-        dy2st_prim = train(to_static=True, enable_prim=True, enable_cinn=False)
-        np.testing.assert_allclose(dy2st_prim, DY2ST_PRIM_GT, rtol=1e-5)
-
-    @unittest.skipIf(
-        not (paddle.is_compiled_with_cinn() and paddle.is_compiled_with_cuda()),
-        "paddle is not compiled with CINN and CUDA",
-    )
-    def test_cinn(self):
-        paddle.set_flags({'FLAGS_deny_cinn_ops': "dropout"})
-        dy2st_cinn = train(to_static=True, enable_prim=False, enable_cinn=True)
-        np.testing.assert_allclose(dy2st_cinn, DY2ST_CINN_GT, rtol=1e-5)
-
     @unittest.skipIf(
         not (paddle.is_compiled_with_cinn() and paddle.is_compiled_with_cuda()),
         "paddle is not compiled with CINN and CUDA",
diff --git a/test/prim/model/test_resnet_cinn.py b/test/prim/model/test_resnet_cinn.py
new file mode 100644
index 0000000000000..db20dc5a605e1
--- /dev/null
+++ b/test/prim/model/test_resnet_cinn.py
@@ -0,0 +1,189 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+import unittest
+
+import numpy as np
+
+import paddle
+from paddle import fluid
+from paddle.fluid import core
+from paddle.vision.models import resnet50
+
+SEED = 2020
+base_lr = 0.001
+momentum_rate = 0.9
+l2_decay = 1e-4
+batch_size = 2
+epoch_num = 1
+
+# In V100, 16G, CUDA 11.2, the results are as follows:
+# DY2ST_CINN_GT = [
+#     5.847336769104004,
+#     8.336246490478516,
+#     5.108744144439697,
+#     8.316713333129883,
+#     8.175262451171875,
+#     7.590441703796387,
+#     9.895681381225586,
+#     8.196207046508789,
+#     8.438933372497559,
+#     10.305074691772461,
+
+
+# The results in ci as as follows:
+DY2ST_CINN_GT = [
+    5.828789710998535,
+    8.340764999389648,
+    4.998944282531738,
+    8.474305152893066,
+    8.09157943725586,
+    7.440057754516602,
+    9.907357215881348,
+    8.304681777954102,
+    8.383116722106934,
+    10.120304107666016,
+]
+
+if core.is_compiled_with_cuda():
+    paddle.set_flags({'FLAGS_cudnn_deterministic': True})
+
+
+def reader_decorator(reader):
+    def __reader__():
+        for item in reader():
+            img = np.array(item[0]).astype('float32').reshape(3, 224, 224)
+            label = np.array(item[1]).astype('int64').reshape(1)
+            yield img, label
+
+    return __reader__
+
+
+def optimizer_setting(parameter_list=None):
+    optimizer = fluid.optimizer.Momentum(
+        learning_rate=base_lr,
+        momentum=momentum_rate,
+        regularization=fluid.regularizer.L2Decay(l2_decay),
+        parameter_list=parameter_list,
+    )
+
+    return optimizer
+
+
+def run(model, data_loader, optimizer, mode):
+    if mode == 'train':
+        model.train()
+        end_step = 9
+    elif mode == 'eval':
+        model.eval()
+        end_step = 1
+
+    for epoch in range(epoch_num):
+        total_acc1 = 0.0
+        total_acc5 = 0.0
+        total_sample = 0
+        losses = []
+
+        for batch_id, data in enumerate(data_loader()):
+            start_time = time.time()
+            img, label = data
+
+            pred = model(img)
+            avg_loss = paddle.nn.functional.cross_entropy(
+                input=pred,
+                label=label,
+                soft_label=False,
+                reduction='mean',
+                use_softmax=True,
+            )
+
+            acc_top1 = paddle.static.accuracy(input=pred, label=label, k=1)
+            acc_top5 = paddle.static.accuracy(input=pred, label=label, k=5)
+
+            if mode == 'train':
+                avg_loss.backward()
+                optimizer.minimize(avg_loss)
+                model.clear_gradients()
+
+            total_acc1 += acc_top1
+            total_acc5 += acc_top5
+            total_sample += 1
+            losses.append(avg_loss.numpy().item())
+
+            end_time = time.time()
+            print(
+                "[%s]epoch %d | batch step %d, loss %0.8f, acc1 %0.3f, acc5 %0.3f, time %f"
+                % (
+                    mode,
+                    epoch,
+                    batch_id,
+                    avg_loss,
+                    total_acc1.numpy() / total_sample,
+                    total_acc5.numpy() / total_sample,
+                    end_time - start_time,
+                )
+            )
+            if batch_id >= end_step:
+                # avoid dataloader throw abort signaal
+                data_loader._reset()
+                break
+    print(losses)
+    return losses
+
+
+def train(to_static, enable_prim, enable_cinn):
+    if core.is_compiled_with_cuda():
+        paddle.set_device('gpu')
+    else:
+        paddle.set_device('cpu')
+    np.random.seed(SEED)
+    paddle.seed(SEED)
+    paddle.framework.random._manual_program_seed(SEED)
+    fluid.core._set_prim_all_enabled(enable_prim)
+
+    train_reader = paddle.batch(
+        reader_decorator(paddle.dataset.flowers.train(use_xmap=False)),
+        batch_size=batch_size,
+        drop_last=True,
+    )
+    data_loader = fluid.io.DataLoader.from_generator(capacity=5, iterable=True)
+    data_loader.set_sample_list_generator(train_reader)
+
+    resnet = resnet50(False)
+    if to_static:
+        build_strategy = paddle.static.BuildStrategy()
+        if enable_cinn:
+            build_strategy.build_cinn_pass = True
+        resnet = paddle.jit.to_static(resnet, build_strategy=build_strategy)
+    optimizer = optimizer_setting(parameter_list=resnet.parameters())
+
+    train_losses = run(resnet, data_loader, optimizer, 'train')
+    if to_static and enable_prim and enable_cinn:
+        eval_losses = run(resnet, data_loader, optimizer, 'eval')
+    return train_losses
+
+
+class TestResnet(unittest.TestCase):
+    @unittest.skipIf(
+        not (paddle.is_compiled_with_cinn() and paddle.is_compiled_with_cuda()),
+        "paddle is not compiled with CINN and CUDA",
+    )
+    def test_cinn(self):
+        dy2st_cinn = train(to_static=True, enable_prim=False, enable_cinn=True)
+        np.testing.assert_allclose(dy2st_cinn, DY2ST_CINN_GT, rtol=1e-5)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/prim/model/test_resnet_prim.py b/test/prim/model/test_resnet_prim.py
new file mode 100644
index 0000000000000..79c169c2272b3
--- /dev/null
+++ b/test/prim/model/test_resnet_prim.py
@@ -0,0 +1,189 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+import unittest
+
+import numpy as np
+
+import paddle
+from paddle import fluid
+from paddle.fluid import core
+from paddle.vision.models import resnet50
+
+SEED = 2020
+base_lr = 0.001
+momentum_rate = 0.9
+l2_decay = 1e-4
+batch_size = 2
+epoch_num = 1
+
+# In V100, 16G, CUDA 11.2, the results are as follows:
+# DY2ST_PRIM_GT = [
+#     5.8473358154296875,
+#     8.354944229125977,
+#     5.098367691040039,
+#     8.533346176147461,
+#     8.179085731506348,
+#     7.285282135009766,
+#     9.824585914611816,
+#     8.56928825378418,
+#     8.539499282836914,
+#     10.256929397583008,
+# ]
+
+# The results in ci as as follows:
+DY2ST_PRIM_GT = [
+    5.82879114151001,
+    8.33370590209961,
+    5.091761589050293,
+    8.776082992553711,
+    8.274380683898926,
+    7.546653747558594,
+    9.607137680053711,
+    8.27371597290039,
+    8.429732322692871,
+    10.362630844116211,
+]
+
+if core.is_compiled_with_cuda():
+    paddle.set_flags({'FLAGS_cudnn_deterministic': True})
+
+
+def reader_decorator(reader):
+    def __reader__():
+        for item in reader():
+            img = np.array(item[0]).astype('float32').reshape(3, 224, 224)
+            label = np.array(item[1]).astype('int64').reshape(1)
+            yield img, label
+
+    return __reader__
+
+
+def optimizer_setting(parameter_list=None):
+    optimizer = fluid.optimizer.Momentum(
+        learning_rate=base_lr,
+        momentum=momentum_rate,
+        regularization=fluid.regularizer.L2Decay(l2_decay),
+        parameter_list=parameter_list,
+    )
+
+    return optimizer
+
+
+def run(model, data_loader, optimizer, mode):
+    if mode == 'train':
+        model.train()
+        end_step = 9
+    elif mode == 'eval':
+        model.eval()
+        end_step = 1
+
+    for epoch in range(epoch_num):
+        total_acc1 = 0.0
+        total_acc5 = 0.0
+        total_sample = 0
+        losses = []
+
+        for batch_id, data in enumerate(data_loader()):
+            start_time = time.time()
+            img, label = data
+
+            pred = model(img)
+            avg_loss = paddle.nn.functional.cross_entropy(
+                input=pred,
+                label=label,
+                soft_label=False,
+                reduction='mean',
+                use_softmax=True,
+            )
+
+            acc_top1 = paddle.static.accuracy(input=pred, label=label, k=1)
+            acc_top5 = paddle.static.accuracy(input=pred, label=label, k=5)
+
+            if mode == 'train':
+                avg_loss.backward()
+                optimizer.minimize(avg_loss)
+                model.clear_gradients()
+
+            total_acc1 += acc_top1
+            total_acc5 += acc_top5
+            total_sample += 1
+            losses.append(avg_loss.numpy().item())
+
+            end_time = time.time()
+            print(
+                "[%s]epoch %d | batch step %d, loss %0.8f, acc1 %0.3f, acc5 %0.3f, time %f"
+                % (
+                    mode,
+                    epoch,
+                    batch_id,
+                    avg_loss,
+                    total_acc1.numpy() / total_sample,
+                    total_acc5.numpy() / total_sample,
+                    end_time - start_time,
+                )
+            )
+            if batch_id >= end_step:
+                # avoid dataloader throw abort signaal
+                data_loader._reset()
+                break
+    print(losses)
+    return losses
+
+
+def train(to_static, enable_prim, enable_cinn):
+    if core.is_compiled_with_cuda():
+        paddle.set_device('gpu')
+    else:
+        paddle.set_device('cpu')
+    np.random.seed(SEED)
+    paddle.seed(SEED)
+    paddle.framework.random._manual_program_seed(SEED)
+    fluid.core._set_prim_all_enabled(enable_prim)
+
+    train_reader = paddle.batch(
+        reader_decorator(paddle.dataset.flowers.train(use_xmap=False)),
+        batch_size=batch_size,
+        drop_last=True,
+    )
+    data_loader = fluid.io.DataLoader.from_generator(capacity=5, iterable=True)
+    data_loader.set_sample_list_generator(train_reader)
+
+    resnet = resnet50(False)
+    if to_static:
+        build_strategy = paddle.static.BuildStrategy()
+        if enable_cinn:
+            build_strategy.build_cinn_pass = True
+        resnet = paddle.jit.to_static(resnet, build_strategy=build_strategy)
+    optimizer = optimizer_setting(parameter_list=resnet.parameters())
+
+    train_losses = run(resnet, data_loader, optimizer, 'train')
+    if to_static and enable_prim and enable_cinn:
+        eval_losses = run(resnet, data_loader, optimizer, 'eval')
+    return train_losses
+
+
+class TestResnet(unittest.TestCase):
+    @unittest.skipIf(
+        not (paddle.is_compiled_with_cinn() and paddle.is_compiled_with_cuda()),
+        "paddle is not compiled with CINN and CUDA",
+    )
+    def test_prim(self):
+        dy2st_prim = train(to_static=True, enable_prim=True, enable_cinn=False)
+        np.testing.assert_allclose(dy2st_prim, DY2ST_PRIM_GT, rtol=1e-5)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/prim/model/test_resnet_prim_cinn.py b/test/prim/model/test_resnet_prim_cinn.py
index 46ea9bfba72a7..5e91de52c7ae6 100644
--- a/test/prim/model/test_resnet_prim_cinn.py
+++ b/test/prim/model/test_resnet_prim_cinn.py
@@ -30,30 +30,6 @@
 epoch_num = 1
 
 # In V100, 16G, CUDA 11.2, the results are as follows:
-# DY2ST_PRIM_GT = [
-#     5.8473358154296875,
-#     8.354944229125977,
-#     5.098367691040039,
-#     8.533346176147461,
-#     8.179085731506348,
-#     7.285282135009766,
-#     9.824585914611816,
-#     8.56928825378418,
-#     8.539499282836914,
-#     10.256929397583008,
-# ]
-# DY2ST_CINN_GT = [
-#     5.847336769104004,
-#     8.336246490478516,
-#     5.108744144439697,
-#     8.316713333129883,
-#     8.175262451171875,
-#     7.590441703796387,
-#     9.895681381225586,
-#     8.196207046508789,
-#     8.438933372497559,
-#     10.305074691772461,
-# ]
 # DY2ST_PRIM_CINN_GT = [
 #     5.8473358154296875,
 #     8.322463989257812,
@@ -68,31 +44,6 @@
 # ]
 
 # The results in ci as as follows:
-DY2ST_PRIM_GT = [
-    5.82879114151001,
-    8.33370590209961,
-    5.091761589050293,
-    8.776082992553711,
-    8.274380683898926,
-    7.546653747558594,
-    9.607137680053711,
-    8.27371597290039,
-    8.429732322692871,
-    10.362630844116211,
-]
-DY2ST_CINN_GT = [
-    5.828789710998535,
-    8.340764999389648,
-    4.998944282531738,
-    8.474305152893066,
-    8.09157943725586,
-    7.440057754516602,
-    9.907357215881348,
-    8.304681777954102,
-    8.383116722106934,
-    10.120304107666016,
-]
-
 DY2ST_PRIM_CINN_GT = [
     5.828786849975586,
     8.332868576049805,
@@ -225,22 +176,6 @@ def train(to_static, enable_prim, enable_cinn):
 
 
 class TestResnet(unittest.TestCase):
-    @unittest.skipIf(
-        not (paddle.is_compiled_with_cinn() and paddle.is_compiled_with_cuda()),
-        "paddle is not compiled with CINN and CUDA",
-    )
-    def test_prim(self):
-        dy2st_prim = train(to_static=True, enable_prim=True, enable_cinn=False)
-        np.testing.assert_allclose(dy2st_prim, DY2ST_PRIM_GT, rtol=1e-5)
-
-    @unittest.skipIf(
-        not (paddle.is_compiled_with_cinn() and paddle.is_compiled_with_cuda()),
-        "paddle is not compiled with CINN and CUDA",
-    )
-    def test_cinn(self):
-        dy2st_cinn = train(to_static=True, enable_prim=False, enable_cinn=True)
-        np.testing.assert_allclose(dy2st_cinn, DY2ST_CINN_GT, rtol=1e-5)
-
     @unittest.skipIf(
         not (paddle.is_compiled_with_cinn() and paddle.is_compiled_with_cuda()),
         "paddle is not compiled with CINN and CUDA",

From c497b43f90cf39b2384700663240d78c4d9450a7 Mon Sep 17 00:00:00 2001
From: ronnywang <ronny1996@163.com>
Date: Fri, 12 May 2023 19:27:43 +0800
Subject: [PATCH 311/405] [CustomDevice] add inference MP support, PART1
 (#53702)

---
 .../fleet_executor/cond_interceptor.cc        |  7 +++
 .../distributed/fleet_executor/dist_model.cc  | 45 ++++++++++++++++++-
 .../distributed/fleet_executor/dist_model.h   |  1 +
 .../distributed/fleet_executor/message_bus.cc |  2 +-
 .../framework/details/multi_devices_helper.cc |  1 +
 paddle/fluid/framework/var_type_traits.h      |  7 +++
 .../ir_params_sync_among_devices_pass.cc      |  5 +--
 .../fluid/inference/api/analysis_predictor.cc | 43 ++++++++++++++++--
 paddle/phi/core/flags.cc                      |  2 +-
 9 files changed, 104 insertions(+), 9 deletions(-)

diff --git a/paddle/fluid/distributed/fleet_executor/cond_interceptor.cc b/paddle/fluid/distributed/fleet_executor/cond_interceptor.cc
index 846f3d722e1fa..2e3389af5feb5 100644
--- a/paddle/fluid/distributed/fleet_executor/cond_interceptor.cc
+++ b/paddle/fluid/distributed/fleet_executor/cond_interceptor.cc
@@ -76,6 +76,13 @@ bool CondInterceptor::GetCondResult() {
     framework::TensorCopy(cond_tensor, platform::CPUPlace(), &cpu_tensor);
     platform::DeviceContextPool::Instance().Get(cond_tensor.place())->Wait();
     res = cpu_tensor.data<bool>()[0];
+#endif
+  } else if (platform::is_custom_place(cond_tensor.place())) {
+#if defined(PADDLE_WITH_CUSTOM_DEVICE)
+    phi::DenseTensor cpu_tensor;
+    framework::TensorCopy(cond_tensor, platform::CPUPlace(), &cpu_tensor);
+    platform::DeviceContextPool::Instance().Get(cond_tensor.place())->Wait();
+    res = cpu_tensor.data<bool>()[0];
 #endif
   } else if (platform::is_cpu_place(cond_tensor.place())) {
     res = cond_tensor.data<bool>()[0];
diff --git a/paddle/fluid/distributed/fleet_executor/dist_model.cc b/paddle/fluid/distributed/fleet_executor/dist_model.cc
index 02ee53a4b444a..4836d656d180f 100644
--- a/paddle/fluid/distributed/fleet_executor/dist_model.cc
+++ b/paddle/fluid/distributed/fleet_executor/dist_model.cc
@@ -102,10 +102,27 @@ bool LoadDataFromDistModelTensor(const DistModelTensor &input_data,
 #else
     PADDLE_THROW(paddle::platform::errors::Fatal(
         "Paddle wasn't compiled with XPU, but place is XPU."));
+#endif
+  } else if (platform::is_custom_place(place)) {
+    VLOG(3) << "Loading data for CustomDevice: " << place;
+#if defined(PADDLE_WITH_CUSTOM_DEVICE)
+    platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance();
+    auto *dev_ctx = dynamic_cast<const phi::CustomContext *>(pool.Get(place));
+    auto custom_place = place;
+    memory::Copy(custom_place,
+                 static_cast<void *>(input_tensor_ptr),
+                 platform::CPUPlace(),
+                 input_data.data.data(),
+                 input_data.data.length(),
+                 dev_ctx->stream());
+#else
+    PADDLE_THROW(paddle::platform::errors::Fatal(
+        "Paddle wasn't compiled with custom_device, but place is "
+        "CustomPlace."));
 #endif
   } else {
     PADDLE_THROW(paddle::platform::errors::InvalidArgument(
-        "DistModel only supports CPU and GPU and XPU."));
+        "DistModel only supports CPU and GPU and XPU and CustomDevice."));
   }
 
   framework::LoD dst_lod;
@@ -204,6 +221,9 @@ bool DistModel::PreparePlace() {
     place_ = paddle::platform::CPUPlace();
   } else if (config_.place == "XPU") {
     place_ = paddle::platform::XPUPlace(config_.device_id);
+  } else if (config_.place == "CUSTOM_DEVICE") {
+    place_ =
+        paddle::platform::CustomPlace(config_.device_type, config_.device_id);
   } else {
     PADDLE_THROW(platform::errors::InvalidArgument(
         "Place must be choosen from GPU or CPU or XPU, but got %s.",
@@ -324,6 +344,29 @@ void DistModel::InsertCommOp(std::string tmp_var_name,
     comm_init_op->SetAttr("op_role",
                           static_cast<int>(framework::OpRole::kForward));
     comm_init_op->CheckAttrs();
+  } else if (config_.place == "CUSTOM_DEVICE") {
+    framework::VarDesc *new_var = block->Var(tmp_var_name);
+    new_var->SetType(framework::proto::VarType::RAW);
+    new_var->SetPersistable(true);
+    framework::OpDesc *gen_bkcl_id_op = block->AppendOp();
+    gen_bkcl_id_op->SetType("c_gen_xccl_id");
+    gen_bkcl_id_op->SetOutput("Out", {tmp_var_name});
+    gen_bkcl_id_op->SetAttr("rank", rank);
+    gen_bkcl_id_op->SetAttr("endpoint", config_.current_endpoint);
+    gen_bkcl_id_op->SetAttr("other_endpoints", peer_endpoints);
+    gen_bkcl_id_op->SetAttr("ring_id", ring_id);
+    gen_bkcl_id_op->SetAttr("op_role",
+                            static_cast<int>(framework::OpRole::kForward));
+    gen_bkcl_id_op->CheckAttrs();
+    framework::OpDesc *comm_init_op = block->AppendOp();
+    comm_init_op->SetType("c_comm_init");
+    comm_init_op->SetInput("X", {tmp_var_name});
+    comm_init_op->SetAttr("rank", rank);
+    comm_init_op->SetAttr("nranks", nranks);
+    comm_init_op->SetAttr("ring_id", ring_id);
+    comm_init_op->SetAttr("op_role",
+                          static_cast<int>(framework::OpRole::kForward));
+    comm_init_op->CheckAttrs();
   } else {
     LOG(WARNING) << "DistModelInf doesn't init comm.";
     // TODO(fleet exe dev): comm init for more devices
diff --git a/paddle/fluid/distributed/fleet_executor/dist_model.h b/paddle/fluid/distributed/fleet_executor/dist_model.h
index dffc33b118c20..e69ce6ee24a16 100644
--- a/paddle/fluid/distributed/fleet_executor/dist_model.h
+++ b/paddle/fluid/distributed/fleet_executor/dist_model.h
@@ -44,6 +44,7 @@ struct DistModelConfig {
   framework::Scope* scope{nullptr};
   std::string place{};
   int64_t device_id{0};
+  std::string device_type{};
   std::vector<std::string> trainer_endpoints{};
   std::string current_endpoint{};
   int64_t nranks{1};
diff --git a/paddle/fluid/distributed/fleet_executor/message_bus.cc b/paddle/fluid/distributed/fleet_executor/message_bus.cc
index 099be64f66746..ba28b52c03fa1 100644
--- a/paddle/fluid/distributed/fleet_executor/message_bus.cc
+++ b/paddle/fluid/distributed/fleet_executor/message_bus.cc
@@ -52,7 +52,7 @@ void MessageBus::Init(
   }
 
 #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) || \
-    defined(PADDLE_WITH_XPU_BKCL)
+    defined(PADDLE_WITH_XPU_BKCL) || defined(PADDLE_WITH_CUSTOM_DEVICE)
   // NOTE: To make the brpc is compatible with collective,
   // need release the handler holding the ip address.
   if (addr_ != "") {
diff --git a/paddle/fluid/framework/details/multi_devices_helper.cc b/paddle/fluid/framework/details/multi_devices_helper.cc
index 945895183d9c0..62de76a8753cb 100644
--- a/paddle/fluid/framework/details/multi_devices_helper.cc
+++ b/paddle/fluid/framework/details/multi_devices_helper.cc
@@ -43,6 +43,7 @@ static std::unordered_set<std::string> kMultiDeviceOps{
     "c_comm_init_multitrainer",
     "c_gen_nccl_id",
     "c_gen_bkcl_id",
+    "c_gen_xccl_id",
     "c_sync_comm_stream",
     "send",
     "recv",
diff --git a/paddle/fluid/framework/var_type_traits.h b/paddle/fluid/framework/var_type_traits.h
index 3ad1b4db709d1..c705fc72f2a82 100644
--- a/paddle/fluid/framework/var_type_traits.h
+++ b/paddle/fluid/framework/var_type_traits.h
@@ -43,6 +43,10 @@
 #include "xpu/bkcl.h"
 #endif
 
+#if defined(PADDLE_WITH_CUSTOM_DEVICE)
+#include "paddle/phi/backends/c_comm_lib.h"
+#endif
+
 namespace phi {
 class DenseTensor;
 class SelectedRows;
@@ -195,6 +199,9 @@ using VarTypeRegistry = detail::VarTypeRegistryImpl<
 #if defined(PADDLE_WITH_XPU_BKCL)
     BKCLUniqueId,
     platform::BKCLCommunicator,
+#endif
+#if defined(PADDLE_WITH_CUSTOM_DEVICE)
+    phi::ccl::CCLRootId,
 #endif
     std::vector<std::unique_ptr<operators::CUDAGraphWithInOuts>>,
     int,
diff --git a/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc b/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc
index 3e0d3348bc790..54b7cd85ff2bb 100644
--- a/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc
+++ b/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc
@@ -139,9 +139,8 @@ void IrParamsSyncAmongDevicesPass::CopyParamsToCustomDevice(
     repetitive_params = graph.Get<std::vector<std::string>>(
         framework::ir::kRepetitiveParamAttr);
 
-  LOG(INFO) << "Sync params from CPU to CustomDevice"
-            << argument->custom_device_type() << "/"
-            << argument->custom_device_id();
+  LOG(INFO) << "Sync params from CPU to " << argument->custom_device_type()
+            << ":" << argument->custom_device_id();
 
   platform::Place place = platform::CustomPlace(argument->custom_device_type(),
                                                 argument->custom_device_id());
diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc
index 1ab5366d34c95..032b5d7734ce7 100644
--- a/paddle/fluid/inference/api/analysis_predictor.cc
+++ b/paddle/fluid/inference/api/analysis_predictor.cc
@@ -475,7 +475,8 @@ void AnalysisPredictor::InitPlace() {
 #endif
   } else if (config_.use_custom_device()) {
 #ifdef PADDLE_WITH_CUSTOM_DEVICE
-    place_ = paddle::platform::CustomPlace(config_.custom_device_type());
+    place_ = paddle::platform::CustomPlace(config_.custom_device_type(),
+                                           config_.custom_device_id());
 #else
     PADDLE_THROW(platform::errors::Unavailable(
         "You tried to use CustomDevice forward propagation, but Paddle was not "
@@ -564,6 +565,14 @@ void *AnalysisPredictor::GetExecStream() const {
           ->stream();
     }
   }
+#endif
+#if defined(PADDLE_WITH_CUSTOM_DEVICE)
+  if (place_.GetType() == phi::AllocationType::CUSTOM) {
+    paddle::platform::DeviceContextPool &pool =
+        paddle::platform::DeviceContextPool::Instance();
+    return reinterpret_cast<const phi::CustomContext *>(pool.Get(place_))
+        ->stream();
+  }
 #endif
   // TODO(inference): Support other backends.
   return nullptr;
@@ -679,12 +688,16 @@ static void DisablePrepareDataOpt(
 }
 
 bool AnalysisPredictor::PrepareExecutor() {
-#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
   if (config_.dist_config().use_dist_model()) {
+#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
     VLOG(3) << "use_dist_model is enabled, will init FleetExecutor.";
     return PrepareFleetExecutor();
-  }
+#else
+    PADDLE_THROW(platform::errors::PermissionDenied(
+        "Paddle can't use FleetExecutor since it's not compiled with PSCORE,"
+        "Please recompile or reinstall Paddle with PSCORE support."));
 #endif
+  }
   DisablePrepareDataOpt(inference_program_, 0, false);
 
   executor_->Prepare(
@@ -875,6 +888,30 @@ void AnalysisPredictor::InsertCommOp(
     comm_init_op->SetAttr("op_role",
                           static_cast<int>(framework::OpRole::kForward));
     comm_init_op->CheckAttrs();
+  } else if (config_.use_custom_device()) {
+    framework::VarDesc *new_var = block->Var(tmp_var_name);
+    new_var->SetType(framework::proto::VarType::RAW);
+    new_var->SetPersistable(true);
+    framework::OpDesc *gen_bkcl_id_op = block->AppendOp();
+    gen_bkcl_id_op->SetType("c_gen_xccl_id");
+    gen_bkcl_id_op->SetOutput("Out", {tmp_var_name});
+    gen_bkcl_id_op->SetAttr("rank", rank);
+    gen_bkcl_id_op->SetAttr("endpoint",
+                            config_.dist_config().current_endpoint());
+    gen_bkcl_id_op->SetAttr("other_endpoints", peer_endpoints);
+    gen_bkcl_id_op->SetAttr("ring_id", ring_id);
+    gen_bkcl_id_op->SetAttr("op_role",
+                            static_cast<int>(framework::OpRole::kForward));
+    gen_bkcl_id_op->CheckAttrs();
+    framework::OpDesc *comm_init_op = block->AppendOp();
+    comm_init_op->SetType("c_comm_init");
+    comm_init_op->SetInput("X", {tmp_var_name});
+    comm_init_op->SetAttr("rank", rank);
+    comm_init_op->SetAttr("nranks", nranks);
+    comm_init_op->SetAttr("ring_id", ring_id);
+    comm_init_op->SetAttr("op_role",
+                          static_cast<int>(framework::OpRole::kForward));
+    comm_init_op->CheckAttrs();
   } else {
     LOG(WARNING) << "DistModelInf doesn't init comm.";
     // TODO(fleet exe dev): comm init for more devices
diff --git a/paddle/phi/core/flags.cc b/paddle/phi/core/flags.cc
index d2428e28b0dec..59c2901d8f4f1 100644
--- a/paddle/phi/core/flags.cc
+++ b/paddle/phi/core/flags.cc
@@ -819,7 +819,7 @@ PHI_DEFINE_EXPORTED_bool(use_fast_math,
  * Note: Get host by name time.
  */
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_XPU) || \
-    defined(PADDLE_WITH_HIP)
+    defined(PADDLE_WITH_HIP) || defined(PADDLE_WITH_CUSTOM_DEVICE)
 PHI_DEFINE_EXPORTED_int32(get_host_by_name_time,
                           120,
                           "The maximum time for get host by name time");

From 4d39cc7feedecd3d106b015b39e2dc23fec3ae6a Mon Sep 17 00:00:00 2001
From: Leo Chen <chenqiuliang@baidu.com>
Date: Fri, 12 May 2023 19:55:27 +0800
Subject: [PATCH 312/405] fix add_n kernel of large shape (#53749)

---
 paddle/phi/kernels/gpu/add_n_kernel.cu | 28 ++++++--------------------
 1 file changed, 6 insertions(+), 22 deletions(-)

diff --git a/paddle/phi/kernels/gpu/add_n_kernel.cu b/paddle/phi/kernels/gpu/add_n_kernel.cu
index 3c224d1e246c8..a30d25b018800 100644
--- a/paddle/phi/kernels/gpu/add_n_kernel.cu
+++ b/paddle/phi/kernels/gpu/add_n_kernel.cu
@@ -21,34 +21,20 @@ namespace phi {
 
 #define CEIL_DIV(x, y) (((x) + (y)-1) / (y))
 
-template <class T>
-__global__ void Sum2CUDAKernel(const T *in_0,
-                               const T *in_1,
-                               T *out,
-                               int64_t N) {
-  int id = blockIdx.x * blockDim.x + threadIdx.x;
-  while (id < N) {
-    out[id] = in_0[id] + in_1[id];
-    id += blockDim.x * gridDim.x;
-  }
-}
-
 template <class T>
 __global__ void SumArrayCUDAKernel(
     T **in, T *out, int64_t N, size_t in_size, bool read_dst) {
   using MPType = typename phi::dtype::MPTypeTrait<T>::Type;
-  int id = blockIdx.x * blockDim.x + threadIdx.x;
-  while (id < N) {
-    MPType total(read_dst ? static_cast<MPType>(out[id])
+  CUDA_KERNEL_LOOP_TYPE(idx, N, int64_t) {
+    MPType total(read_dst ? static_cast<MPType>(out[idx])
                           : static_cast<MPType>(0));
     for (int i = 0; i < in_size; ++i) {
       const T *tmp = in[i];
       if (tmp) {
-        total += static_cast<MPType>(tmp[id]);
+        total += static_cast<MPType>(tmp[idx]);
       }
     }
-    out[id] = static_cast<T>(total);
-    id += blockDim.x * gridDim.x;
+    out[idx] = static_cast<T>(total);
   }
 }
 
@@ -56,16 +42,14 @@ template <class T>
 __global__ void SumSelectedRowsCUDAKernel(T **sr_in_out,
                                           int64_t N,
                                           size_t rows) {
-  int id = blockIdx.x * blockDim.x + threadIdx.x;
-  while (id < N) {
+  CUDA_KERNEL_LOOP_TYPE(idx, N, int64_t) {
     for (int i = 0; i < 2 * rows; i += 2) {
       const T *tmp = sr_in_out[i];
       T *tmp_out = sr_in_out[i + 1];
       if (tmp && tmp_out) {
-        tmp_out[id] += tmp[id];
+        tmp_out[idx] += tmp[idx];
       }
     }
-    id += blockDim.x * gridDim.x;
   }
 }
 

From ce256f7532ca541d3d8195dfd7897c43faeb5fa3 Mon Sep 17 00:00:00 2001
From: PommesPeter <54879512+PommesPeter@users.noreply.github.com>
Date: Fri, 12 May 2023 22:32:09 +0800
Subject: [PATCH 313/405] =?UTF-8?q?=E3=80=90Hackathon=204=20No.20=E3=80=91?=
 =?UTF-8?q?Add=20i0=20/=20i0e=20to=20paddle=20(#52058)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* added base code for i0 and i0e

* added grad base code for i0 and i0e

* added i0 and i0e python code

* added ops and backward yaml config

* added i0 and i0e cpu kernel, but not test.

* added i0 and i0e code and unitest files

* added test files

* added i0/i0e gpu implementation code

* updated code style

* updated code style

* fixed unitests code

* updated i0 with eigen3

* fixed bug and added more test cases

* refactor: fixed static graph bug

* refactor: removed i0 and i0e from op_compat

* refactor: updated code style

* refactor: updated op_compat.yaml

* refactor: updated op_compat.yaml

* refactor: fixed op name mapping and optimize unittest case

* refactor: manually implement i0 / i0e

* refactor: added grad kernel for i0 / i0e,didn't finish

* Update math.py

* refactor: added equation to doc in English and added comments for computing i0 / i0e gradient

* refactor: removed eigen implementation

* refactor: finished i0 / i0e cpu and gpu op

* refactor: updated code style

* fix: find  a bug but not fix

* fix: incorrect unittest cases

* update: updated code style and remove my file

* update: updated unittest case

* fix: fixed sign error

* fix: fixed mistakes when merging

* refactor: updated code style

* refactor: remove unused code

* refactor: updated code style
---
 paddle/phi/api/yaml/backward.yaml             |  20 ++
 paddle/phi/api/yaml/ops.yaml                  |  18 ++
 paddle/phi/kernels/cpu/i0_grad_kernel.cc      |  42 +++
 paddle/phi/kernels/cpu/i0_kernel.cc           |  37 +++
 paddle/phi/kernels/cpu/i0e_grad_kernel.cc     |  44 +++
 paddle/phi/kernels/cpu/i0e_kernel.cc          |  37 +++
 paddle/phi/kernels/gpu/i0_grad_kernel.cu      |  39 +++
 paddle/phi/kernels/gpu/i0_kernel.cu           |  35 +++
 paddle/phi/kernels/gpu/i0e_grad_kernel.cu     |  40 +++
 paddle/phi/kernels/gpu/i0e_kernel.cu          |  35 +++
 paddle/phi/kernels/i0_grad_kernel.h           |  28 ++
 paddle/phi/kernels/i0_kernel.h                |  31 +++
 paddle/phi/kernels/i0e_grad_kernel.h          |  29 ++
 paddle/phi/kernels/i0e_kernel.h               |  32 +++
 .../impl/bessel_grad_kernel_cuda_impl.h       |  90 +++++++
 .../kernels/impl/bessel_grad_kernel_impl.h    | 112 ++++++++
 .../kernels/impl/bessel_kernel_cuda_impl.h    | 231 ++++++++++++++++
 paddle/phi/kernels/impl/bessel_kernel_impl.h  | 254 ++++++++++++++++++
 python/paddle/__init__.py                     |   4 +
 .../fluid/tests/unittests/test_i0_op.py       | 144 ++++++++++
 .../fluid/tests/unittests/test_i0e_op.py      | 148 ++++++++++
 python/paddle/tensor/__init__.py              |   4 +
 python/paddle/tensor/math.py                  |  73 +++++
 23 files changed, 1527 insertions(+)
 create mode 100644 paddle/phi/kernels/cpu/i0_grad_kernel.cc
 create mode 100644 paddle/phi/kernels/cpu/i0_kernel.cc
 create mode 100644 paddle/phi/kernels/cpu/i0e_grad_kernel.cc
 create mode 100644 paddle/phi/kernels/cpu/i0e_kernel.cc
 create mode 100644 paddle/phi/kernels/gpu/i0_grad_kernel.cu
 create mode 100644 paddle/phi/kernels/gpu/i0_kernel.cu
 create mode 100644 paddle/phi/kernels/gpu/i0e_grad_kernel.cu
 create mode 100644 paddle/phi/kernels/gpu/i0e_kernel.cu
 create mode 100644 paddle/phi/kernels/i0_grad_kernel.h
 create mode 100644 paddle/phi/kernels/i0_kernel.h
 create mode 100644 paddle/phi/kernels/i0e_grad_kernel.h
 create mode 100644 paddle/phi/kernels/i0e_kernel.h
 create mode 100644 paddle/phi/kernels/impl/bessel_grad_kernel_cuda_impl.h
 create mode 100644 paddle/phi/kernels/impl/bessel_grad_kernel_impl.h
 create mode 100644 paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h
 create mode 100644 paddle/phi/kernels/impl/bessel_kernel_impl.h
 create mode 100644 python/paddle/fluid/tests/unittests/test_i0_op.py
 create mode 100644 python/paddle/fluid/tests/unittests/test_i0e_op.py

diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index 27bdf904eee6e..e3ef0460884b1 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -775,6 +775,26 @@
   kernel :
     func : huber_loss_grad
 
+- backward_op : i0_grad
+  forward : i0 (Tensor x) -> Tensor(out)
+  args : (Tensor x, Tensor out_grad)
+  output : Tensor(x_grad)
+  infer_meta :
+    func : UnchangedInferMeta
+    param : [x]
+  kernel :
+    func : i0_grad
+
+- backward_op : i0e_grad
+  forward : i0e (Tensor x) -> Tensor(out)
+  args : (Tensor x, Tensor out, Tensor out_grad)
+  output : Tensor(x_grad)
+  infer_meta :
+    func : UnchangedInferMeta
+    param : [x]
+  kernel :
+    func : i0e_grad
+
 - backward_op : imag_grad
   forward : imag (Tensor x) -> Tensor(out)
   args : (Tensor out_grad)
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index de9d6c4f29c57..b8f99bbe106b0 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -920,6 +920,24 @@
   intermediate : residual
   backward : huber_loss_grad
 
+- op : i0
+  args : (Tensor x)
+  output : Tensor(out)
+  infer_meta :
+    func : UnchangedInferMeta
+  kernel :
+    func : i0
+  backward : i0_grad
+
+- op : i0e
+  args : (Tensor x)
+  output : Tensor(out)
+  infer_meta :
+    func : UnchangedInferMeta
+  kernel :
+    func : i0e
+  backward : i0e_grad
+
 - op : imag
   args : (Tensor x)
   output : Tensor (out)
diff --git a/paddle/phi/kernels/cpu/i0_grad_kernel.cc b/paddle/phi/kernels/cpu/i0_grad_kernel.cc
new file mode 100644
index 0000000000000..12e4ba437e157
--- /dev/null
+++ b/paddle/phi/kernels/cpu/i0_grad_kernel.cc
@@ -0,0 +1,42 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/i0_grad_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
+#include "paddle/phi/kernels/impl/bessel_grad_kernel_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I0GradKernel(const Context& ctx,
+                  const DenseTensor& x,
+                  const DenseTensor& out_grad,
+                  DenseTensor* x_grad) {
+  auto size = x.numel();
+  auto* x_data = x.data<T>();
+  auto* out_grad_data = out_grad.data<T>();
+  auto* x_grad_data = ctx.template Alloc<T>(x_grad);
+
+  phi::funcs::ForRange<Context> for_range(ctx, size);
+  I0GradFunctor<T> functor(x_data, out_grad_data, x_grad_data, size);
+  for_range(functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(i0_grad, CPU, ALL_LAYOUT, phi::I0GradKernel, float, double) {
+}
diff --git a/paddle/phi/kernels/cpu/i0_kernel.cc b/paddle/phi/kernels/cpu/i0_kernel.cc
new file mode 100644
index 0000000000000..3c6ba8f78aeb7
--- /dev/null
+++ b/paddle/phi/kernels/cpu/i0_kernel.cc
@@ -0,0 +1,37 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/i0_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
+#include "paddle/phi/kernels/impl/bessel_kernel_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I0Kernel(const Context& ctx, const DenseTensor& x, DenseTensor* out) {
+  const int64_t size = x.numel();
+  const T* x_data = x.data<T>();
+  T* out_data = ctx.template Alloc<T>(out);
+
+  phi::funcs::ForRange<Context> for_range(ctx, size);
+  I0Functor<T> functor(x_data, out_data, size);
+  for_range(functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(i0, CPU, ALL_LAYOUT, phi::I0Kernel, float, double) {}
diff --git a/paddle/phi/kernels/cpu/i0e_grad_kernel.cc b/paddle/phi/kernels/cpu/i0e_grad_kernel.cc
new file mode 100644
index 0000000000000..4521e0845857b
--- /dev/null
+++ b/paddle/phi/kernels/cpu/i0e_grad_kernel.cc
@@ -0,0 +1,44 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/i0e_grad_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
+#include "paddle/phi/kernels/impl/bessel_grad_kernel_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I0eGradKernel(const Context& ctx,
+                   const DenseTensor& x,
+                   const DenseTensor& out,
+                   const DenseTensor& out_grad,
+                   DenseTensor* x_grad) {
+  auto size = x.numel();
+  auto* x_data = x.data<T>();
+  auto* out_data = out.data<T>();
+  auto* out_grad_data = out_grad.data<T>();
+  auto* x_gard_data = ctx.template Alloc<T>(x_grad);
+
+  phi::funcs::ForRange<Context> for_range(ctx, size);
+  I0eGradFunctor<T> functor(x_data, out_data, out_grad_data, x_gard_data, size);
+  for_range(functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(
+    i0e_grad, CPU, ALL_LAYOUT, phi::I0eGradKernel, float, double) {}
diff --git a/paddle/phi/kernels/cpu/i0e_kernel.cc b/paddle/phi/kernels/cpu/i0e_kernel.cc
new file mode 100644
index 0000000000000..c3a9ff7780d28
--- /dev/null
+++ b/paddle/phi/kernels/cpu/i0e_kernel.cc
@@ -0,0 +1,37 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/i0e_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
+#include "paddle/phi/kernels/impl/bessel_kernel_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I0eKernel(const Context& ctx, const DenseTensor& x, DenseTensor* out) {
+  int64_t size = x.numel();
+  const T* x_data = x.data<T>();
+  T* out_data = ctx.template Alloc<T>(out);
+
+  phi::funcs::ForRange<Context> for_range(ctx, size);
+  I0eFunctor<T> functor(x_data, out_data, size);
+  for_range(functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(i0e, CPU, ALL_LAYOUT, phi::I0eKernel, float, double) {}
diff --git a/paddle/phi/kernels/gpu/i0_grad_kernel.cu b/paddle/phi/kernels/gpu/i0_grad_kernel.cu
new file mode 100644
index 0000000000000..9883a7db054b8
--- /dev/null
+++ b/paddle/phi/kernels/gpu/i0_grad_kernel.cu
@@ -0,0 +1,39 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/i0_grad_kernel.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/elementwise_base.h"
+#include "paddle/phi/kernels/impl/bessel_grad_kernel_cuda_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I0GradKernel(const Context& ctx,
+                  const DenseTensor& x,
+                  const DenseTensor& out_grad,
+                  DenseTensor* x_grad) {
+  ctx.template Alloc<T>(x_grad);
+  std::vector<const DenseTensor*> ins = {&x, &out_grad};
+  std::vector<DenseTensor*> outs = {x_grad};
+  auto functor = CudaI0GradFunctor<T>();
+  phi::funcs::ElementwiseKernel<T>(ctx, ins, &outs, functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(i0_grad, GPU, ALL_LAYOUT, phi::I0GradKernel, float, double) {
+}
diff --git a/paddle/phi/kernels/gpu/i0_kernel.cu b/paddle/phi/kernels/gpu/i0_kernel.cu
new file mode 100644
index 0000000000000..e8641933c37e7
--- /dev/null
+++ b/paddle/phi/kernels/gpu/i0_kernel.cu
@@ -0,0 +1,35 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/i0_kernel.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/elementwise_base.h"
+#include "paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I0Kernel(const Context& ctx, const DenseTensor& x, DenseTensor* out) {
+  ctx.template Alloc<T>(out);
+  std::vector<const DenseTensor*> ins = {&x};
+  std::vector<DenseTensor*> outs = {out};
+  auto functor = CudaI0Functor<T>();
+  phi::funcs::ElementwiseKernel<T>(ctx, ins, &outs, functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(i0, GPU, ALL_LAYOUT, phi::I0Kernel, float, double) {}
diff --git a/paddle/phi/kernels/gpu/i0e_grad_kernel.cu b/paddle/phi/kernels/gpu/i0e_grad_kernel.cu
new file mode 100644
index 0000000000000..1f50de93ce2c0
--- /dev/null
+++ b/paddle/phi/kernels/gpu/i0e_grad_kernel.cu
@@ -0,0 +1,40 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/i0e_grad_kernel.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/elementwise_base.h"
+#include "paddle/phi/kernels/impl/bessel_grad_kernel_cuda_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I0eGradKernel(const Context& ctx,
+                   const DenseTensor& x,
+                   const DenseTensor& out,
+                   const DenseTensor& out_grad,
+                   DenseTensor* x_grad) {
+  ctx.template Alloc<T>(x_grad);
+  std::vector<const DenseTensor*> ins = {&x, &out, &out_grad};
+  std::vector<DenseTensor*> outs = {x_grad};
+  auto functor = CudaI0eGradFunctor<T>();
+  phi::funcs::ElementwiseKernel<T>(ctx, ins, &outs, functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(
+    i0e_grad, GPU, ALL_LAYOUT, phi::I0eGradKernel, float, double) {}
diff --git a/paddle/phi/kernels/gpu/i0e_kernel.cu b/paddle/phi/kernels/gpu/i0e_kernel.cu
new file mode 100644
index 0000000000000..06f8758ef3902
--- /dev/null
+++ b/paddle/phi/kernels/gpu/i0e_kernel.cu
@@ -0,0 +1,35 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/i0e_kernel.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/elementwise_base.h"
+#include "paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I0eKernel(const Context& ctx, const DenseTensor& x, DenseTensor* out) {
+  ctx.template Alloc<T>(out);
+  std::vector<const DenseTensor*> ins = {&x};
+  std::vector<DenseTensor*> outs = {out};
+  auto functor = CudaI0eFunctor<T>();
+  phi::funcs::ElementwiseKernel<T>(ctx, ins, &outs, functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(i0e, GPU, ALL_LAYOUT, phi::I0eKernel, float, double) {}
diff --git a/paddle/phi/kernels/i0_grad_kernel.h b/paddle/phi/kernels/i0_grad_kernel.h
new file mode 100644
index 0000000000000..618e9336a4133
--- /dev/null
+++ b/paddle/phi/kernels/i0_grad_kernel.h
@@ -0,0 +1,28 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/device_context.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I0GradKernel(const Context& ctx,
+                  const DenseTensor& x,
+                  const DenseTensor& out_grad,
+                  DenseTensor* x_grad);
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/i0_kernel.h b/paddle/phi/kernels/i0_kernel.h
new file mode 100644
index 0000000000000..6378965d0043c
--- /dev/null
+++ b/paddle/phi/kernels/i0_kernel.h
@@ -0,0 +1,31 @@
+// Copyright (c) 2023PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+
+/**
+ * @brief This kernel calculate Modified Bessel function of order 0.
+ * @param  ctx   device context
+ * @param  x     The input tensor of i0
+ * @param  out   The output tensor of i0 kernel, it has the same shape and
+ *               dtype with input. Each element corresponds to input tensor
+ */
+template <typename T, typename Context>
+void I0Kernel(const Context& ctx, const DenseTensor& x, DenseTensor* out);
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/i0e_grad_kernel.h b/paddle/phi/kernels/i0e_grad_kernel.h
new file mode 100644
index 0000000000000..78a80cc505934
--- /dev/null
+++ b/paddle/phi/kernels/i0e_grad_kernel.h
@@ -0,0 +1,29 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/device_context.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I0eGradKernel(const Context& ctx,
+                   const DenseTensor& x,
+                   const DenseTensor& out,
+                   const DenseTensor& out_grad,
+                   DenseTensor* x_grad);
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/i0e_kernel.h b/paddle/phi/kernels/i0e_kernel.h
new file mode 100644
index 0000000000000..db298dea83ae3
--- /dev/null
+++ b/paddle/phi/kernels/i0e_kernel.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2023PaddlePaddle Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+
+/**
+ * @brief This kernel calculate Exponentially scaled modified Bessel function of
+ * order 0..
+ * @param  ctx   device context
+ * @param  x     The input tensor of i0e
+ * @param  out   The output tensor of i0e kernel, it has the same shape and
+ *               dtype with input. Each element corresponds to input tensor
+ */
+template <typename T, typename Context>
+void I0eKernel(const Context& ctx, const DenseTensor& x, DenseTensor* out);
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/impl/bessel_grad_kernel_cuda_impl.h b/paddle/phi/kernels/impl/bessel_grad_kernel_cuda_impl.h
new file mode 100644
index 0000000000000..7789e3d8e9312
--- /dev/null
+++ b/paddle/phi/kernels/impl/bessel_grad_kernel_cuda_impl.h
@@ -0,0 +1,90 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h"
+
+namespace phi {
+
+template <typename T>
+struct CudaI0GradFunctor {
+  __device__ __forceinline__ T operator()(const T _x, const T _out_grad) const {
+    using MT = typename phi::dtype::MPTypeTrait<T>::Type;
+    const MT mp_x = static_cast<MT>(_x);
+    const MT mp_out_grad = static_cast<MT>(_out_grad);
+    // get ouput of i1
+    MT x = std::abs(mp_x);
+    if (x <= MT{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI1e_A<MT>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      MT y = (x / MT{2.0}) - MT{2.0};
+
+      const MT i1_out = std::exp(x) * x * Chbevl<MT>(y, A, len);
+      const MT i1_data = (mp_x < MT{0.0}) ? -i1_out : i1_out;
+      // calculate i0 gradient
+      return static_cast<T>(i1_data * mp_out_grad);
+    }
+    auto coeff_pair_B = ChebyshevCoefficientsI1e_B<MT>();
+    auto B = std::get<0>(coeff_pair_B);
+    auto len = std::get<1>(coeff_pair_B);
+    MT y = (MT{32.0} / x) - MT{2.0};
+
+    const MT i1_out = (std::exp(x) * Chbevl<MT>(y, B, len)) / std::sqrt(x);
+    const MT i1_data = (mp_x < MT{0.0}) ? -i1_out : i1_out;
+
+    return static_cast<T>(i1_data * mp_out_grad);
+  }
+};
+
+template <typename T>
+struct CudaI0eGradFunctor {
+  __device__ __forceinline__ T operator()(const T _x,
+                                          const T _out,
+                                          const T _out_grad) const {
+    using MT = typename phi::dtype::MPTypeTrait<T>::Type;
+    const MT mp_x = static_cast<MT>(_x);
+    const MT mp_out = static_cast<MT>(_out);
+    const MT mp_out_grad = static_cast<MT>(_out_grad);
+    // get output of i1e
+    MT x = std::abs(mp_x);
+    if (x <= MT{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI1e_A<MT>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      MT y = (x / MT{2.0}) - MT{2.0};
+
+      const MT i1e_out = Chbevl<MT>(y, A, len) * x;
+      const MT i1e_data = (mp_x < MT{0.0}) ? -i1e_out : i1e_out;
+      // calculate i0e gradient
+      return static_cast<T>((i1e_data - std::copysign(MT{1.0}, mp_x) * mp_out) *
+                            mp_out_grad);
+    }
+    auto coeff_pair_B = ChebyshevCoefficientsI1e_B<MT>();
+    auto B = std::get<0>(coeff_pair_B);
+    auto len = std::get<1>(coeff_pair_B);
+    MT y = (MT{32.0} / x) - MT{2.0};
+
+    const MT i1e_out = Chbevl<MT>(y, B, len) / std::sqrt(x);
+    const MT i1e_data = (mp_x < MT{0.0}) ? -i1e_out : i1e_out;
+
+    return static_cast<T>((i1e_data - std::copysign(MT{1.0}, mp_x) * mp_out) *
+                          mp_out_grad);
+  }
+};
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/impl/bessel_grad_kernel_impl.h b/paddle/phi/kernels/impl/bessel_grad_kernel_impl.h
new file mode 100644
index 0000000000000..1de9a9fcbfc4b
--- /dev/null
+++ b/paddle/phi/kernels/impl/bessel_grad_kernel_impl.h
@@ -0,0 +1,112 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/phi/backends/all_context.h"
+#include "paddle/phi/common/amp_type_traits.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
+#include "paddle/phi/kernels/impl/bessel_kernel_impl.h"
+
+namespace phi {
+
+template <typename T>
+struct I0GradFunctor {
+  I0GradFunctor(const T* x, const T* out_grad, T* x_grad, int64_t numel)
+      : inp_x_(x),
+        inp_out_grad_(out_grad),
+        output_x_grad_(x_grad),
+        numel_(numel) {}
+
+  HOSTDEVICE void operator()(int64_t idx) const {
+    using MT = typename phi::dtype::MPTypeTrait<T>::Type;
+    const MT mp_x = static_cast<MT>(inp_x_[idx]);
+    const MT mp_out_grad = static_cast<MT>(inp_out_grad_[idx]);
+
+    MT x = std::abs(mp_x);
+    if (x <= T{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI1e_A<MT>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      MT y = (x / MT{2.0}) - MT{2.0};
+
+      const MT i1_out = std::exp(x) * x * Chbevl<MT>(y, A, len);
+      const MT i1_data = (mp_x < T{0.0}) ? -i1_out : i1_out;
+      output_x_grad_[idx] = static_cast<T>(i1_data * mp_out_grad);
+    } else {
+      auto coeff_pair_B = ChebyshevCoefficientsI1e_B<MT>();
+      auto B = std::get<0>(coeff_pair_B);
+      auto len = std::get<1>(coeff_pair_B);
+      MT y = (MT{32.0} / x) - MT{2.0};
+
+      const MT i1_out = (std::exp(x) * Chbevl<MT>(y, B, len)) / std::sqrt(x);
+      const MT i1_data = (mp_x < MT{0.0}) ? -i1_out : i1_out;
+      output_x_grad_[idx] = static_cast<T>(i1_data * mp_out_grad);
+    }
+  }
+
+ private:
+  const T* inp_x_;
+  const T* inp_out_grad_;
+  T* output_x_grad_;
+  int64_t numel_;
+};
+
+template <typename T>
+struct I0eGradFunctor {
+  I0eGradFunctor(
+      const T* x, const T* out, const T* out_grad, T* x_grad, int64_t numel)
+      : inp_x_(x),
+        inp_out_(out),
+        inp_out_grad_(out_grad),
+        output_x_grad_(x_grad),
+        numel_(numel) {}
+
+  HOSTDEVICE void operator()(int64_t idx) const {
+    T x = std::abs(inp_x_[idx]);
+    if (x <= T{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI1e_A<T>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      T y = (x / T{2.0}) - T{2.0};
+
+      const T out = Chbevl<T>(y, A, len) * x;
+      const T i1e_out = (inp_x_[idx] < T{0.0}) ? -out : out;
+      output_x_grad_[idx] =
+          (i1e_out - std::copysign(T{1.0}, inp_x_[idx]) * inp_out_[idx]) *
+          inp_out_grad_[idx];
+    } else {
+      auto coeff_pair_B = ChebyshevCoefficientsI1e_B<T>();
+      auto B = std::get<0>(coeff_pair_B);
+      auto len = std::get<1>(coeff_pair_B);
+      T y = (T{32.0} / x) - T{2.0};
+
+      const T out = Chbevl<T>(y, B, len) / std::sqrt(x);
+      const T i1e_out = (inp_x_[idx] < T{0.0}) ? -out : out;
+      output_x_grad_[idx] =
+          (i1e_out - std::copysign(T{1.0}, inp_x_[idx]) * inp_out_[idx]) *
+          inp_out_grad_[idx];
+    }
+  }
+
+ private:
+  const T* inp_x_;
+  const T* inp_out_;
+  const T* inp_out_grad_;
+  T* output_x_grad_;
+  int64_t numel_;
+};
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h b/paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h
new file mode 100644
index 0000000000000..32649c901906b
--- /dev/null
+++ b/paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h
@@ -0,0 +1,231 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+
+template <typename T>
+__host__ __device__ std::tuple<const T*, size_t> ChebyshevCoefficientsI0e_A() {
+  /* Chebyshev coefficients for I0e(x) in the interval [0,8]. */
+  static const T coeff[] = {
+      -4.41534164647933937950E-18, 3.33079451882223809783E-17,
+      -2.43127984654795469359E-16, 1.71539128555513303061E-15,
+      -1.16853328779934516808E-14, 7.67618549860493561688E-14,
+      -4.85644678311192946090E-13, 2.95505266312963983461E-12,
+      -1.72682629144155570723E-11, 9.67580903537323691224E-11,
+      -5.18979560163526290666E-10, 2.65982372468238665035E-9,
+      -1.30002500998624804212E-8,  6.04699502254191894932E-8,
+      -2.67079385394061173391E-7,  1.11738753912010371815E-6,
+      -4.41673835845875056359E-6,  1.64484480707288970893E-5,
+      -5.75419501008210370398E-5,  1.88502885095841655729E-4,
+      -5.76375574538582365885E-4,  1.63947561694133579842E-3,
+      -4.32430999505057594430E-3,  1.05464603945949983183E-2,
+      -2.37374148058994688156E-2,  4.93052842396707084878E-2,
+      -9.49010970480476444210E-2,  1.71620901522208775349E-1,
+      -3.04682672343198398683E-1,  6.76795274409476084995E-1};
+  return std::make_tuple(coeff, 30);
+}
+
+template <typename T>
+__host__ __device__ std::tuple<const T*, size_t> ChebyshevCoefficientsI0e_B() {
+  /* Chebyshev coefficients for I0e(x) in the inverted interval [8,infinity]. */
+  static const T coeff[] = {
+      -7.23318048787475395456E-18, -4.83050448594418207126E-18,
+      4.46562142029675999901E-17,  3.46122286769746109310E-17,
+      -2.82762398051658348494E-16, -3.42548561967721913462E-16,
+      1.77256013305652638360E-15,  3.81168066935262242075E-15,
+      -9.55484669882830764870E-15, -4.15056934728722208663E-14,
+      1.54008621752140982691E-14,  3.85277838274214270114E-13,
+      7.18012445138366623367E-13,  -1.79417853150680611778E-12,
+      -1.32158118404477131188E-11, -3.14991652796324136454E-11,
+      1.18891471078464383424E-11,  4.94060238822496958910E-10,
+      3.39623202570838634515E-9,   2.26666899049817806459E-8,
+      2.04891858946906374183E-7,   2.89137052083475648297E-6,
+      6.88975834691682398426E-5,   3.36911647825569408990E-3,
+      8.04490411014108831608E-1};
+
+  return std::make_tuple(coeff, 25);
+}
+
+template <typename T>
+__host__ __device__ T Chbevl(T x, const T array[], size_t len) {
+  T b0, b1, b2;
+
+  b0 = array[0];
+  b1 = static_cast<T>(0.0);
+
+  for (size_t i = 1; i < len; ++i) {
+    b2 = b1;
+    b1 = b0;
+    b0 = x * b1 - b2 + array[i];
+  }
+
+  return (static_cast<T>(0.5) * (b0 - b2));
+}
+
+template <typename T>
+struct CudaI0Functor {
+  __device__ __forceinline__ T operator()(const T _x) const {
+    using MT = typename phi::dtype::MPTypeTrait<T>::Type;
+    const MT mp_x = static_cast<MT>(_x);
+    MT x = std::abs(mp_x);
+    if (x <= MT{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI0e_A<MT>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      MT y = (x / MT{2.0}) - MT{2.0};
+
+      return static_cast<T>(std::exp(x) * Chbevl<MT>(y, A, len));
+    }
+    auto coeff_pair_B = ChebyshevCoefficientsI0e_B<MT>();
+    auto B = std::get<0>(coeff_pair_B);
+    auto len = std::get<1>(coeff_pair_B);
+    MT y = (MT{32.0} / x) - MT{2.0};
+
+    return static_cast<T>(std::exp(x) * Chbevl<T>(y, B, len) / std::sqrt(x));
+  }
+};
+
+template <typename T>
+struct CudaI0eFunctor {
+  __device__ __forceinline__ T operator()(const T _x) const {
+    using MT = typename phi::dtype::MPTypeTrait<T>::Type;
+    const MT mp_x = static_cast<MT>(_x);
+    MT x = std::abs(mp_x);
+    if (x <= MT{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI0e_A<MT>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      MT y = (x / MT{2.0}) - MT{2.0};
+
+      return static_cast<T>(Chbevl<MT>(y, A, len));
+    }
+    auto coeff_pair_B = ChebyshevCoefficientsI0e_B<MT>();
+    auto B = std::get<0>(coeff_pair_B);
+    auto len = std::get<1>(coeff_pair_B);
+    MT y = (MT{32.0} / x) - MT{2.0};
+
+    return static_cast<T>(Chbevl<T>(y, B, len) / std::sqrt(x));
+  }
+};
+
+template <typename T>
+__host__ __device__ typename std::enable_if<std::is_same<double, T>::value,
+                                            std::tuple<const T*, size_t>>::type
+ChebyshevCoefficientsI1e_A() {
+  /* Chebyshev coefficients for exp(-x) I1(x)
+   * in the interval [0,8].
+   *
+   * lim(x->0){ exp(-x) I1(x) / x } = 1/2.
+   */
+  static const T coeff[] = {
+      2.77791411276104639959E-18, -2.11142121435816608115E-17,
+      1.55363195773620046921E-16, -1.10559694773538630805E-15,
+      7.60068429473540693410E-15, -5.04218550472791168711E-14,
+      3.22379336594557470981E-13, -1.98397439776494371520E-12,
+      1.17361862988909016308E-11, -6.66348972350202774223E-11,
+      3.62559028155211703701E-10, -1.88724975172282928790E-9,
+      9.38153738649577178388E-9,  -4.44505912879632808065E-8,
+      2.00329475355213526229E-7,  -8.56872026469545474066E-7,
+      3.47025130813767847674E-6,  -1.32731636560394358279E-5,
+      4.78156510755005422638E-5,  -1.61760815825896745588E-4,
+      5.12285956168575772895E-4,  -1.51357245063125314899E-3,
+      4.15642294431288815669E-3,  -1.05640848946261981558E-2,
+      2.47264490306265168283E-2,  -5.29459812080949914269E-2,
+      1.02643658689847095384E-1,  -1.76416518357834055153E-1,
+      2.52587186443633654823E-1};
+  return std::make_tuple(coeff, 29);
+}
+
+template <typename T>
+__host__ __device__ typename std::enable_if<std::is_same<float, T>::value,
+                                            std::tuple<const T*, size_t>>::type
+ChebyshevCoefficientsI1e_A() {
+  /* Chebyshev coefficients for exp(-x) I1(x)
+   * in the interval [0,8].
+   *
+   * lim(x->0){ exp(-x) I1(x) / x } = 1/2.
+   */
+  static const T coeff[] = {9.38153738649577178388E-9f,
+                            -4.44505912879632808065E-8f,
+                            2.00329475355213526229E-7f,
+                            -8.56872026469545474066E-7f,
+                            3.47025130813767847674E-6f,
+                            -1.32731636560394358279E-5f,
+                            4.78156510755005422638E-5f,
+                            -1.61760815825896745588E-4f,
+                            5.12285956168575772895E-4f,
+                            -1.51357245063125314899E-3f,
+                            4.15642294431288815669E-3f,
+                            -1.05640848946261981558E-2f,
+                            2.47264490306265168283E-2f,
+                            -5.29459812080949914269E-2f,
+                            1.02643658689847095384E-1f,
+                            -1.76416518357834055153E-1f,
+                            2.52587186443633654823E-1f};
+  return std::make_tuple(coeff, 17);
+}
+
+template <typename T>
+__host__ __device__ typename std::enable_if<std::is_same<double, T>::value,
+                                            std::tuple<const T*, size_t>>::type
+ChebyshevCoefficientsI1e_B() {
+  /* Chebyshev coefficients for exp(-x) sqrt(x) I1(x)
+   * in the inverted interval [8,infinity].
+   *
+   * lim(x->inf){ exp(-x) sqrt(x) I1(x) } = 1/sqrt(2pi).
+   */
+  static const T coeff[] = {
+      7.51729631084210481353E-18,  4.41434832307170791151E-18,
+      -4.65030536848935832153E-17, -3.20952592199342395980E-17,
+      2.96262899764595013876E-16,  3.30820231092092828324E-16,
+      -1.88035477551078244854E-15, -3.81440307243700780478E-15,
+      1.04202769841288027642E-14,  4.27244001671195135429E-14,
+      -2.10154184277266431302E-14, -4.08355111109219731823E-13,
+      -7.19855177624590851209E-13, 2.03562854414708950722E-12,
+      1.41258074366137813316E-11,  3.25260358301548823856E-11,
+      -1.89749581235054123450E-11, -5.58974346219658380687E-10,
+      -3.83538038596423702205E-9,  -2.63146884688951950684E-8,
+      -2.51223623787020892529E-7,  -3.88256480887769039346E-6,
+      -1.10588938762623716291E-4,  -9.76109749136146840777E-3,
+      7.78576235018280120474E-1};
+
+  return std::make_tuple(coeff, 25);
+}
+
+template <typename T>
+__host__ __device__ typename std::enable_if<std::is_same<float, T>::value,
+                                            std::tuple<const T*, size_t>>::type
+ChebyshevCoefficientsI1e_B() {
+  /* Chebyshev coefficients for exp(-x) sqrt(x) I1(x)
+   * in the inverted interval [8,infinity].
+   *
+   * lim(x->inf){ exp(-x) sqrt(x) I1(x) } = 1/sqrt(2pi).
+   */
+  static const T coeff[] = {-3.83538038596423702205E-9f,
+                            -2.63146884688951950684E-8f,
+                            -2.51223623787020892529E-7f,
+                            -3.88256480887769039346E-6f,
+                            -1.10588938762623716291E-4f,
+                            -9.76109749136146840777E-3f,
+                            7.78576235018280120474E-1f};
+
+  return std::make_tuple(coeff, 7);
+}
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/impl/bessel_kernel_impl.h b/paddle/phi/kernels/impl/bessel_kernel_impl.h
new file mode 100644
index 0000000000000..1df090753930d
--- /dev/null
+++ b/paddle/phi/kernels/impl/bessel_kernel_impl.h
@@ -0,0 +1,254 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/phi/backends/all_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+
+template <typename T>
+static inline std::tuple<const T*, size_t> ChebyshevCoefficientsI0e_A() {
+  /* Chebyshev coefficients for exp(-x) I0(x)
+   * in the interval [0,8].
+   *
+   * lim(x->0) { exp(-x) I0(x) } = 1.
+   */
+  static const T coeff[] = {
+      -4.41534164647933937950E-18, 3.33079451882223809783E-17,
+      -2.43127984654795469359E-16, 1.71539128555513303061E-15,
+      -1.16853328779934516808E-14, 7.67618549860493561688E-14,
+      -4.85644678311192946090E-13, 2.95505266312963983461E-12,
+      -1.72682629144155570723E-11, 9.67580903537323691224E-11,
+      -5.18979560163526290666E-10, 2.65982372468238665035E-9,
+      -1.30002500998624804212E-8,  6.04699502254191894932E-8,
+      -2.67079385394061173391E-7,  1.11738753912010371815E-6,
+      -4.41673835845875056359E-6,  1.64484480707288970893E-5,
+      -5.75419501008210370398E-5,  1.88502885095841655729E-4,
+      -5.76375574538582365885E-4,  1.63947561694133579842E-3,
+      -4.32430999505057594430E-3,  1.05464603945949983183E-2,
+      -2.37374148058994688156E-2,  4.93052842396707084878E-2,
+      -9.49010970480476444210E-2,  1.71620901522208775349E-1,
+      -3.04682672343198398683E-1,  6.76795274409476084995E-1};
+  return std::make_tuple(coeff, 30);
+}
+
+template <typename T>
+static inline std::tuple<const T*, size_t> ChebyshevCoefficientsI0e_B() {
+  /* Chebyshev coefficients for exp(-x) sqrt(x) I0(x)
+   * in the inverted interval [8,infinity].
+   *
+   * lim(x->inf){ exp(-x) sqrt(x) I0(x) } = 1/sqrt(2pi).
+   */
+  static const T coeff[] = {
+      -7.23318048787475395456E-18, -4.83050448594418207126E-18,
+      4.46562142029675999901E-17,  3.46122286769746109310E-17,
+      -2.82762398051658348494E-16, -3.42548561967721913462E-16,
+      1.77256013305652638360E-15,  3.81168066935262242075E-15,
+      -9.55484669882830764870E-15, -4.15056934728722208663E-14,
+      1.54008621752140982691E-14,  3.85277838274214270114E-13,
+      7.18012445138366623367E-13,  -1.79417853150680611778E-12,
+      -1.32158118404477131188E-11, -3.14991652796324136454E-11,
+      1.18891471078464383424E-11,  4.94060238822496958910E-10,
+      3.39623202570838634515E-9,   2.26666899049817806459E-8,
+      2.04891858946906374183E-7,   2.89137052083475648297E-6,
+      6.88975834691682398426E-5,   3.36911647825569408990E-3,
+      8.04490411014108831608E-1};
+
+  return std::make_tuple(coeff, 25);
+}
+
+template <typename T>
+static inline T Chbevl(T x, const T array[], size_t len) {
+  T b0, b1, b2;
+
+  b0 = array[0];
+  b1 = static_cast<T>(0.0);
+
+  for (size_t i = 1; i < len; ++i) {
+    b2 = b1;
+    b1 = b0;
+    b0 = x * b1 - b2 + array[i];
+  }
+
+  return (static_cast<T>(0.5) * (b0 - b2));
+}
+
+template <typename T>
+struct I0eFunctor {
+  I0eFunctor(const T* input, T* output, int64_t numel)
+      : input_(input), output_(output), numel_(numel) {}
+
+  HOSTDEVICE void operator()(int64_t idx) const {
+    T x = std::abs(input_[idx]);
+    if (x <= T{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI0e_A<T>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      T y = (x / T{2.0}) - T{2.0};
+
+      output_[idx] = static_cast<T>(Chbevl<T>(y, A, len));
+    } else {
+      auto coeff_pair_B = ChebyshevCoefficientsI0e_B<T>();
+      auto B = std::get<0>(coeff_pair_B);
+      auto len = std::get<1>(coeff_pair_B);
+      T y = (T{32.0} / x) - T{2.0};
+
+      output_[idx] = static_cast<T>(Chbevl<T>(y, B, len) / std::sqrt(x));
+    }
+  }
+
+ private:
+  const T* input_;
+  T* output_;
+  int64_t numel_;
+};
+
+template <typename T>
+struct I0Functor {
+  I0Functor(const T* input, T* output, int64_t numel)
+      : input_(input), output_(output), numel_(numel) {}
+
+  HOSTDEVICE void operator()(int64_t idx) const {
+    T x = std::abs(input_[idx]);
+    if (x <= T{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI0e_A<T>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      T y = (x / T{2.0}) - T{2.0};
+
+      output_[idx] = static_cast<T>(std::exp(x) * Chbevl<T>(y, A, len));
+    } else {
+      auto coeff_pair_B = ChebyshevCoefficientsI0e_B<T>();
+      auto B = std::get<0>(coeff_pair_B);
+      auto len = std::get<1>(coeff_pair_B);
+      T y = (T{32.0} / x) - T{2.0};
+
+      output_[idx] =
+          static_cast<T>(std::exp(x) * Chbevl<T>(y, B, len) / std::sqrt(x));
+    }
+  }
+
+ private:
+  const T* input_;
+  T* output_;
+  int64_t numel_;
+};
+
+template <typename T>
+static inline typename std::enable_if<std::is_same<double, T>::value,
+                                      std::tuple<const T*, size_t>>::type
+ChebyshevCoefficientsI1e_A() {
+  /* Chebyshev coefficients for exp(-x) I1(x)
+   * in the interval [0,8].
+   *
+   * lim(x->0){ exp(-x) I1(x) / x } = 1/2.
+   */
+  static const T coeff[] = {
+      2.77791411276104639959E-18, -2.11142121435816608115E-17,
+      1.55363195773620046921E-16, -1.10559694773538630805E-15,
+      7.60068429473540693410E-15, -5.04218550472791168711E-14,
+      3.22379336594557470981E-13, -1.98397439776494371520E-12,
+      1.17361862988909016308E-11, -6.66348972350202774223E-11,
+      3.62559028155211703701E-10, -1.88724975172282928790E-9,
+      9.38153738649577178388E-9,  -4.44505912879632808065E-8,
+      2.00329475355213526229E-7,  -8.56872026469545474066E-7,
+      3.47025130813767847674E-6,  -1.32731636560394358279E-5,
+      4.78156510755005422638E-5,  -1.61760815825896745588E-4,
+      5.12285956168575772895E-4,  -1.51357245063125314899E-3,
+      4.15642294431288815669E-3,  -1.05640848946261981558E-2,
+      2.47264490306265168283E-2,  -5.29459812080949914269E-2,
+      1.02643658689847095384E-1,  -1.76416518357834055153E-1,
+      2.52587186443633654823E-1};
+  return std::make_tuple(coeff, 29);
+}
+
+template <typename T>
+static inline typename std::enable_if<std::is_same<float, T>::value,
+                                      std::tuple<const T*, size_t>>::type
+ChebyshevCoefficientsI1e_A() {
+  /* Chebyshev coefficients for exp(-x) I1(x)
+   * in the interval [0,8].
+   *
+   * lim(x->0){ exp(-x) I1(x) / x } = 1/2.
+   */
+  static const T coeff[] = {9.38153738649577178388E-9f,
+                            -4.44505912879632808065E-8f,
+                            2.00329475355213526229E-7f,
+                            -8.56872026469545474066E-7f,
+                            3.47025130813767847674E-6f,
+                            -1.32731636560394358279E-5f,
+                            4.78156510755005422638E-5f,
+                            -1.61760815825896745588E-4f,
+                            5.12285956168575772895E-4f,
+                            -1.51357245063125314899E-3f,
+                            4.15642294431288815669E-3f,
+                            -1.05640848946261981558E-2f,
+                            2.47264490306265168283E-2f,
+                            -5.29459812080949914269E-2f,
+                            1.02643658689847095384E-1f,
+                            -1.76416518357834055153E-1f,
+                            2.52587186443633654823E-1f};
+  return std::make_tuple(coeff, 17);
+}
+
+template <typename T>
+static inline typename std::enable_if<std::is_same<double, T>::value,
+                                      std::tuple<const T*, size_t>>::type
+ChebyshevCoefficientsI1e_B() {
+  /* Chebyshev coefficients for exp(-x) sqrt(x) I1(x)
+   * in the inverted interval [8,infinity].
+   *
+   * lim(x->inf){ exp(-x) sqrt(x) I1(x) } = 1/sqrt(2pi).
+   */
+  static const T coeff[] = {
+      7.51729631084210481353E-18,  4.41434832307170791151E-18,
+      -4.65030536848935832153E-17, -3.20952592199342395980E-17,
+      2.96262899764595013876E-16,  3.30820231092092828324E-16,
+      -1.88035477551078244854E-15, -3.81440307243700780478E-15,
+      1.04202769841288027642E-14,  4.27244001671195135429E-14,
+      -2.10154184277266431302E-14, -4.08355111109219731823E-13,
+      -7.19855177624590851209E-13, 2.03562854414708950722E-12,
+      1.41258074366137813316E-11,  3.25260358301548823856E-11,
+      -1.89749581235054123450E-11, -5.58974346219658380687E-10,
+      -3.83538038596423702205E-9,  -2.63146884688951950684E-8,
+      -2.51223623787020892529E-7,  -3.88256480887769039346E-6,
+      -1.10588938762623716291E-4,  -9.76109749136146840777E-3,
+      7.78576235018280120474E-1};
+
+  return std::make_tuple(coeff, 25);
+}
+
+template <typename T>
+static inline typename std::enable_if<std::is_same<float, T>::value,
+                                      std::tuple<const T*, size_t>>::type
+ChebyshevCoefficientsI1e_B() {
+  /* Chebyshev coefficients for exp(-x) sqrt(x) I1(x)
+   * in the inverted interval [8,infinity].
+   *
+   * lim(x->inf){ exp(-x) sqrt(x) I1(x) } = 1/sqrt(2pi).
+   */
+  static const T coeff[] = {-3.83538038596423702205E-9f,
+                            -2.63146884688951950684E-8f,
+                            -2.51223623787020892529E-7f,
+                            -3.88256480887769039346E-6f,
+                            -1.10588938762623716291E-4f,
+                            -9.76109749136146840777E-3f,
+                            7.78576235018280120474E-1f};
+
+  return std::make_tuple(coeff, 7);
+}
+
+}  // namespace phi
diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py
index 8804cfff6de19..8626e5e9f37ff 100644
--- a/python/paddle/__init__.py
+++ b/python/paddle/__init__.py
@@ -304,6 +304,8 @@
 from .tensor.math import cumulative_trapezoid  # noqa: F401
 from .tensor.math import vander  # noqa: F401
 from .tensor.math import nextafter  # noqa: F401
+from .tensor.math import i0  # noqa: F401
+from .tensor.math import i0e  # noqa: F401
 
 from .tensor.random import bernoulli  # noqa: F401
 from .tensor.random import poisson  # noqa: F401
@@ -698,4 +700,6 @@
     'vander',
     'unflatten',
     'nextafter',
+    'i0',
+    'i0e',
 ]
diff --git a/python/paddle/fluid/tests/unittests/test_i0_op.py b/python/paddle/fluid/tests/unittests/test_i0_op.py
new file mode 100644
index 0000000000000..a30f67312c8d0
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_i0_op.py
@@ -0,0 +1,144 @@
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from eager_op_test import OpTest
+from scipy import special
+
+import paddle
+from paddle.fluid import core
+
+np.random.seed(100)
+paddle.seed(100)
+
+
+def output_i0(x):
+    return special.i0(x)
+
+
+def ref_i0_grad(x, dout):
+    gradx = special.i1(x)
+    return dout * gradx
+
+
+class TestI0API(unittest.TestCase):
+    DTYPE = "float64"
+    DATA = [0, 1, 2, 3, 4, 5]
+
+    def setUp(self):
+        self.x = np.array(self.DATA).astype(self.DTYPE)
+        self.place = [paddle.CPUPlace()]
+        if core.is_compiled_with_cuda():
+            self.place.append(paddle.CUDAPlace(0))
+
+    def test_api_static(self):
+        def run(place):
+            paddle.enable_static()
+            with paddle.static.program_guard(paddle.static.Program()):
+                x = paddle.static.data(
+                    name="x", shape=self.x.shape, dtype=self.DTYPE
+                )
+                out = paddle.i0(x)
+                exe = paddle.static.Executor(place)
+                res = exe.run(
+                    paddle.static.default_main_program(),
+                    feed={"x": self.x},
+                    fetch_list=[out],
+                )
+                out_ref = output_i0(self.x)
+                np.testing.assert_allclose(res[0], out_ref, rtol=1e-5)
+            paddle.disable_static()
+
+        for place in self.place:
+            run(place)
+
+    def test_api_dygraph(self):
+        def run(place):
+            paddle.disable_static(place)
+            x = paddle.to_tensor(self.x)
+            out = paddle.i0(x)
+
+            out_ref = output_i0(self.x)
+            np.testing.assert_allclose(out.numpy(), out_ref, rtol=1e-5)
+            paddle.enable_static()
+
+        for place in self.place:
+            run(place)
+
+    def test_empty_input_error(self):
+        for place in self.place:
+            paddle.disable_static(place)
+            x = None
+            self.assertRaises(ValueError, paddle.i0, x)
+            paddle.enable_static()
+
+
+class TestI0Float32Zero2EightCase(TestI0API):
+    DTYPE = "float32"
+    DATA = [0, 1, 2, 3, 4, 5, 6, 7, 8]
+
+
+class TestI0Float32OverEightCase(TestI0API):
+    DTYPE = "float32"
+    DATA = [9, 10, 11, 12]
+
+
+class TestI0Float64Zero2EightCase(TestI0API):
+    DTYPE = "float64"
+    DATA = [0, 1, 2, 3, 4, 5, 6, 7, 8]
+
+
+class TestI0Float64OverEightCase(TestI0API):
+    DTYPE = "float64"
+    DATA = [9, 10, 11, 12]
+
+
+class TestI0Op(OpTest):
+    def setUp(self) -> None:
+        self.op_type = "i0"
+        self.python_api = paddle.i0
+        self.init_config()
+        self.outputs = {"out": self.target}
+
+    def init_config(self):
+        self.dtype = np.float64
+        zero_case = np.zeros(1).astype(self.dtype)
+        rand_case = np.random.randn(100).astype(self.dtype)
+        one2eight_case = np.random.uniform(low=1, high=8, size=100).astype(
+            self.dtype
+        )
+        over_eight_case = np.random.uniform(low=9, high=15, size=100).astype(
+            self.dtype
+        )
+        self.case = np.concatenate(
+            [zero_case, rand_case, one2eight_case, over_eight_case]
+        )
+        self.inputs = {'x': self.case}
+        self.target = output_i0(self.inputs['x'])
+
+    def test_check_output(self):
+        self.check_output()
+
+    def test_check_grad(self):
+        self.check_grad(
+            ['x'],
+            'out',
+            user_defined_grads=[ref_i0_grad(self.case, 1 / self.case.size)],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/python/paddle/fluid/tests/unittests/test_i0e_op.py b/python/paddle/fluid/tests/unittests/test_i0e_op.py
new file mode 100644
index 0000000000000..0543d53effec8
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_i0e_op.py
@@ -0,0 +1,148 @@
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from eager_op_test import OpTest
+from scipy import special
+
+import paddle
+from paddle.fluid import core
+
+np.random.seed(100)
+paddle.seed(100)
+
+
+def output_i0e(x):
+    return special.i0e(x)
+
+
+def ref_i0e_grad(x, dout):
+    eps = np.finfo(x.dtype).eps
+    not_tiny = abs(x) > eps
+    safe_x = np.where(not_tiny, x, eps)
+    gradx = special.i1e(x) - np.sign(x) * output_i0e(safe_x)
+    gradx = np.where(not_tiny, gradx, -1.0)
+    return dout * gradx
+
+
+class TestI0eAPI(unittest.TestCase):
+    DTYPE = "float64"
+    DATA = [0, 1, 2, 3, 4, 5]
+
+    def setUp(self):
+        self.x = np.array(self.DATA).astype(self.DTYPE)
+        self.place = [paddle.CPUPlace()]
+        if core.is_compiled_with_cuda():
+            self.place.append(paddle.CUDAPlace(0))
+
+    def test_api_static(self):
+        def run(place):
+            paddle.enable_static()
+            with paddle.static.program_guard(paddle.static.Program()):
+                x = paddle.static.data(
+                    name="x", shape=self.x.shape, dtype=self.DTYPE
+                )
+                y = paddle.i0e(x)
+                exe = paddle.static.Executor(place)
+                res = exe.run(
+                    paddle.static.default_main_program(),
+                    feed={"x": self.x},
+                    fetch_list=[y],
+                )
+                out_ref = output_i0e(self.x)
+                np.testing.assert_allclose(out_ref, res[0], rtol=1e-5)
+            paddle.disable_static()
+
+        for place in self.place:
+            run(place)
+
+    def test_api_dygraph(self):
+        def run(place):
+            paddle.disable_static(place)
+            x = paddle.to_tensor(self.x)
+            out = paddle.i0e(x)
+
+            out_ref = output_i0e(self.x)
+            np.testing.assert_allclose(out_ref, out.numpy(), rtol=1e-5)
+            paddle.enable_static()
+
+        for place in self.place:
+            run(place)
+
+    def test_empty_input_error(self):
+        for place in self.place:
+            paddle.disable_static(place)
+            x = None
+            self.assertRaises(ValueError, paddle.i0e, x)
+            paddle.enable_static()
+
+
+class TestI0eFloat32Zero2EightCase(TestI0eAPI):
+    DTYPE = "float32"
+    DATA = [0, 1, 2, 3, 4, 5, 6, 7, 8]
+
+
+class TestI0eFloat32OverEightCase(TestI0eAPI):
+    DTYPE = "float32"
+    DATA = [9, 10, 11, 12]
+
+
+class TestI0eFloat64Zero2EightCase(TestI0eAPI):
+    DTYPE = "float64"
+    DATA = [0, 1, 2, 3, 4, 5, 6, 7, 8]
+
+
+class TestI0eFloat64OverEightCase(TestI0eAPI):
+    DTYPE = "float64"
+    DATA = [9, 10, 11, 12]
+
+
+class TestI0eOp(OpTest):
+    def setUp(self) -> None:
+        self.op_type = "i0e"
+        self.python_api = paddle.i0e
+        self.init_config()
+        self.outputs = {"out": self.target}
+
+    def init_config(self):
+        self.dtype = np.float64
+        zero_case = np.zeros(1).astype(self.dtype)
+        rand_case = np.random.randn(100).astype(self.dtype)
+        one2eight_case = np.random.uniform(low=1, high=8, size=100).astype(
+            self.dtype
+        )
+        over_eight_case = np.random.uniform(low=9, high=15, size=100).astype(
+            self.dtype
+        )
+        self.case = np.concatenate(
+            [zero_case, rand_case, one2eight_case, over_eight_case]
+        )
+        self.inputs = {'x': self.case}
+        self.target = output_i0e(self.inputs['x'])
+
+    def test_check_output(self):
+        self.check_output()
+
+    def test_check_grad(self):
+        self.check_grad(
+            ['x'],
+            'out',
+            user_defined_grads=[ref_i0e_grad(self.case, 1 / self.case.size)],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/python/paddle/tensor/__init__.py b/python/paddle/tensor/__init__.py
index a399fad5a8fb4..1e26e4ce56d13 100644
--- a/python/paddle/tensor/__init__.py
+++ b/python/paddle/tensor/__init__.py
@@ -256,6 +256,8 @@
 from .math import sigmoid_  # noqa: F401
 from .math import vander  # noqa: F401
 from .math import nextafter  # noqa: F401
+from .math import i0  # noqa: F401
+from .math import i0e  # noqa: F401
 
 from .random import multinomial  # noqa: F401
 from .random import standard_normal  # noqa: F401
@@ -550,6 +552,8 @@
     'vander',
     'nextafter',
     'unflatten',
+    'i0',
+    'i0e',
 ]
 
 # this list used in math_op_patch.py for magic_method bind
diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py
index 40163ddf5d9d6..cbe0474960de2 100644
--- a/python/paddle/tensor/math.py
+++ b/python/paddle/tensor/math.py
@@ -5584,3 +5584,76 @@ def nextafter(x, y, name=None):
         outputs = {"out": out}
         helper.append_op(type=op_type, inputs=inputs, outputs=outputs)
     return out
+
+
+def i0(x, name=None):
+    r"""
+    The function used to calculate modified bessel function of order 0.
+
+    Equation:
+        ..  math::
+
+            I_0(x) = \sum^{\infty}_{k=0}\frac{(x^2/4)^k}{(k!)^2}
+
+    Args:
+        x (Tensor): The input tensor, it's data type should be float32, float64.
+        name (str, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
+
+    Returns:
+        - out (Tensor), A Tensor. the value of the modified bessel function of order 0 at x.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+
+            x = paddle.to_tensor([0, 1, 2, 3, 4], dtype="float32")
+            print(paddle.i0(x))
+            # (Tensor(shape=[5], dtype=float32, place=Place(cpu), stop_gradient=True, [0.99999994 , 1.26606596 , 2.27958512 , 4.88079262 , 11.30192089]),
+    """
+    if in_dygraph_mode():
+        return _C_ops.i0(x)
+    else:
+        check_variable_and_dtype(x, "x", ["float32", "float64"], "i0")
+
+        helper = LayerHelper("i0", **locals())
+        out = helper.create_variable_for_type_inference(dtype=x.dtype)
+        helper.append_op(type='i0', inputs={'x': x}, outputs={'out': out})
+    return out
+
+
+def i0e(x, name=None):
+    r"""
+    The function used to calculate exponentially scaled modified Bessel function of order 0.
+
+    Equation:
+        ..  math::
+
+            I_0(x) = \sum^{\infty}_{k=0}\frac{(x^2/4)^k}{(k!)^2} \\
+            I_{0e}(x) = e^{-|x|}I_0(x)
+
+    Args:
+        x (Tensor): The input tensor, it's data type should be float32, float64.
+        name (str, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
+
+    Returns:
+        - out (Tensor), A Tensor. the value of the exponentially scaled modified Bessel function of order 0 at x.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+
+            x = paddle.to_tensor([0, 1, 2, 3, 4], dtype="float32")
+            print(paddle.i0e(x))
+            # (Tensor(shape=[5], dtype=float32, place=Place(cpu), stop_gradient=True, [1., 0.46575961, 0.30850832, 0.24300035, 0.20700192]),
+    """
+    if in_dygraph_mode():
+        return _C_ops.i0e(x)
+    else:
+        check_variable_and_dtype(x, "x", ["float32", "float64"], "i0e")
+
+        helper = LayerHelper("i0e", **locals())
+        out = helper.create_variable_for_type_inference(dtype=x.dtype)
+        helper.append_op(type='i0e', inputs={'x': x}, outputs={'out': out})
+    return out

From b75d8c7ec535bb26d0d40ad56336d70e53452453 Mon Sep 17 00:00:00 2001
From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com>
Date: Sat, 13 May 2023 09:13:54 +0800
Subject: [PATCH 314/405] Revert elementwise add (#53745)

* modify concat_grad add sum comp rule

* delete default mul_double_grad

* delete high grad test

* recover yaml

* modify yaml

* recover add_double_grad prim
---
 .../eager/auto_code_generator/generator/eager_gen.py |  1 -
 paddle/phi/api/yaml/legacy_backward.yaml             | 12 ++++++++++++
 test/prim/prim/vjp/test_comp_high_grad.py            |  3 ++-
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
index 08a5f57d293af..8d28fa2438f23 100644
--- a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
+++ b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
@@ -67,7 +67,6 @@
 prim_white_list = [
     "matmul_double_grad",
     "tanh_double_grad",
-    "add_double_grad",
     "subtract_double_grad",
 ]
 
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index 3eaafe2b407ad..758a2ac3147e7 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -30,6 +30,7 @@
   kernel :
     func : add_double_grad
   optional : grad_x_grad, grad_y_grad
+  backward : add_triple_grad
   inplace : (grad_x_grad -> grad_out_grad)
   composite : add_double_grad(y, grad_out, grad_x_grad, grad_y_grad, axis, grad_out_grad)
 
@@ -47,6 +48,17 @@
   backward : add_double_grad
   inplace : (out_grad -> x_grad)
 
+- backward_op : add_triple_grad
+  forward : add_double_grad (Tensor y, Tensor grad_out, Tensor grad_grad_x, Tensor grad_grad_y, int axis = -1) -> Tensor(grad_grad_out)
+  args : (Tensor grad_grad_x, Tensor grad_grad_y, Tensor grad_grad_out_grad, int axis = -1)
+  output : Tensor(grad_grad_x_grad), Tensor(grad_grad_y_grad)
+  infer_meta :
+    func : GeneralBinaryGradInferMeta
+    param : [grad_grad_x, grad_grad_y]
+  kernel :
+    func : add_triple_grad
+  inplace : (grad_grad_out_grad -> grad_grad_x_grad)
+
 - backward_op : amax_grad
   forward: amax (Tensor x,  int64_t[] axis={},  bool keepdim=false) -> Tensor(out)
   args : (Tensor x, Tensor out, Tensor out_grad, int64_t[] axis={},  bool keepdim=false, bool reduce_all=false)
diff --git a/test/prim/prim/vjp/test_comp_high_grad.py b/test/prim/prim/vjp/test_comp_high_grad.py
index 87b3c8f300ecd..99268b1b58ed9 100644
--- a/test/prim/prim/vjp/test_comp_high_grad.py
+++ b/test/prim/prim/vjp/test_comp_high_grad.py
@@ -25,7 +25,7 @@
 from paddle import fluid
 from paddle.fluid import core
 
-
+'''
 @param.parameterized_class(
     ('shape1', 'shape2'),
     [
@@ -120,6 +120,7 @@ def test_high_grad(self):
         for p in places:
             self.func_double(p)
             self.func_triple(p)
+'''
 
 
 @param.parameterized_class(

From 3e90a4616235ce77dc7d98ad477273f01a73848d Mon Sep 17 00:00:00 2001
From: tianshuo78520a <707759223@qq.com>
Date: Sun, 14 May 2023 15:14:17 +0800
Subject: [PATCH 315/405] fix build error (#53790)

* fix build error

* fix build error

* fix
---
 cmake/external/gflags.cmake        | 1 +
 paddle/phi/capi/lib/CMakeLists.txt | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake
index 606eb7ce1be6a..d73469003eed3 100755
--- a/cmake/external/gflags.cmake
+++ b/cmake/external/gflags.cmake
@@ -21,6 +21,7 @@ set(GFLAGS_INCLUDE_DIR
     CACHE PATH "gflags include directory." FORCE)
 set(GFLAGS_REPOSITORY ${GIT_URL}/gflags/gflags.git)
 set(GFLAGS_TAG "v2.2.2")
+
 if(WIN32)
   set(GFLAGS_LIBRARIES
       "${GFLAGS_INSTALL_DIR}/lib/gflags_static.lib"
diff --git a/paddle/phi/capi/lib/CMakeLists.txt b/paddle/phi/capi/lib/CMakeLists.txt
index 60afb74a6d451..185de92cd20ec 100644
--- a/paddle/phi/capi/lib/CMakeLists.txt
+++ b/paddle/phi/capi/lib/CMakeLists.txt
@@ -6,7 +6,7 @@ cc_library(
 cc_library(
   phi_c_device_context
   SRCS c_device_context.cc
-  DEPS phi_backends)
+  DEPS auto_parallel_proto phi_backends)
 
 cc_library(
   phi_c_int_array

From 34122e3ec281d59c15cc2e6cc1f7f27bc5bc2431 Mon Sep 17 00:00:00 2001
From: zhangyuqin1998 <75946871+zhangyuqin1998@users.noreply.github.com>
Date: Mon, 15 May 2023 10:11:45 +0800
Subject: [PATCH 316/405] move OneHotRawKernel to legacy (#53200)

* move OneHotRawKernel to legacy

* fix
---
 paddle/phi/kernels/cpu/one_hot_kernel.cc      | 41 +++++---
 paddle/phi/kernels/gpu/one_hot_kernel.cu      | 59 ++++--------
 .../phi/kernels/legacy/cpu/one_hot_kernel.cc  | 88 +++++++++++++++++
 .../phi/kernels/legacy/gpu/one_hot_kernel.cu  | 96 +++++++++++++++++++
 .../phi/kernels/legacy/xpu/one_hot_kernel.cc  | 68 +++++++++++++
 paddle/phi/kernels/one_hot_kernel.cc          | 47 ---------
 paddle/phi/kernels/one_hot_kernel.h           |  8 --
 paddle/phi/kernels/xpu/one_hot_kernel.cc      | 47 +++------
 8 files changed, 312 insertions(+), 142 deletions(-)
 create mode 100644 paddle/phi/kernels/legacy/cpu/one_hot_kernel.cc
 create mode 100644 paddle/phi/kernels/legacy/gpu/one_hot_kernel.cu
 create mode 100644 paddle/phi/kernels/legacy/xpu/one_hot_kernel.cc
 delete mode 100644 paddle/phi/kernels/one_hot_kernel.cc

diff --git a/paddle/phi/kernels/cpu/one_hot_kernel.cc b/paddle/phi/kernels/cpu/one_hot_kernel.cc
index 9d1daaf065f25..0958e2c02b4c1 100644
--- a/paddle/phi/kernels/cpu/one_hot_kernel.cc
+++ b/paddle/phi/kernels/cpu/one_hot_kernel.cc
@@ -63,12 +63,10 @@ struct OneHotV2OpFunctor {
 };
 
 template <typename T, typename Context>
-void OneHotRawKernel(const Context& dev_ctx,
-                     const DenseTensor& x,
-                     const Scalar& depth,
-                     DataType dtype,
-                     bool allow_out_of_range,
-                     DenseTensor* out) {
+void OneHotKernel(const Context& dev_ctx,
+                  const DenseTensor& x,
+                  const Scalar& depth,
+                  DenseTensor* out) {
   auto depth_v = depth.to<int>();
   auto out_dims = out->dims();
   if (out_dims[out_dims.size() - 1] == -1) {
@@ -76,13 +74,34 @@ void OneHotRawKernel(const Context& dev_ctx,
     out->Resize(out_dims);
   }
 
-  phi::VisitDataType(dtype,
-                     OneHotV2OpFunctor<Context, T>(&x, out, depth_v, dev_ctx));
+  auto* p_in_data = x.data<T>();
+  auto numel = x.numel();
+  auto* p_out_data = dev_ctx.template Alloc<float>(out);
+  funcs::set_constant(dev_ctx, out, 0.0);
+
+  for (int i = 0; i < numel; ++i) {
+    PADDLE_ENFORCE_GE(
+        p_in_data[i],
+        0,
+        phi::errors::InvalidArgument(
+            "Illegal index value, Input(input) value should be at least 0, "
+            "but received input (%d) less than 0",
+            p_in_data[i]));
+    PADDLE_ENFORCE_LT(
+        p_in_data[i],
+        depth_v,
+        phi::errors::InvalidArgument(
+            "Illegal index value, Input(input) value should be less than "
+            "Input(depth), "
+            "but received input (%d) not less than depth (%d)",
+            p_in_data[i],
+            depth_v));
+    *(p_out_data + i * depth_v + p_in_data[i]) = 1.0;
+  }
 }
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(
-    one_hot_raw, CPU, ALL_LAYOUT, phi::OneHotRawKernel, int, int64_t) {
-  kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
+PD_REGISTER_KERNEL(one_hot, CPU, ALL_LAYOUT, phi::OneHotKernel, int, int64_t) {
+  kernel->OutputAt(0).SetDataType(phi::DataType::FLOAT32);
 }
diff --git a/paddle/phi/kernels/gpu/one_hot_kernel.cu b/paddle/phi/kernels/gpu/one_hot_kernel.cu
index 45d208498a55a..63084585d809d 100644
--- a/paddle/phi/kernels/gpu/one_hot_kernel.cu
+++ b/paddle/phi/kernels/gpu/one_hot_kernel.cu
@@ -40,43 +40,11 @@ __global__ void FillOutputKernel(const InT* p_in_data,
   }
 }
 
-template <typename DeviceContext, typename InT>
-struct OneHotV2OpCUDAFunctor {
-  const DenseTensor* in_;
-  DenseTensor* out_;
-  const DeviceContext& ctx_;
-  int depth_;
-
-  OneHotV2OpCUDAFunctor(const DenseTensor* in,
-                        DenseTensor* out,
-                        int depth,
-                        const DeviceContext& ctx)
-      : in_(in), out_(out), depth_(depth), ctx_(ctx) {}
-
-  template <typename OutT>
-  void apply() const {
-    auto* p_in_data = in_->data<InT>();
-    auto numel = in_->numel();
-    auto* p_out_data = ctx_.template Alloc<OutT>(out_);
-    auto stream = ctx_.stream();
-    funcs::set_constant(ctx_, out_, 0.0);
-
-    auto config = phi::backends::gpu::GetGpuLaunchConfig1D(ctx_, numel);
-
-    FillOutputKernel<<<config.block_per_grid,
-                       config.thread_per_block,
-                       0,
-                       stream>>>(p_in_data, p_out_data, numel, depth_);
-  }
-};
-
 template <typename T, typename Context>
-void OneHotRawKernel(const Context& dev_ctx,
-                     const DenseTensor& x,
-                     const Scalar& depth,
-                     DataType dtype,
-                     bool allow_out_of_range,
-                     DenseTensor* out) {
+void OneHotKernel(const Context& dev_ctx,
+                  const DenseTensor& x,
+                  const Scalar& depth,
+                  DenseTensor* out) {
   auto depth_v = depth.to<int>();
   auto out_dims = out->dims();
   if (out_dims[out_dims.size() - 1] == -1) {
@@ -84,13 +52,22 @@ void OneHotRawKernel(const Context& dev_ctx,
     out->Resize(out_dims);
   }
 
-  phi::VisitDataType(
-      dtype, OneHotV2OpCUDAFunctor<Context, T>(&x, out, depth_v, dev_ctx));
+  auto* p_in_data = x.data<T>();
+  auto numel = x.numel();
+  auto* p_out_data = dev_ctx.template Alloc<float>(out);
+  auto stream = dev_ctx.stream();
+  funcs::set_constant(dev_ctx, out, 0.0);
+
+  auto config = phi::backends::gpu::GetGpuLaunchConfig1D(dev_ctx, numel);
+
+  FillOutputKernel<<<config.block_per_grid,
+                     config.thread_per_block,
+                     0,
+                     stream>>>(p_in_data, p_out_data, numel, depth_v);
 }
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(
-    one_hot_raw, GPU, ALL_LAYOUT, phi::OneHotRawKernel, int, int64_t) {
-  kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
+PD_REGISTER_KERNEL(one_hot, GPU, ALL_LAYOUT, phi::OneHotKernel, int, int64_t) {
+  kernel->OutputAt(0).SetDataType(phi::DataType::FLOAT32);
 }
diff --git a/paddle/phi/kernels/legacy/cpu/one_hot_kernel.cc b/paddle/phi/kernels/legacy/cpu/one_hot_kernel.cc
new file mode 100644
index 0000000000000..040a85599149a
--- /dev/null
+++ b/paddle/phi/kernels/legacy/cpu/one_hot_kernel.cc
@@ -0,0 +1,88 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/common/scalar.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/utils/data_type.h"
+#include "paddle/phi/kernels/funcs/math_function.h"
+
+namespace phi {
+
+template <typename DeviceContext, typename InT>
+struct OneHotV2OpFunctor {
+  const DenseTensor* in_;
+  DenseTensor* out_;
+  int depth_;
+  const DeviceContext& ctx_;
+
+  OneHotV2OpFunctor(const DenseTensor* in,
+                    DenseTensor* out,
+                    int depth,
+                    const DeviceContext& ctx)
+      : in_(in), out_(out), depth_(depth), ctx_(ctx) {}
+
+  template <typename OutT>
+  void apply() const {
+    auto* p_in_data = in_->data<InT>();
+    auto numel = in_->numel();
+    auto* p_out_data = ctx_.template Alloc<OutT>(out_);
+    funcs::set_constant(ctx_, out_, 0.0);
+
+    for (int i = 0; i < numel; ++i) {
+      PADDLE_ENFORCE_GE(
+          p_in_data[i],
+          0,
+          phi::errors::InvalidArgument(
+              "Illegal index value, Input(input) value should be at least 0, "
+              "but received input (%d) less than 0",
+              p_in_data[i]));
+      PADDLE_ENFORCE_LT(
+          p_in_data[i],
+          depth_,
+          phi::errors::InvalidArgument(
+              "Illegal index value, Input(input) value should be less than "
+              "Input(depth), "
+              "but received input (%d) not less than depth (%d)",
+              p_in_data[i],
+              depth_));
+      *(p_out_data + i * depth_ + p_in_data[i]) = 1.0;
+    }
+  }
+};
+
+template <typename T, typename Context>
+void OneHotRawKernel(const Context& dev_ctx,
+                     const DenseTensor& x,
+                     const Scalar& depth,
+                     DataType dtype,
+                     bool allow_out_of_range,
+                     DenseTensor* out) {
+  auto depth_v = depth.to<int>();
+  auto out_dims = out->dims();
+  if (out_dims[out_dims.size() - 1] == -1) {
+    out_dims[out_dims.size() - 1] = depth_v;
+    out->Resize(out_dims);
+  }
+
+  phi::VisitDataType(dtype,
+                     OneHotV2OpFunctor<Context, T>(&x, out, depth_v, dev_ctx));
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(
+    one_hot_raw, CPU, ALL_LAYOUT, phi::OneHotRawKernel, int, int64_t) {
+  kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
+}
diff --git a/paddle/phi/kernels/legacy/gpu/one_hot_kernel.cu b/paddle/phi/kernels/legacy/gpu/one_hot_kernel.cu
new file mode 100644
index 0000000000000..c64f2e2d75566
--- /dev/null
+++ b/paddle/phi/kernels/legacy/gpu/one_hot_kernel.cu
@@ -0,0 +1,96 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/backends/gpu/gpu_launch_config.h"
+#include "paddle/phi/backends/gpu/gpu_primitives.h"
+#include "paddle/phi/common/scalar.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/math_function.h"
+
+namespace phi {
+
+using phi::PADDLE_CUDA_NUM_THREADS;
+
+template <typename InT, typename OutT>
+__global__ void FillOutputKernel(const InT* p_in_data,
+                                 OutT* p_out_data,
+                                 const int64_t numel,
+                                 const int depth) {
+  CUDA_KERNEL_LOOP_TYPE(idx, numel, int64_t) {
+    PADDLE_ENFORCE(p_in_data[idx] >= 0 && p_in_data[idx] < depth,
+                   "Illegal index value, Input(input) value should be "
+                   "greater than or equal to 0, and less than depth [%d], "
+                   "but received [%lld].",
+                   depth,
+                   p_in_data[idx]);
+
+    *(p_out_data + (idx * depth) + p_in_data[idx]) = 1.0;
+  }
+}
+
+template <typename DeviceContext, typename InT>
+struct OneHotV2OpCUDAFunctor {
+  const DenseTensor* in_;
+  DenseTensor* out_;
+  const DeviceContext& ctx_;
+  int depth_;
+
+  OneHotV2OpCUDAFunctor(const DenseTensor* in,
+                        DenseTensor* out,
+                        int depth,
+                        const DeviceContext& ctx)
+      : in_(in), out_(out), depth_(depth), ctx_(ctx) {}
+
+  template <typename OutT>
+  void apply() const {
+    auto* p_in_data = in_->data<InT>();
+    auto numel = in_->numel();
+    auto* p_out_data = ctx_.template Alloc<OutT>(out_);
+    auto stream = ctx_.stream();
+    funcs::set_constant(ctx_, out_, 0.0);
+
+    auto config = phi::backends::gpu::GetGpuLaunchConfig1D(ctx_, numel);
+
+    FillOutputKernel<<<config.block_per_grid,
+                       config.thread_per_block,
+                       0,
+                       stream>>>(p_in_data, p_out_data, numel, depth_);
+  }
+};
+
+template <typename T, typename Context>
+void OneHotRawKernel(const Context& dev_ctx,
+                     const DenseTensor& x,
+                     const Scalar& depth,
+                     DataType dtype,
+                     bool allow_out_of_range,
+                     DenseTensor* out) {
+  auto depth_v = depth.to<int>();
+  auto out_dims = out->dims();
+  if (out_dims[out_dims.size() - 1] == -1) {
+    out_dims[out_dims.size() - 1] = depth_v;
+    out->Resize(out_dims);
+  }
+
+  phi::VisitDataType(
+      dtype, OneHotV2OpCUDAFunctor<Context, T>(&x, out, depth_v, dev_ctx));
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(
+    one_hot_raw, GPU, ALL_LAYOUT, phi::OneHotRawKernel, int, int64_t) {
+  kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
+}
diff --git a/paddle/phi/kernels/legacy/xpu/one_hot_kernel.cc b/paddle/phi/kernels/legacy/xpu/one_hot_kernel.cc
new file mode 100644
index 0000000000000..02edbd128430b
--- /dev/null
+++ b/paddle/phi/kernels/legacy/xpu/one_hot_kernel.cc
@@ -0,0 +1,68 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/backends/xpu/enforce_xpu.h"
+#include "paddle/phi/backends/xpu/xpu_context.h"
+#include "paddle/phi/common/scalar.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/utils/data_type.h"
+
+namespace phi {
+template <typename Context, typename InT>
+struct OneHotV2OpFunctor {
+  const DenseTensor* in_;
+  DenseTensor* out_;
+  int depth_;
+  const Context& ctx_;
+
+  OneHotV2OpFunctor(const DenseTensor* in,
+                    DenseTensor* out,
+                    int depth,
+                    const Context& ctx)
+      : in_(in), out_(out), depth_(depth), ctx_(ctx) {}
+
+  template <typename OutT>
+  void apply() const {
+    auto* p_in_data = in_->data<InT>();
+    auto numel = in_->numel();
+    auto* p_out_data = ctx_.template Alloc<float>(out_);
+    int r = xpu::one_hot<InT>(
+        ctx_.x_context(), p_in_data, p_out_data, numel, depth_, 1.0, 0.0);
+    PADDLE_ENFORCE_XDNN_SUCCESS(r, "one_hot");
+  }
+};
+
+template <typename T, typename Context>
+void OneHotRawKernel(const Context& dev_ctx,
+                     const DenseTensor& x,
+                     const Scalar& depth,
+                     DataType dtype,
+                     bool allow_out_of_range,
+                     DenseTensor* out) {
+  auto depth_v = depth.to<int>();
+  auto out_dims = out->dims();
+  if (out_dims[out_dims.size() - 1] == -1) {
+    out_dims[out_dims.size() - 1] = depth_v;
+    out->Resize(out_dims);
+  }
+  phi::VisitDataType(dtype,
+                     OneHotV2OpFunctor<Context, T>(&x, out, depth_v, dev_ctx));
+}
+}  // namespace phi
+
+PD_REGISTER_KERNEL(
+    one_hot_raw, XPU, ALL_LAYOUT, phi::OneHotRawKernel, int, int64_t) {
+  kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
+}
diff --git a/paddle/phi/kernels/one_hot_kernel.cc b/paddle/phi/kernels/one_hot_kernel.cc
deleted file mode 100644
index 3ad4799ef8a6f..0000000000000
--- a/paddle/phi/kernels/one_hot_kernel.cc
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/kernels/one_hot_kernel.h"
-
-#include "paddle/phi/backends/all_context.h"
-#include "paddle/phi/core/kernel_registry.h"
-
-namespace phi {
-
-template <typename T, typename Context>
-void OneHotKernel(const Context& dev_ctx,
-                  const DenseTensor& x,
-                  const Scalar& num_classes_s,
-                  DenseTensor* out) {
-  OneHotRawKernel<T>(
-      dev_ctx, x, num_classes_s, phi::DataType::FLOAT32, false, out);
-}
-
-}  // namespace phi
-
-PD_REGISTER_KERNEL(one_hot, CPU, ALL_LAYOUT, phi::OneHotKernel, int, int64_t) {
-  kernel->OutputAt(0).SetDataType(phi::DataType::FLOAT32);
-}
-
-#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-PD_REGISTER_KERNEL(one_hot, GPU, ALL_LAYOUT, phi::OneHotKernel, int, int64_t) {
-  kernel->OutputAt(0).SetDataType(phi::DataType::FLOAT32);
-}
-#endif
-
-#ifdef PADDLE_WITH_XPU
-PD_REGISTER_KERNEL(one_hot, XPU, ALL_LAYOUT, phi::OneHotKernel, int, int64_t) {
-  kernel->OutputAt(0).SetDataType(phi::DataType::FLOAT32);
-}
-#endif
diff --git a/paddle/phi/kernels/one_hot_kernel.h b/paddle/phi/kernels/one_hot_kernel.h
index 79af88473b278..b3b0bf3e9e4e3 100644
--- a/paddle/phi/kernels/one_hot_kernel.h
+++ b/paddle/phi/kernels/one_hot_kernel.h
@@ -25,12 +25,4 @@ void OneHotKernel(const Context& dev_ctx,
                   const Scalar& num_classes,
                   DenseTensor* out);
 
-template <typename T, typename Context>
-void OneHotRawKernel(const Context& dev_ctx,
-                     const DenseTensor& x,
-                     const Scalar& depth,
-                     DataType dtype,
-                     bool allow_out_of_range,
-                     DenseTensor* out);
-
 }  // namespace phi
diff --git a/paddle/phi/kernels/xpu/one_hot_kernel.cc b/paddle/phi/kernels/xpu/one_hot_kernel.cc
index c0b17b122eab6..ad96d4858f7ed 100644
--- a/paddle/phi/kernels/xpu/one_hot_kernel.cc
+++ b/paddle/phi/kernels/xpu/one_hot_kernel.cc
@@ -19,49 +19,26 @@
 #include "paddle/phi/core/utils/data_type.h"
 
 namespace phi {
-template <typename Context, typename InT>
-struct OneHotV2OpFunctor {
-  const DenseTensor* in_;
-  DenseTensor* out_;
-  int depth_;
-  const Context& ctx_;
-
-  OneHotV2OpFunctor(const DenseTensor* in,
-                    DenseTensor* out,
-                    int depth,
-                    const Context& ctx)
-      : in_(in), out_(out), depth_(depth), ctx_(ctx) {}
-
-  template <typename OutT>
-  void apply() const {
-    auto* p_in_data = in_->data<InT>();
-    auto numel = in_->numel();
-    auto* p_out_data = ctx_.template Alloc<float>(out_);
-    int r = xpu::one_hot<InT>(
-        ctx_.x_context(), p_in_data, p_out_data, numel, depth_, 1.0, 0.0);
-    PADDLE_ENFORCE_XDNN_SUCCESS(r, "one_hot");
-  }
-};
-
 template <typename T, typename Context>
-void OneHotRawKernel(const Context& dev_ctx,
-                     const DenseTensor& x,
-                     const Scalar& depth,
-                     DataType dtype,
-                     bool allow_out_of_range,
-                     DenseTensor* out) {
+void OneHotKernel(const Context& dev_ctx,
+                  const DenseTensor& x,
+                  const Scalar& depth,
+                  DenseTensor* out) {
   auto depth_v = depth.to<int>();
   auto out_dims = out->dims();
   if (out_dims[out_dims.size() - 1] == -1) {
     out_dims[out_dims.size() - 1] = depth_v;
     out->Resize(out_dims);
   }
-  phi::VisitDataType(dtype,
-                     OneHotV2OpFunctor<Context, T>(&x, out, depth_v, dev_ctx));
+  auto* p_in_data = x.data<T>();
+  auto numel = x.numel();
+  auto* p_out_data = dev_ctx.template Alloc<float>(out);
+  int r = xpu::one_hot<T>(
+      dev_ctx.x_context(), p_in_data, p_out_data, numel, depth_v, 1.0, 0.0);
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "one_hot");
 }
 }  // namespace phi
 
-PD_REGISTER_KERNEL(
-    one_hot_raw, XPU, ALL_LAYOUT, phi::OneHotRawKernel, int, int64_t) {
-  kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
+PD_REGISTER_KERNEL(one_hot, XPU, ALL_LAYOUT, phi::OneHotKernel, int, int64_t) {
+  kernel->OutputAt(0).SetDataType(phi::DataType::FLOAT32);
 }

From 3dce9f0a760b2a7bbf1ad92ee36f60029832a0b8 Mon Sep 17 00:00:00 2001
From: niuliling123 <51102941+niuliling123@users.noreply.github.com>
Date: Mon, 15 May 2023 10:37:08 +0800
Subject: [PATCH 317/405] Tranpose layout (#53351)

* update

* Update backward.h

* Update composite_backward_api.h

* Update tensor_utils.cc

* Update backward.cc

* update

* stype

* update

* add ctest

* code stype
---
 paddle/fluid/eager/eager_layout_auto_tune.h   |   2 +-
 paddle/fluid/eager/eager_layout_transformer.h |   2 +-
 paddle/phi/api/yaml/legacy_backward.yaml      |   9 ++
 paddle/phi/api/yaml/legacy_ops.yaml           |   9 ++
 paddle/phi/infermeta/backward.cc              |  27 +++++
 paddle/phi/infermeta/backward.h               |   8 ++
 paddle/phi/infermeta/unary.cc                 |  19 ---
 .../phi/kernels/cpu/transpose_grad_kernel.cc  |  13 ++
 .../phi/kernels/gpu/transpose_grad_kernel.cu  |  14 +++
 .../kernels/impl/transpose_grad_kernel_impl.h |   9 ++
 paddle/phi/kernels/transpose_grad_kernel.h    |  12 ++
 .../tests/unittests/test_trans_layout_op.py   | 112 ++++++++++++++++++
 12 files changed, 215 insertions(+), 21 deletions(-)
 create mode 100644 python/paddle/fluid/tests/unittests/test_trans_layout_op.py

diff --git a/paddle/fluid/eager/eager_layout_auto_tune.h b/paddle/fluid/eager/eager_layout_auto_tune.h
index d95d464523c2f..61a5218d36ce3 100644
--- a/paddle/fluid/eager/eager_layout_auto_tune.h
+++ b/paddle/fluid/eager/eager_layout_auto_tune.h
@@ -150,7 +150,7 @@ inline std::shared_ptr<EagerLayoutTransformer> EagerLayoutAutotune(
         op_name, tensors_vector, tensors_vector[0][0].layout());
   }
 
-  if (op_name == "transpose2" &&
+  if ((op_name == "transpose2" || op_name == "trans_layout") &&
       (tensors_vector[0][0].layout() == DesiredLayout())) {
     auto trans = std::make_shared<EagerTransposeOpTransformer>(op_name);
     trans->SetAttr(attr,
diff --git a/paddle/fluid/eager/eager_layout_transformer.h b/paddle/fluid/eager/eager_layout_transformer.h
index ce80f6c948a7f..ddba37166cac2 100644
--- a/paddle/fluid/eager/eager_layout_transformer.h
+++ b/paddle/fluid/eager/eager_layout_transformer.h
@@ -34,7 +34,7 @@ inline paddle::Tensor EagerTraceTransposeOp(const phi::DataLayout layout,
   } else {
     axis = {0, 1, 2, 3};
   }
-  auto out_tensor = transpose_ad_func(in, axis);
+  auto out_tensor = trans_layout_ad_func(in, axis);
   VLOG(4) << "AutoTune Transpose from " << in.layout() << " to " << layout;
   return out_tensor;
 }
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index 758a2ac3147e7..37a3626861c08 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -1019,6 +1019,15 @@
   composite : tile_grad(x, outgrad, repeat_times, x_grad)
   backward : tile_double_grad
 
+- backward_op : trans_layout_grad
+  forward : trans_layout (Tensor x, int[] perm) -> Tensor(out)
+  args : (Tensor x, Tensor out_grad, int[] perm)
+  output : Tensor(x_grad)
+  infer_meta :
+    func : TransLayoutGradInferMeta
+  kernel :
+    func : trans_layout_grad
+
 - backward_op : transpose_double_grad
   forward : transpose_grad (Tensor grad_out, int[] perm) -> Tensor(grad_x)
   args : (Tensor grad_x_grad, int[] perm)
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index 9b76564c83259..cf85f1b8bfe32 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -1103,6 +1103,15 @@
     func : tile
   backward : tile_grad
 
+- op : trans_layout
+  args : (Tensor x, int[] perm)
+  output : Tensor
+  infer_meta :
+    func : TransposeInferMeta
+  kernel :
+    func : transpose
+  backward : trans_layout_grad
+
 - op : transpose
   args : (Tensor x, int[] perm)
   output : Tensor
diff --git a/paddle/phi/infermeta/backward.cc b/paddle/phi/infermeta/backward.cc
index bd9fae6bd155b..cb49124017221 100644
--- a/paddle/phi/infermeta/backward.cc
+++ b/paddle/phi/infermeta/backward.cc
@@ -1069,6 +1069,33 @@ void StackGradInferMeta(const MetaTensor& out_grad,
   }
 }
 
+void TransposeGradInferMeta(const MetaTensor& x,
+                            const std::vector<int>& axis,
+                            MetaTensor* out) {
+  size_t x_rank = x.dims().size();
+  std::vector<int> formated_axis = axis;
+  for (size_t i = 0; i < axis.size(); i++) {
+    if (axis[i] < 0) {
+      formated_axis[i] = axis[i] + x_rank;
+    }
+  }
+
+  std::vector<int> reversed_axis(axis);
+  for (size_t i = 0; i < formated_axis.size(); i++) {
+    reversed_axis[formated_axis[i]] = i;
+  }
+
+  TransposeInferMeta(x, reversed_axis, out);
+}
+
+void TransLayoutGradInferMeta(const MetaTensor& x,
+                              const MetaTensor& out_grad,
+                              const std::vector<int>& axis,
+                              MetaTensor* x_grad) {
+  TransposeGradInferMeta(out_grad, axis, x_grad);
+  x_grad->set_layout(static_cast<DataLayout>(x.layout()));
+}
+
 void UniformRandomInplaceGradInferMeta(const MetaTensor& out_grad,
                                        float min,
                                        float max,
diff --git a/paddle/phi/infermeta/backward.h b/paddle/phi/infermeta/backward.h
index 8d3edf2a40a94..4dc995cb296f6 100644
--- a/paddle/phi/infermeta/backward.h
+++ b/paddle/phi/infermeta/backward.h
@@ -413,6 +413,14 @@ void StackGradInferMeta(const MetaTensor& out_grad,
                         int axis,
                         std::vector<MetaTensor*> x_grad);
 
+void TransposeInferMeta(const MetaTensor& x,
+                        const std::vector<int>& axis,
+                        MetaTensor* out);
+
+void TransLayoutGradInferMeta(const MetaTensor& x,
+                              const MetaTensor& out_grad,
+                              const std::vector<int>& axis,
+                              MetaTensor* out);
 void UniformRandomInplaceGradInferMeta(const MetaTensor& out_grad,
                                        float min,
                                        float max,
diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc
index 42b99dee7cb50..67ad639f648d8 100644
--- a/paddle/phi/infermeta/unary.cc
+++ b/paddle/phi/infermeta/unary.cc
@@ -4483,25 +4483,6 @@ void TransposeInferMeta(const MetaTensor& x,
   out->set_dtype(x.dtype());
 }
 
-void TransposeGradInferMeta(const MetaTensor& x,
-                            const std::vector<int>& axis,
-                            MetaTensor* out) {
-  size_t x_rank = x.dims().size();
-  std::vector<int> formated_axis = axis;
-  for (size_t i = 0; i < axis.size(); i++) {
-    if (axis[i] < 0) {
-      formated_axis[i] = axis[i] + x_rank;
-    }
-  }
-
-  std::vector<int> reversed_axis(axis);
-  for (size_t i = 0; i < formated_axis.size(); i++) {
-    reversed_axis[formated_axis[i]] = i;
-  }
-
-  TransposeInferMeta(x, reversed_axis, out);
-}
-
 void UnbindInferMeta(const MetaTensor& x,
                      int axis,
                      std::vector<MetaTensor*> outs) {
diff --git a/paddle/phi/kernels/cpu/transpose_grad_kernel.cc b/paddle/phi/kernels/cpu/transpose_grad_kernel.cc
index dee69222e6dc0..590d227d1e518 100644
--- a/paddle/phi/kernels/cpu/transpose_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/transpose_grad_kernel.cc
@@ -31,3 +31,16 @@ PD_REGISTER_KERNEL(transpose_grad,
                    phi::dtype::bfloat16,
                    phi::dtype::complex<float>,
                    phi::dtype::complex<double>) {}
+
+PD_REGISTER_KERNEL(trans_layout_grad,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::TransLayoutGradKernel,
+                   bool,
+                   float,
+                   double,
+                   int32_t,
+                   int64_t,
+                   phi::dtype::bfloat16,
+                   phi::dtype::complex<float>,
+                   phi::dtype::complex<double>) {}
diff --git a/paddle/phi/kernels/gpu/transpose_grad_kernel.cu b/paddle/phi/kernels/gpu/transpose_grad_kernel.cu
index e8ad456084ab1..35215ed5b96ed 100644
--- a/paddle/phi/kernels/gpu/transpose_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/transpose_grad_kernel.cu
@@ -32,3 +32,17 @@ PD_REGISTER_KERNEL(transpose_grad,
                    phi::dtype::bfloat16,
                    phi::dtype::complex<float>,
                    phi::dtype::complex<double>) {}
+
+PD_REGISTER_KERNEL(trans_layout_grad,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::TransLayoutGradKernel,
+                   bool,
+                   float,
+                   double,
+                   int32_t,
+                   int64_t,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16,
+                   phi::dtype::complex<float>,
+                   phi::dtype::complex<double>) {}
diff --git a/paddle/phi/kernels/impl/transpose_grad_kernel_impl.h b/paddle/phi/kernels/impl/transpose_grad_kernel_impl.h
index 71f35fa1fdf0e..770fdcd20077e 100644
--- a/paddle/phi/kernels/impl/transpose_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/transpose_grad_kernel_impl.h
@@ -42,4 +42,13 @@ void TransposeGradKernel(const Context& dev_ctx,
   TransposeKernel<T, Context>(dev_ctx, out_grad, reversed_axis, x_grad);
 }
 
+template <typename T, typename Context>
+void TransLayoutGradKernel(const Context& dev_ctx,
+                           const DenseTensor& x,
+                           const DenseTensor& out_grad,
+                           const std::vector<int>& axis,
+                           DenseTensor* x_grad) {
+  TransposeGradKernel<T, Context>(dev_ctx, out_grad, axis, x_grad);
+}
+
 }  // namespace phi
diff --git a/paddle/phi/kernels/transpose_grad_kernel.h b/paddle/phi/kernels/transpose_grad_kernel.h
index e224da81a25d0..be617eae143ad 100644
--- a/paddle/phi/kernels/transpose_grad_kernel.h
+++ b/paddle/phi/kernels/transpose_grad_kernel.h
@@ -26,4 +26,16 @@ void TransposeGradKernel(const Context& dev_ctx,
                          const std::vector<int>& axis,
                          DenseTensor* x_grad);
 
+template <typename Context>
+void TransposeGradStrideKernel(const Context& dev_ctx,
+                               const DenseTensor& out_grad,
+                               const std::vector<int>& axis,
+                               DenseTensor* x_grad);
+template <typename T, typename Context>
+void TransLayoutGradKernel(const Context& dev_ctx,
+                           const DenseTensor& x,
+                           const DenseTensor& out_grad,
+                           const std::vector<int>& axis,
+                           DenseTensor* x_grad);
+
 }  // namespace phi
diff --git a/python/paddle/fluid/tests/unittests/test_trans_layout_op.py b/python/paddle/fluid/tests/unittests/test_trans_layout_op.py
new file mode 100644
index 0000000000000..e6e1967d75e00
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_trans_layout_op.py
@@ -0,0 +1,112 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import tempfile
+import unittest
+
+import numpy as np
+from eager_op_test import OpTest
+
+import paddle
+
+
+def transpose_layout(x, src_layout, dst_layout):
+    return x.transpose([0, 2, 3, 1])
+
+
+class TestTransferLayoutFP16Op(OpTest):
+    def setUp(self):
+        paddle.enable_static()
+        self.op_type = 'transfer_layout'
+        self.dtype = np.float16
+        x = np.random.random(size=[2, 5, 10, 10])
+        self.inputs = {'X': x.astype(self.dtype)}
+        self.outputs = {'Out': x.transpose([0, 2, 3, 1])}
+        self.attrs = {'src_layout': 0, 'dst_layout': 1}
+        self.python_api = transpose_layout
+
+    def test_check_output(self):
+        self.check_output()
+
+
+class LayoutAutoTune(unittest.TestCase):
+    def test_config(self):
+        paddle.fluid.core.enable_layout_autotune()
+        if self.use_autoune():
+            self.assertEqual(paddle.fluid.core.use_layout_autotune(), True)
+            paddle.fluid.core.disable_layout_autotune()
+        self.assertEqual(paddle.fluid.core.use_layout_autotune(), False)
+        self.use_autoune()
+
+    def setUp(self):
+        paddle.disable_static()
+        self.use_autoune()
+
+    def use_autoune(self):
+        if paddle.is_compiled_with_cuda():
+            paddle.incubate.autotune.set_config(
+                config={"layout": {"enable": True}}
+            )
+            return paddle.fluid.core.use_layout_autotune()
+        else:
+            config = {"layout": {"enable": False}}
+            tfile = tempfile.NamedTemporaryFile(mode="w+", delete=False)
+            json.dump(config, tfile)
+            tfile.close()
+            paddle.incubate.autotune.set_config(tfile.name)
+            os.remove(tfile.name)
+            return paddle.fluid.core.use_layout_autotune()
+
+    def test_flatten_op_transposer(self):
+        conv = paddle.nn.Conv2D(3, 8, (3, 3))
+        flatten = paddle.nn.Flatten(start_axis=1, stop_axis=2)
+        data = paddle.rand([1, 3, 16, 14])
+        with paddle.amp.auto_cast(level="O2"):
+            conv_out = conv(data)
+            # conv_out.shape = [1, 14, 12, 8] with NHWC
+            # layout tuner will transpose conv_out to
+            # [1, 8, 14, 12] with NCHW before the following flatten op
+            # because it flatten the C and H dimensions.
+            out = flatten(conv_out)
+
+        self.assertEqual(conv_out.shape, [1, 8, 14, 12])
+        self.assertEqual(out.shape, [1, 112, 12])
+
+    def test_argmax_op_transposer_keep_dims(self):
+        conv = paddle.nn.Conv2D(3, 8, (3, 3))
+        data = paddle.rand([1, 3, 16, 14])
+        with paddle.amp.auto_cast(level="O2"):
+            conv_out = conv(data)
+            # conv_out.shape = [1, 14, 12, 8] with NHWC
+            out = paddle.argmax(conv_out, axis=1, keepdim=True)
+        self.assertEqual(conv_out.shape, [1, 8, 14, 12])
+        self.assertEqual(out.shape, [1, 1, 14, 12])
+
+    def test_concat_op_transposer(self):
+        in1 = paddle.rand([1, 8, 14, 12])
+        conv = paddle.nn.Conv2D(3, 8, (3, 3))
+        data = paddle.rand([1, 3, 16, 14])
+        with paddle.amp.auto_cast(level="O2"):
+            conv_out = conv(data)
+            # conv_out.shape = [1, 14, 12, 8] with NHWC
+            out = paddle.concat(x=[conv_out, in1], axis=0)
+
+        self.assertEqual(conv_out.shape, [1, 8, 14, 12])
+        self.assertEqual(out.shape, [2, 8, 14, 12])
+
+
+if __name__ == '__main__':
+    unittest.main()

From 81056073b80abc2fcd86da32fdeb8ee8ae082313 Mon Sep 17 00:00:00 2001
From: RuohengMa <120699764+RuohengMa@users.noreply.github.com>
Date: Mon, 15 May 2023 10:59:49 +0800
Subject: [PATCH 318/405] [XPU][PHI] bind index_sample_grad xpu kernel (#53753)

---
 paddle/phi/backends/xpu/xpu2_op_list.cc       |  1 +
 .../kernels/xpu/index_sample_grad_kernel.cc   | 82 +++++++++++++++++++
 2 files changed, 83 insertions(+)
 create mode 100644 paddle/phi/kernels/xpu/index_sample_grad_kernel.cc

diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc
index ba402aed5a6cd..ba9742a98d018 100644
--- a/paddle/phi/backends/xpu/xpu2_op_list.cc
+++ b/paddle/phi/backends/xpu/xpu2_op_list.cc
@@ -420,6 +420,7 @@ XPUOpMap& get_kl2_ops() {
        XPUKernelSet({phi::DataType::FLOAT32,
                      phi::DataType::INT32,
                      phi::DataType::INT64})},
+      {"index_sample_grad", XPUKernelSet({phi::DataType::FLOAT32})},
       {"index_sample",
        XPUKernelSet({phi::DataType::INT8,
                      phi::DataType::INT16,
diff --git a/paddle/phi/kernels/xpu/index_sample_grad_kernel.cc b/paddle/phi/kernels/xpu/index_sample_grad_kernel.cc
new file mode 100644
index 0000000000000..22c35ef46840f
--- /dev/null
+++ b/paddle/phi/kernels/xpu/index_sample_grad_kernel.cc
@@ -0,0 +1,82 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/index_sample_grad_kernel.h"
+
+#include "paddle/phi/backends/xpu/enforce_xpu.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/utils/data_type.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void IndexSampleGradKernel(const Context& ctx,
+                           const DenseTensor& x,
+                           const DenseTensor& index,
+                           const DenseTensor& out_grad,
+                           DenseTensor* in_grad) {
+  using XPUType = typename XPUTypeTrait<T>::Type;
+  const auto& index_type = index.dtype();
+  bool index_type_match =
+      index_type == phi::DataType::INT32 || index_type == phi::DataType::INT64;
+  PADDLE_ENFORCE_EQ(index_type_match,
+                    true,
+                    phi::errors::InvalidArgument(
+                        "Input(Index) holds the wrong type, it holds %s, but "
+                        "desires to be %s or %s",
+                        index_type,
+                        phi::DataType::INT32,
+                        phi::DataType::INT64));
+
+  XPUType* in_grad_data = ctx.template Alloc<XPUType>(in_grad);
+  const XPUType* out_grad_data = out_grad.data<XPUType>();
+  auto in_grad_shape = phi::vectorize<int64_t>(in_grad->dims());
+  auto out_grad_shape = phi::vectorize<int64_t>(out_grad.dims());
+  auto index_shape = phi::vectorize<int64_t>(index.dims());
+
+  int r = xpu::constant(
+      ctx.x_context(), in_grad_data, in_grad->numel(), static_cast<XPUType>(0));
+
+  if (index_type == phi::DataType::INT32) {
+    const int* index_data = index.data<int>();
+    r = xpu::scatter_element(ctx.x_context(),
+                             in_grad_data,
+                             out_grad_data,
+                             index_data,
+                             in_grad_data,
+                             in_grad_shape,
+                             out_grad_shape,
+                             index_shape,
+                             1,
+                             1);
+  } else if (index_type == phi::DataType::INT64) {
+    const int64_t* index_data = index.data<int64_t>();
+    r = xpu::scatter_element(ctx.x_context(),
+                             in_grad_data,
+                             out_grad_data,
+                             index_data,
+                             in_grad_data,
+                             in_grad_shape,
+                             out_grad_shape,
+                             index_shape,
+                             1,
+                             1);
+  }
+  PADDLE_ENFORCE_XDNN_SUCCESS(r, "scatter_element");
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(
+    index_sample_grad, XPU, ALL_LAYOUT, phi::IndexSampleGradKernel, float) {}

From 00e415de15b9b45ea65c602e2697b778c4a6ee54 Mon Sep 17 00:00:00 2001
From: LoneRanger <836253168@qq.com>
Date: Mon, 15 May 2023 11:14:31 +0800
Subject: [PATCH 319/405] relocate python/paddle/fluid/regularizer.py (#53106)

* relocate regularizer.py

* fix bug

* fix bug

* fix bug

* relocate the import

* replace _regularization_coeff with coeff

* remove the L1DecayRegularizer and L2DecayRegularizer
---
 .../fleet/meta_optimizers/dgc_optimizer.py    |   4 +-
 python/paddle/fluid/__init__.py               |   2 -
 python/paddle/fluid/contrib/optimizer.py      |  10 +-
 .../fluid/layers/learning_rate_scheduler.py   |   2 +-
 python/paddle/fluid/param_attr.py             |   2 +-
 python/paddle/fluid/regularizer.py            | 300 ------------------
 .../fleet/parallel_dygraph_se_resnext.py      |   4 +-
 .../collective/fleet/test_dgc_optimizer.py    |   5 +-
 .../collective/fleet/test_fleet_amp_init.py   |   4 +-
 .../test_fleet_sharding_meta_optimizer.py     |   2 +-
 .../paddle/fluid/tests/unittests/dist_ctr.py  |   4 +-
 .../fluid/tests/unittests/dist_se_resnext.py  |   4 +-
 .../fluid/tests/unittests/seresnext_net.py    |   2 +-
 .../fluid/tests/unittests/test_adam_op.py     |   6 +-
 .../tests/unittests/test_dist_transpiler.py   |   4 +-
 .../unittests/test_fuse_all_reduce_pass.py    |   2 +-
 .../test_fuse_elewise_add_act_pass.py         |   2 +-
 .../test_fuse_relu_depthwise_conv_pass.py     |   2 +-
 .../tests/unittests/test_imperative_resnet.py |   2 +-
 .../test_imperative_resnet_sorted_gradient.py |   2 +-
 .../fluid/tests/unittests/test_momentum_op.py |  12 +-
 .../unittests/test_program_prune_backward.py  |   8 +-
 .../fluid/tests/unittests/test_regularizer.py |  27 +-
 .../fluid/tests/unittests/test_var_base.py    |   2 +-
 python/paddle/optimizer/momentum.py           |  22 +-
 python/paddle/optimizer/optimizer.py          |   5 +-
 python/paddle/regularizer.py                  | 131 +++++++-
 .../test_multi_precision_fp16_train.py        |   2 +-
 test/dygraph_to_static/darknet.py             |   2 +-
 test/dygraph_to_static/test_bmn.py            |   4 +-
 test/dygraph_to_static/test_lac.py            |  20 +-
 test/dygraph_to_static/test_mobile_net.py     |   2 +-
 test/dygraph_to_static/test_resnet.py         |   2 +-
 test/dygraph_to_static/test_se_resnet.py      |   2 +-
 test/dygraph_to_static/test_tsm.py            |   2 +-
 test/dygraph_to_static/test_yolov3.py         |   2 +-
 test/dygraph_to_static/yolov3.py              |   2 +-
 test/prim/model/test_resnet_prim_cinn.py      |   2 +-
 38 files changed, 198 insertions(+), 416 deletions(-)
 delete mode 100644 python/paddle/fluid/regularizer.py

diff --git a/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py
index 5e637de833ae2..27c1a5b651b6b 100644
--- a/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py
+++ b/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py
@@ -24,6 +24,7 @@
 from paddle.fluid.optimizer import Momentum, Optimizer
 from paddle.framework import core, in_dygraph_mode
 from paddle.nn.clip import ClipGradByNorm, append_gradient_clip_ops
+from paddle.regularizer import L1Decay, L2Decay
 from paddle.static import create_global_var
 
 
@@ -99,8 +100,7 @@ def _get_regularization_param(self, regularization):
         regular_coeff = 0.0
 
         if regularization is not None:
-            regular_coeff = regularization._regularization_coeff
-            from paddle.fluid.regularizer import L1Decay, L2Decay
+            regular_coeff = regularization._coeff
 
             if isinstance(regularization, L1Decay):
                 regular_type = 1
diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py
index a82febe3ad6d7..7151a8182cd74 100644
--- a/python/paddle/fluid/__init__.py
+++ b/python/paddle/fluid/__init__.py
@@ -58,7 +58,6 @@
 from . import optimizer
 from . import backward
 from .backward import gradients
-from . import regularizer
 from . import incubate
 from .param_attr import ParamAttr, WeightNormParamAttr
 from .data_feeder import DataFeeder
@@ -116,7 +115,6 @@
         'nets',
         'optimizer',
         'backward',
-        'regularizer',
         'LoDTensor',
         'LoDTensorArray',
         'CPUPlace',
diff --git a/python/paddle/fluid/contrib/optimizer.py b/python/paddle/fluid/contrib/optimizer.py
index c5dc5859e82f8..57f6e8114dbc5 100644
--- a/python/paddle/fluid/contrib/optimizer.py
+++ b/python/paddle/fluid/contrib/optimizer.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from paddle.fluid.optimizer import Optimizer
-from paddle.fluid.regularizer import L1DecayRegularizer
-from paddle.fluid.regularizer import L2DecayRegularizer
+from paddle.regularizer import L1Decay
+from paddle.regularizer import L2Decay
 from paddle.fluid import core
 from paddle.fluid import framework
 from paddle.fluid.framework import program_guard
@@ -117,7 +117,7 @@ def __init__(
     ):
         assert learning_rate is not None
         assert momentum is not None
-        predicate = lambda regular: isinstance(regular, L2DecayRegularizer)
+        predicate = lambda regular: isinstance(regular, L2Decay)
         py_regular = None if predicate(regularization) else regularization
         super().__init__(
             learning_rate=learning_rate,
@@ -131,9 +131,9 @@ def __init__(
         self._use_nesterov = bool(use_nesterov)
         self._regularization_method = ""
         self._regularization_coeff = 0
-        if isinstance(regularization, L2DecayRegularizer):
+        if isinstance(regularization, L2Decay):
             self._regularization_method = "l2_decay"
-            self._regularization_coeff = regularization._regularization_coeff
+            self._regularization_coeff = regularization._coeff
         self._multi_precision = multi_precision
         self._rescale_grad = rescale_grad
         self._master_weights = {}
diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py
index 65fe1d1e77b35..6995f72560183 100644
--- a/python/paddle/fluid/layers/learning_rate_scheduler.py
+++ b/python/paddle/fluid/layers/learning_rate_scheduler.py
@@ -413,7 +413,7 @@ def piecewise_decay(boundaries, values):
               optimizer = fluid.optimizer.Momentum(
                   momentum=0.9,
                   learning_rate=fluid.layers.piecewise_decay(boundaries=boundaries, values=values),
-                  regularization=fluid.regularizer.L2Decay(1e-4))
+                  regularization=paddle.regularizer.L2Decay(1e-4))
 
 
     """
diff --git a/python/paddle/fluid/param_attr.py b/python/paddle/fluid/param_attr.py
index 6fdadd7904bd4..7fa542b75d6b9 100644
--- a/python/paddle/fluid/param_attr.py
+++ b/python/paddle/fluid/param_attr.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import paddle
-from .regularizer import WeightDecayRegularizer
+from paddle.regularizer import WeightDecayRegularizer
 from paddle.fluid.data_feeder import check_type
 
 __all__ = [
diff --git a/python/paddle/fluid/regularizer.py b/python/paddle/fluid/regularizer.py
deleted file mode 100644
index 6ef5f3a5f5986..0000000000000
--- a/python/paddle/fluid/regularizer.py
+++ /dev/null
@@ -1,300 +0,0 @@
-#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-from . import framework
-from .framework import _non_static_mode, in_dygraph_mode
-from . import core
-from paddle import _C_ops, _legacy_C_ops
-
-__all__ = ['L1Decay', 'L2Decay', 'L1DecayRegularizer', 'L2DecayRegularizer']
-
-
-class WeightDecayRegularizer:
-    """Base class for weight decay regularizers
-
-    Defines the common interface of weight-decay regularizers.
-    Weight-decay regularizers are added only during the backward
-    pass for faster regularization. They add operations to the network
-    that correspond to gradient of the regularization function.
-    Users should not use this class directly, but need to use one
-    of its implementations
-    """
-
-    def __init__(self):
-        pass
-
-    def __call__(self, param, grad, block):
-        """Add corresponding weight decay operations to the network"""
-        raise NotImplementedError()
-
-    def __str__(self):
-        """Debug string"""
-        raise NotImplementedError()
-
-
-class L2DecayRegularizer(WeightDecayRegularizer):
-    r"""
-    Implement the L2 Weight Decay Regularization, which helps to prevent the model over-fitting.
-
-    It can be set in :ref:`api_fluid_ParamAttr` or ``optimizer`` (such as :ref:`api_fluid_optimizer_SGDOptimizer` ).
-    When set in ``ParamAttr`` , it only takes effect for trainable parameters in this layer. When set in
-    ``optimizer`` , it takes effect for all trainable parameters. When set together, ``ParamAttr`` has
-    higher priority than ``optimizer`` .
-
-    In the implementation, the formula of L2 Weight Decay Regularization is as follows:
-
-    .. math::
-
-        L2WeightDecay = reg\_coeff * parameter
-
-    Args:
-        regularization_coeff(float, optional): regularization coeff. Default:0.0
-
-    Examples:
-        .. code-block:: python
-
-            # Example1: set Regularizer in optimizer
-            import paddle.fluid as fluid
-            import paddle
-            paddle.enable_static()
-
-            main_prog = fluid.Program()
-            startup_prog = fluid.Program()
-            with fluid.program_guard(main_prog, startup_prog):
-                data = paddle.static.data(name='image', shape=[-1, 3, 28, 28], dtype='float32')
-                label = paddle.static.data(name='label', shape=[-1, 1], dtype='int64')
-                hidden = paddle.static.nn.fc(x=data, size=128, activation='relu')
-                prediction = paddle.static.nn.fc(x=hidden, size=10, activation='softmax')
-                loss = paddle.nn.functional.cross_entropy(
-                    input=prediction, label=label,
-                    reduction='none', use_softmax=False
-                )
-                avg_loss = paddle.mean(loss)
-            optimizer = fluid.optimizer.Adagrad(
-                learning_rate=1e-4,
-                regularization=fluid.regularizer.L2Decay(
-                    regularization_coeff=0.1))
-            optimizer.minimize(avg_loss)
-
-
-            # Example2: set Regularizer both in ParamAttr and optimizer
-            import paddle.fluid as fluid
-            import paddle
-            paddle.enable_static()
-
-            l1 = fluid.regularizer.L1Decay(regularization_coeff=0.1)
-            l2 = fluid.regularizer.L2Decay(regularization_coeff=0.1)
-            x = paddle.uniform([3,4])
-
-            # set L1 regularization in fluid.ParamAttr
-            w_param = fluid.ParamAttr(regularizer=l1)
-            hidden1 = paddle.static.nn.fc(x, 8, weight_attr=w_param)  # fc_0.w_0(L1), fc_0.b_0
-            hidden2 = paddle.static.nn.fc(hidden1, 16, weight_attr=w_param)   # fc_1.w_0(L1), fc_1.b_0
-            predict = paddle.static.nn.fc(hidden2, 32)    # fc_3.w_0, fc_3.b_0
-            avg_loss = paddle.mean(predict)
-
-            # set L2 regularization in optimizer
-            optimizer = fluid.optimizer.SGD(learning_rate=1e-4, regularization=l2)
-            optimizer.minimize(avg_loss)
-
-            # it will Print Message:
-            # Regularization of [fc_0.w_0, fc_1.w_0] have been set by ParamAttr or WeightNormParamAttr already.
-            # So, the Regularization of Optimizer will not take effect for these parameters!
-
-    """
-
-    def __init__(self, regularization_coeff=0.0):
-        assert regularization_coeff is not None
-        super().__init__()
-        self._regularization_coeff = regularization_coeff
-
-    def __call__(self, param, grad, block):
-        """Add L2 weight decay ops to network
-
-        Adds L2 weight decay ops.
-        L2WeightDecay = reg_coeff * parameter
-
-        Args:
-            param: parameter variable for which regularization is applied
-            block: block in which variable is to be created
-
-        Returns:
-            new variable for weight decay
-        """
-        assert isinstance(param, framework.Variable)
-        assert isinstance(block, framework.Block)
-
-        if framework._non_static_mode():
-            if framework.in_dygraph_mode():
-                return _C_ops.scale(
-                    param, self._regularization_coeff, 0.0, True
-                )
-            else:
-                return _legacy_C_ops.scale(
-                    param, "scale", self._regularization_coeff
-                )
-        else:
-            decay = block.create_var(
-                dtype=param.dtype, shape=param.shape, lod_level=param.lod_level
-            )
-
-            # Append Op to calculate decay
-            block.append_op(
-                type='scale',
-                inputs={"X": param},
-                outputs={"Out": decay},
-                attrs={"scale": self._regularization_coeff},
-            )
-
-            return decay
-
-    def __str__(self):
-        return "L2Decay, regularization_coeff=%f" % self._regularization_coeff
-
-
-class L1DecayRegularizer(WeightDecayRegularizer):
-    r"""
-    Implement the L1 Weight Decay Regularization, which encourages the weights to be sparse.
-
-    It can be set in :ref:`api_fluid_ParamAttr` or ``optimizer`` (such as :ref:`api_fluid_optimizer_SGDOptimizer` ).
-    When set in ``ParamAttr`` , it only takes effect for trainable parameters in this layer. When set in
-    ``optimizer`` , it takes effect for all trainable parameters. When set together, ``ParamAttr`` has
-    higher priority than ``optimizer`` .
-
-    In the implementation, the formula of L1 Weight Decay Regularization is as follows:
-
-    .. math::
-
-        L1WeightDecay = reg\_coeff * sign(parameter)
-
-    Args:
-        regularization_coeff(float, optional): regularization coeff. Default:0.0.
-
-    Examples:
-        .. code-block:: python
-
-            # Example1: set Regularizer in optimizer
-            import paddle.fluid as fluid
-            import paddle
-            paddle.enable_static()
-            main_prog = fluid.Program()
-            startup_prog = fluid.Program()
-            with fluid.program_guard(main_prog, startup_prog):
-                data = paddle.static.data(name='image', shape=[-1, 3, 28, 28], dtype='float32')
-                label = paddle.static.data(name='label', shape=[-1, 1], dtype='int64')
-                hidden = paddle.static.nn.fc(x=data, size=128, activation='relu')
-                prediction = paddle.static.nn.fc(x=hidden, size=10, activation='softmax')
-                loss = paddle.nn.functional.cross_entropy(
-                    input=prediction, label=label,
-                    reduction='none', use_softmax=False
-                )
-                avg_loss = paddle.mean(loss)
-            optimizer = fluid.optimizer.Adagrad(
-                learning_rate=1e-4,
-                regularization=fluid.regularizer.L1DecayRegularizer(
-                    regularization_coeff=0.1))
-            optimizer.minimize(avg_loss)
-
-
-            # Example2: set Regularizer both in ParamAttr and optimizer
-            import paddle.fluid as fluid
-            import paddle
-            paddle.enable_static()
-            l1 = fluid.regularizer.L1Decay(regularization_coeff=0.1)
-            l2 = fluid.regularizer.L2Decay(regularization_coeff=0.1)
-            x = paddle.uniform([3,4])
-
-            # set L1 regularization in fluid.ParamAttr
-            w_param = fluid.ParamAttr(regularizer=l1)
-            hidden1 = paddle.static.nn.fc(x, 8, weight_attr=w_param)  # fc_0.w_0(L1), fc_0.b_0
-            hidden2 = paddle.static.nn.fc(hidden1, 16, weight_attr=w_param)  # fc_1.w_0(L1), fc_1.b_0
-            predict = paddle.static.nn.fc(hidden2, 32)   # fc_3.w_0, fc_3.b_0
-            avg_loss = paddle.mean(predict)
-
-            # set L2 regularization in optimizer
-            optimizer = fluid.optimizer.SGD(learning_rate=1e-4, regularization=l2)
-            optimizer.minimize(avg_loss)
-
-            # it will Print Message:
-            # Regularization of [fc_0.w_0, fc_1.w_0] have been set by ParamAttr or WeightNormParamAttr already.
-            # So, the Regularization of Optimizer will not take effect for these parameters!
-
-    """
-
-    def __init__(self, regularization_coeff=0.0):
-        assert regularization_coeff is not None
-        super().__init__()
-        self._regularization_coeff = regularization_coeff
-
-    def __call__(self, param, grad, block):
-        """Add L1 weight decay ops to network
-
-        Adds L1 weight decay ops.
-        L1WeightDecay = reg_coeff * sign(parameter)
-
-        Args:
-            param: parameter variable for which regularization is applied
-            block: block in which variable is to be created
-
-        Returns:
-            new variable for weight decay
-        """
-        assert isinstance(param, framework.Variable)
-        assert isinstance(block, framework.Block)
-
-        if framework._non_static_mode():
-            sign = block.create_var(dtype=param.dtype, shape=param.shape)
-            decay = block.create_var(dtype=param.dtype, shape=param.shape)
-        else:
-            sign = block.create_var(
-                dtype=param.dtype, shape=param.shape, lod_level=param.lod_level
-            )
-            decay = block.create_var(
-                dtype=param.dtype, shape=param.shape, lod_level=param.lod_level
-            )
-        if in_dygraph_mode():
-            sign = _C_ops.sign(param)
-            return _C_ops.scale(sign, self._regularization_coeff, 0.0, True)
-
-        # Append sign op
-        block.append_op(type='sign', inputs={"X": param}, outputs={"Out": sign})
-
-        # Append scale op to the output of sign op
-        block.append_op(
-            type='scale',
-            inputs={"X": sign},
-            outputs={"Out": decay},
-            attrs={"scale": self._regularization_coeff},
-        )
-
-        return decay
-
-    def __str__(self):
-        return "L1Decay, regularization_coeff=%f" % self._regularization_coeff
-
-
-# We short the class name, since users will use the regulaizer with the package
-# name. The sample code:
-#
-# import paddle
-# import paddle.fluid as fluid
-#
-# hidden = paddle.static.nn.fc(...,
-#                          weight_attr=fluid.regularizer.Xavier())
-#
-# It is no need to add a `Regularizer` as the class suffix
-L1Decay = L1DecayRegularizer
-L2Decay = L2DecayRegularizer
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_se_resnext.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_se_resnext.py
index 6b0fa3fe7b9eb..0aae9d7cc63d5 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_se_resnext.py
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_se_resnext.py
@@ -61,7 +61,7 @@ def optimizer_setting(params, parameter_list=None):
                 learning_rate=lr, step_each_epoch=step, epochs=num_epochs
             ),
             momentum=momentum_rate,
-            regularization=fluid.regularizer.L2Decay(l2_decay),
+            regularization=paddle.regularizer.L2Decay(l2_decay),
             parameter_list=parameter_list,
         )
     else:
@@ -70,7 +70,7 @@ def optimizer_setting(params, parameter_list=None):
                 learning_rate=lr, step_each_epoch=step, epochs=num_epochs
             ),
             momentum=momentum_rate,
-            regularization=fluid.regularizer.L2Decay(l2_decay),
+            regularization=paddle.regularizer.L2Decay(l2_decay),
         )
 
     return optimizer
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_optimizer.py
index 5571734320bb9..3101b21257e3b 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_optimizer.py
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_optimizer.py
@@ -15,7 +15,8 @@
 import unittest
 
 import paddle
-from paddle.fluid import framework, optimizer, regularizer
+from paddle import regularizer
+from paddle.fluid import framework, optimizer
 from paddle.nn import clip
 
 paddle.enable_static()
@@ -49,7 +50,7 @@ def check_dgc_momentum_optimizer(
             optimize_attr={'learning_rate': 1.1},
             regularizer=None
             if regularization is not None
-            else regularizer.L2DecayRegularizer(2e-4),
+            else regularizer.L2Decay(2e-4),
         )
         mul_y = block.create_var(
             dtype="float32", shape=[dims[1], dims[2]], lod_level=0, name="mul.y"
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_init.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_init.py
index 1c942dbf9803f..e7263d1cf990b 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_init.py
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_init.py
@@ -66,7 +66,7 @@ def test_fleet_amp_init(self):
             optimizer = paddle.optimizer.Momentum(
                 learning_rate=0.001,
                 momentum=0.9,
-                weight_decay=fluid.regularizer.L2Decay(1e-4),
+                weight_decay=paddle.regularizer.L2Decay(1e-4),
                 multi_precision=True,
             )
 
@@ -110,7 +110,7 @@ def test_fleet_amp_meta_optimizer_init(self):
             optimizer = paddle.optimizer.Momentum(
                 learning_rate=0.001,
                 momentum=0.9,
-                weight_decay=fluid.regularizer.L2Decay(1e-4),
+                weight_decay=paddle.regularizer.L2Decay(1e-4),
                 multi_precision=True,
             )
 
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_sharding_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_sharding_meta_optimizer.py
index f69eb04ee9bb6..e42dad2b6c710 100755
--- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_sharding_meta_optimizer.py
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_sharding_meta_optimizer.py
@@ -539,7 +539,7 @@ def test_sharding_weight_decay(self):
         )
         avg_cost, strategy = self.net(train_prog, startup_prog)
         self.set_strategy(strategy, 'sharding')
-        regularization = paddle.fluid.regularizer.L2Decay(0.0001)
+        regularization = paddle.regularizer.L2Decay(0.0001)
         self.optimizer(
             avg_cost,
             strategy,
diff --git a/python/paddle/fluid/tests/unittests/dist_ctr.py b/python/paddle/fluid/tests/unittests/dist_ctr.py
index 271129ba9ee03..da3bfc796e881 100644
--- a/python/paddle/fluid/tests/unittests/dist_ctr.py
+++ b/python/paddle/fluid/tests/unittests/dist_ctr.py
@@ -114,9 +114,7 @@ def get_model(self, batch_size=2):
         regularization = None
         use_l2_decay = bool(os.getenv('USE_L2_DECAY', 0))
         if use_l2_decay:
-            regularization = fluid.regularizer.L2DecayRegularizer(
-                regularization_coeff=1e-1
-            )
+            regularization = paddle.regularizer.L2Decay(coeff=1e-1)
         use_lr_decay = bool(os.getenv('LR_DECAY', 0))
         lr = 0.0001
         if use_lr_decay:
diff --git a/python/paddle/fluid/tests/unittests/dist_se_resnext.py b/python/paddle/fluid/tests/unittests/dist_se_resnext.py
index 69e9fbd135d48..a3d525a18a84f 100644
--- a/python/paddle/fluid/tests/unittests/dist_se_resnext.py
+++ b/python/paddle/fluid/tests/unittests/dist_se_resnext.py
@@ -243,7 +243,7 @@ def get_model(self, batch_size=2, use_dgc=False):
                     boundaries=bd, values=lr
                 ),
                 momentum=0.9,
-                regularization=fluid.regularizer.L2Decay(1e-4),
+                regularization=paddle.regularizer.L2Decay(1e-4),
             )
         else:
             optimizer = (
@@ -253,7 +253,7 @@ def get_model(self, batch_size=2, use_dgc=False):
                     ),
                     momentum=0.9,
                     rampup_begin_step=0,
-                    regularization=fluid.regularizer.L2Decay(1e-4),
+                    regularization=paddle.regularizer.L2Decay(1e-4),
                 )
             )
         optimizer.minimize(avg_cost)
diff --git a/python/paddle/fluid/tests/unittests/seresnext_net.py b/python/paddle/fluid/tests/unittests/seresnext_net.py
index 029c3ceac4fa3..a2d296a1981d9 100644
--- a/python/paddle/fluid/tests/unittests/seresnext_net.py
+++ b/python/paddle/fluid/tests/unittests/seresnext_net.py
@@ -185,7 +185,7 @@ def optimizer(learning_rate=0.01):
             learning_rate=learning_rate, step_each_epoch=2, epochs=1
         ),
         momentum=0.9,
-        regularization=fluid.regularizer.L2Decay(1e-4),
+        regularization=paddle.regularizer.L2Decay(1e-4),
     )
     return optimizer
 
diff --git a/python/paddle/fluid/tests/unittests/test_adam_op.py b/python/paddle/fluid/tests/unittests/test_adam_op.py
index d4f9740208480..3196e1b2830a9 100644
--- a/python/paddle/fluid/tests/unittests/test_adam_op.py
+++ b/python/paddle/fluid/tests/unittests/test_adam_op.py
@@ -711,7 +711,7 @@ def test_adam_op_with_state_dict(self):
         )
         adam = paddle.optimizer.Adam(
             learning_rate=learning_rate,
-            weight_decay=fluid.regularizer.L2Decay(0.001),
+            weight_decay=paddle.regularizer.L2Decay(0.001),
             parameters=emb.parameters(),
         )
         lr = adam.get_lr()
@@ -976,9 +976,7 @@ def test_adam_flatten_param_grads_with_regularizer(self):
         weight_attr = paddle.ParamAttr(
             name="weight1",
             initializer=paddle.nn.initializer.Constant(value=1.0),
-            regularizer=fluid.regularizer.L1DecayRegularizer(
-                regularization_coeff=0.1
-            ),
+            regularizer=paddle.regularizer.L1Decay(coeff=0.1),
             trainable=True,
         )
         with fluid.program_guard(main):
diff --git a/python/paddle/fluid/tests/unittests/test_dist_transpiler.py b/python/paddle/fluid/tests/unittests/test_dist_transpiler.py
index e6dc8cac95a42..246887cff102f 100644
--- a/python/paddle/fluid/tests/unittests/test_dist_transpiler.py
+++ b/python/paddle/fluid/tests/unittests/test_dist_transpiler.py
@@ -574,7 +574,7 @@ def net_conf(self):
             x,
             size=1000,
             weight_attr=fluid.ParamAttr(
-                name='fc_w', regularizer=fluid.regularizer.L2Decay()
+                name='fc_w', regularizer=paddle.regularizer.L2Decay()
             ),
             bias_attr=fluid.ParamAttr(name='fc_b'),
         )
@@ -625,7 +625,7 @@ def net_conf(self):
                 boundaries=bd, values=lr
             ),
             momentum=0.9,
-            regularization=fluid.regularizer.L2Decay(1e-4),
+            regularization=paddle.regularizer.L2Decay(1e-4),
         )
         sgd_optimizer.minimize(avg_cost)
 
diff --git a/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py
index 379010c8e42a4..837742853d184 100644
--- a/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py
+++ b/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py
@@ -88,7 +88,7 @@ def compare_fuse_all_reduce_ops(
     def optimizer(self, learning_rate=1e-3):
         optimizer = fluid.optimizer.SGD(
             learning_rate=learning_rate,
-            regularization=fluid.regularizer.L2Decay(1e-3),
+            regularization=paddle.regularizer.L2Decay(1e-3),
         )
         return optimizer
 
diff --git a/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py
index cce3451ed5f61..b2590d974712f 100644
--- a/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py
+++ b/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py
@@ -38,7 +38,7 @@ def _compare_fuse_elewise_add_act_ops(self, model, use_device):
         def _optimizer(learning_rate=1e-6):
             optimizer = fluid.optimizer.SGD(
                 learning_rate=learning_rate,
-                regularization=fluid.regularizer.L2Decay(1e-6),
+                regularization=paddle.regularizer.L2Decay(1e-6),
             )
             return optimizer
 
diff --git a/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py
index 2a48de7bedda9..84772d0691042 100644
--- a/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py
+++ b/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py
@@ -87,7 +87,7 @@ def _compare(self, model, use_device, random_data=True, only_forward=False):
         def _optimizer(learning_rate=1e-6):
             optimizer = fluid.optimizer.SGD(
                 learning_rate=learning_rate,
-                regularization=fluid.regularizer.L2Decay(1e-6),
+                regularization=paddle.regularizer.L2Decay(1e-6),
             )
             return optimizer
 
diff --git a/python/paddle/fluid/tests/unittests/test_imperative_resnet.py b/python/paddle/fluid/tests/unittests/test_imperative_resnet.py
index 6f782aa43fb84..af046305ff7be 100644
--- a/python/paddle/fluid/tests/unittests/test_imperative_resnet.py
+++ b/python/paddle/fluid/tests/unittests/test_imperative_resnet.py
@@ -70,7 +70,7 @@ def optimizer_setting(params, parameter_list=None):
         #  learning_rate=fluid.layers.piecewise_decay(
         #  boundaries=bd, values=lr),
         #  momentum=0.9,
-        #  regularization=fluid.regularizer.L2Decay(1e-4))
+        #  regularization=paddle.regularizer.L2Decay(1e-4))
 
     return optimizer
 
diff --git a/python/paddle/fluid/tests/unittests/test_imperative_resnet_sorted_gradient.py b/python/paddle/fluid/tests/unittests/test_imperative_resnet_sorted_gradient.py
index 2be3e943ea2fb..bc1dbd49af300 100644
--- a/python/paddle/fluid/tests/unittests/test_imperative_resnet_sorted_gradient.py
+++ b/python/paddle/fluid/tests/unittests/test_imperative_resnet_sorted_gradient.py
@@ -66,7 +66,7 @@ def optimizer_setting(params, parameter_list=None):
         #  learning_rate=fluid.layers.piecewise_decay(
         #  boundaries=bd, values=lr),
         #  momentum=0.9,
-        #  regularization=fluid.regularizer.L2Decay(1e-4))
+        #  regularization=paddle.regularizer.L2Decay(1e-4))
 
     return optimizer
 
diff --git a/python/paddle/fluid/tests/unittests/test_momentum_op.py b/python/paddle/fluid/tests/unittests/test_momentum_op.py
index 30172c8d758b7..acb2e76af4d7d 100644
--- a/python/paddle/fluid/tests/unittests/test_momentum_op.py
+++ b/python/paddle/fluid/tests/unittests/test_momentum_op.py
@@ -687,9 +687,7 @@ def _test_momentum_dygraph_common(self, regularization):
 
     def test_momentum_dygraph_1(self):
         self._test_momentum_dygraph_common(
-            regularization=paddle.fluid.regularizer.L2Decay(
-                regularization_coeff=0.1
-            )
+            regularization=paddle.regularizer.L2Decay(coeff=0.1)
         )
 
     def test_momentum_static(self):
@@ -825,9 +823,7 @@ def __test_vs(self, place=fluid.CPUPlace()):
             learning_rate=0.01,
             momentum=0.9,
             parameter_list=linear_old.parameters(),
-            regularization=paddle.fluid.regularizer.L2Decay(
-                regularization_coeff=0.1
-            ),
+            regularization=paddle.regularizer.L2Decay(coeff=0.1),
         )
         self.__update_params(momentum=momentum_old, linear=linear_old)
 
@@ -841,9 +837,7 @@ def __test_vs(self, place=fluid.CPUPlace()):
             learning_rate=0.01,
             momentum=0.9,
             parameter_list=linear_new.parameters(),
-            regularization=paddle.fluid.regularizer.L2Decay(
-                regularization_coeff=0.1
-            ),
+            regularization=paddle.regularizer.L2Decay(coeff=0.1),
         )
         self.__update_params(momentum=momentum_new, linear=linear_new)
 
diff --git a/python/paddle/fluid/tests/unittests/test_program_prune_backward.py b/python/paddle/fluid/tests/unittests/test_program_prune_backward.py
index d163a562fc0ed..72f739eb67971 100755
--- a/python/paddle/fluid/tests/unittests/test_program_prune_backward.py
+++ b/python/paddle/fluid/tests/unittests/test_program_prune_backward.py
@@ -167,7 +167,7 @@ def test_simple_fc_net(self):
         def optimizer():
             optimizer = fluid.optimizer.SGD(
                 learning_rate=0.001,
-                regularization=fluid.regularizer.L2Decay(1e-4),
+                regularization=paddle.regularizer.L2Decay(1e-4),
             )
             return optimizer
 
@@ -183,7 +183,7 @@ def test_simple_fc_net_with_accuracy(self):
         def optimizer():
             optimizer = fluid.optimizer.SGD(
                 learning_rate=0.001,
-                regularization=fluid.regularizer.L2Decay(1e-4),
+                regularization=paddle.regularizer.L2Decay(1e-4),
             )
             return optimizer
 
@@ -199,7 +199,7 @@ def test_batchnorm_fc(self):
         def optimizer():
             optimizer = fluid.optimizer.SGD(
                 learning_rate=0.001,
-                regularization=fluid.regularizer.L2Decay(1e-4),
+                regularization=paddle.regularizer.L2Decay(1e-4),
             )
             return optimizer
 
@@ -223,7 +223,7 @@ def test_transformer(self):
         def optimizer():
             optimizer = fluid.optimizer.Adam(
                 learning_rate=0.001,
-                regularization=fluid.regularizer.L2Decay(1e-4),
+                regularization=paddle.regularizer.L2Decay(1e-4),
             )
             return optimizer
 
diff --git a/python/paddle/fluid/tests/unittests/test_regularizer.py b/python/paddle/fluid/tests/unittests/test_regularizer.py
index 21abf32d91dce..74826ab9d440c 100644
--- a/python/paddle/fluid/tests/unittests/test_regularizer.py
+++ b/python/paddle/fluid/tests/unittests/test_regularizer.py
@@ -20,12 +20,12 @@
 import numpy as np
 
 import paddle
-from paddle import fluid
-from paddle.fluid import core, framework, regularizer
+from paddle import fluid, regularizer
+from paddle.fluid import core, framework
 from paddle.fluid.backward import append_backward
 
 
-class TestL2DecayRegularizer(unittest.TestCase):
+class TestL2Decay(unittest.TestCase):
     def test_l2decay_regularizer(self):
         paddle.enable_static()
         program = framework.Program()
@@ -35,12 +35,10 @@ def test_l2decay_regularizer(self):
             shape=[5, 10],
             lod_level=0,
             name="mul.x",
-            regularizer=regularizer.L2DecayRegularizer(0.5),
+            regularizer=regularizer.L2Decay(0.5),
         )
         self.assertIsNotNone(mul_x.regularizer)
-        self.assertTrue(
-            isinstance(mul_x.regularizer, regularizer.L2DecayRegularizer)
-        )
+        self.assertTrue(isinstance(mul_x.regularizer, regularizer.L2Decay))
         mul_y = block.create_var(
             dtype="float32", shape=[10, 8], lod_level=0, name="mul.y"
         )
@@ -70,7 +68,7 @@ def test_l2decay_regularizer(self):
         self.assertEqual(block.ops[-2].type, 'scale')
 
 
-class TestL1DecayRegularizer(unittest.TestCase):
+class TestL1Decay(unittest.TestCase):
     def test_l2decay_regularizer(self):
         paddle.enable_static()
         program = framework.Program()
@@ -80,12 +78,10 @@ def test_l2decay_regularizer(self):
             shape=[5, 10],
             lod_level=0,
             name="mul.x",
-            regularizer=regularizer.L1DecayRegularizer(0.5),
+            regularizer=regularizer.L1Decay(0.5),
         )
         self.assertIsNotNone(mul_x.regularizer)
-        self.assertTrue(
-            isinstance(mul_x.regularizer, regularizer.L1DecayRegularizer)
-        )
+        self.assertTrue(isinstance(mul_x.regularizer, regularizer.L1Decay))
         mul_y = block.create_var(
             dtype="float32", shape=[10, 8], lod_level=0, name="mul.y"
         )
@@ -208,7 +204,8 @@ def check_l2decay_regularizer(self, place, model):
             avg_cost = model(data, label, self.word_len)
 
             optimizer = fluid.optimizer.Adagrad(
-                learning_rate=0.1, regularization=fluid.regularizer.L2Decay(1.0)
+                learning_rate=0.1,
+                regularization=paddle.regularizer.L2Decay(1.0),
             )
             optimizer.minimize(avg_cost)
             param_sum = self.run_program(place, [data, label])
@@ -265,8 +262,8 @@ def test_l2(self):
                 )
 
     def test_repeated_regularization(self):
-        l1 = fluid.regularizer.L1Decay(regularization_coeff=0.1)
-        l2 = fluid.regularizer.L2Decay(regularization_coeff=0.01)
+        l1 = paddle.regularizer.L1Decay(coeff=0.1)
+        l2 = paddle.regularizer.L2Decay(coeff=0.01)
         fc_param_attr = paddle.ParamAttr(
             regularizer=paddle.regularizer.L1Decay()
         )
diff --git a/python/paddle/fluid/tests/unittests/test_var_base.py b/python/paddle/fluid/tests/unittests/test_var_base.py
index 5fc9914fa1e88..f96f25281d800 100644
--- a/python/paddle/fluid/tests/unittests/test_var_base.py
+++ b/python/paddle/fluid/tests/unittests/test_var_base.py
@@ -1127,7 +1127,7 @@ def _assert_to_static(self, var_base, static_var, is_param=False):
                 )
                 self.assertTrue(
                     isinstance(
-                        static_var.regularizer, fluid.regularizer.L1Decay
+                        static_var.regularizer, paddle.regularizer.L1Decay
                     )
                 )
         else:
diff --git a/python/paddle/optimizer/momentum.py b/python/paddle/optimizer/momentum.py
index b0801a419d7ea..f46cde0b09944 100644
--- a/python/paddle/optimizer/momentum.py
+++ b/python/paddle/optimizer/momentum.py
@@ -17,7 +17,7 @@
 import paddle
 from paddle import _C_ops
 from paddle.fluid.framework import in_dygraph_mode
-from paddle.fluid.regularizer import L2DecayRegularizer
+from paddle.regularizer import L2Decay
 
 from ..fluid import core, framework
 from .optimizer import Optimizer
@@ -136,9 +136,7 @@ def __init__(
         if momentum is None:
             raise ValueError("momentum is not set")
 
-        predicate = lambda regular: isinstance(
-            regular, (L2DecayRegularizer, float)
-        )
+        predicate = lambda regular: isinstance(regular, (L2Decay, float))
         if isinstance(parameters, list):
             if isinstance(parameters[0], dict):
                 for param_group in parameters:
@@ -192,9 +190,9 @@ def _update_regularization(self, weight_decay):
         reg_method = ""
         reg_coeff = 0.0
 
-        if isinstance(weight_decay, L2DecayRegularizer):
+        if isinstance(weight_decay, L2Decay):
             reg_method = "l2_decay"
-            reg_coeff = weight_decay._regularization_coeff
+            reg_coeff = weight_decay._coeff
         if isinstance(weight_decay, float):
             reg_method = "l2_decay"
             reg_coeff = weight_decay
@@ -237,7 +235,7 @@ def _create_regularization_of_grad(self, param, grad, regularization=None):
         # If ParamAttr is set to L2Decay, we skip doing regularization here. And then we fused
         # L2Decay with momentum which can refer to _append_optimize_op below.
         if hasattr(param, 'regularizer') and isinstance(
-            param.regularizer, L2DecayRegularizer
+            param.regularizer, L2Decay
         ):
             return grad
         return super()._create_regularization_of_grad(
@@ -260,9 +258,9 @@ def _append_optimize_op(self, block, param_and_grad):
         regularization_coeff = self._regularization_coeff
         if hasattr(param, 'regularizer'):
             # we skip param's l2decay before, so fuse it with momentum here.
-            if isinstance(param.regularizer, L2DecayRegularizer):
+            if isinstance(param.regularizer, L2Decay):
                 regularization_method = "l2_decay"
-                regularization_coeff = param.regularizer._regularization_coeff
+                regularization_coeff = param.regularizer._coeff
             # the param's regularization has been done before, we avoid do l2decay in momentum.
             elif param.regularizer is not None:
                 regularization_method = ""
@@ -348,11 +346,9 @@ def _multi_tensor_init(self, target_block, parameters, param_group_idx):
             regularization_coeff = self._regularization_coeff
             if hasattr(param, 'regularizer'):
                 # we skip param's l2decay before, so fuse it with momentum here.
-                if isinstance(param.regularizer, L2DecayRegularizer):
+                if isinstance(param.regularizer, L2Decay):
                     regularization_method = "l2_decay"
-                    regularization_coeff = (
-                        param.regularizer._regularization_coeff
-                    )
+                    regularization_coeff = param.regularizer._coeff
                 elif param.regularizer is not None:
                     regularization_method = ""
                     regularization_coeff = 0.0
diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py
index 99d9ce3140566..ce271a7169b8e 100644
--- a/python/paddle/optimizer/optimizer.py
+++ b/python/paddle/optimizer/optimizer.py
@@ -30,6 +30,7 @@
     in_dygraph_mode,
     name_scope,
 )
+from paddle.regularizer import L2Decay
 
 from ..fluid import framework, unique_name
 from ..fluid.backward import _get_no_grad_set_name, append_backward
@@ -224,8 +225,6 @@ def __init__(
                     "'grad_clip' should be an instance of GradientClipBase's derived class"
                 )
         if isinstance(weight_decay, float):
-            from ..fluid.regularizer import L2Decay
-
             self.regularization = L2Decay(weight_decay)
         else:
             self.regularization = weight_decay
@@ -1571,8 +1570,6 @@ def _add_param_group(self, param_group):
         for param in param_group['params']:
             weight_decay = param_group['weight_decay']
             if isinstance(weight_decay, float):
-                from ..fluid.regularizer import L2Decay
-
                 regularization = L2Decay(weight_decay)
             else:
                 regularization = weight_decay
diff --git a/python/paddle/regularizer.py b/python/paddle/regularizer.py
index d58145ed7ffd5..8c245f99b8b87 100644
--- a/python/paddle/regularizer.py
+++ b/python/paddle/regularizer.py
@@ -12,12 +12,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
+from paddle import _C_ops, _legacy_C_ops
+from paddle.fluid import framework
+from paddle.fluid.framework import in_dygraph_mode
+
 __all__ = ['L1Decay', 'L2Decay']
 
-from paddle import fluid
+
+class WeightDecayRegularizer:
+    """Base class for weight decay regularizers
+
+    Defines the common interface of weight-decay regularizers.
+    Weight-decay regularizers are added only during the backward
+    pass for faster regularization. They add operations to the network
+    that correspond to gradient of the regularization function.
+    Users should not use this class directly, but need to use one
+    of its implementations
+    """
+
+    def __init__(self):
+        pass
+
+    def __call__(self, param, grad, block):
+        """Add corresponding weight decay operations to the network"""
+        raise NotImplementedError()
+
+    def __str__(self):
+        """Debug string"""
+        raise NotImplementedError()
 
 
-class L1Decay(fluid.regularizer.L1Decay):
+class L1Decay(WeightDecayRegularizer):
     r"""
     Implement the L1 Weight Decay Regularization, which encourages the weights to be sparse.
 
@@ -76,10 +102,58 @@ class L1Decay(fluid.regularizer.L1Decay):
     """
 
     def __init__(self, coeff=0.0):
-        super().__init__(coeff)
-
-
-class L2Decay(fluid.regularizer.L2Decay):
+        assert coeff is not None
+        super().__init__()
+        self._coeff = coeff
+
+    def __call__(self, param, grad, block):
+        """Add L1 weight decay ops to network
+
+        Adds L1 weight decay ops.
+        L1WeightDecay = reg_coeff * sign(parameter)
+
+        Args:
+            param: parameter variable for which regularization is applied
+            block: block in which variable is to be created
+
+        Returns:
+            new variable for weight decay
+        """
+        assert isinstance(param, framework.Variable)
+        assert isinstance(block, framework.Block)
+
+        if framework._non_static_mode():
+            sign = block.create_var(dtype=param.dtype, shape=param.shape)
+            decay = block.create_var(dtype=param.dtype, shape=param.shape)
+        else:
+            sign = block.create_var(
+                dtype=param.dtype, shape=param.shape, lod_level=param.lod_level
+            )
+            decay = block.create_var(
+                dtype=param.dtype, shape=param.shape, lod_level=param.lod_level
+            )
+        if in_dygraph_mode():
+            sign = _C_ops.sign(param)
+            return _C_ops.scale(sign, self._coeff, 0.0, True)
+
+        # Append sign op
+        block.append_op(type='sign', inputs={"X": param}, outputs={"Out": sign})
+
+        # Append scale op to the output of sign op
+        block.append_op(
+            type='scale',
+            inputs={"X": sign},
+            outputs={"Out": decay},
+            attrs={"scale": self._coeff},
+        )
+
+        return decay
+
+    def __str__(self):
+        return "L1Decay, coeff=%f" % self._coeff
+
+
+class L2Decay(WeightDecayRegularizer):
     r"""
     Implement the L2 Weight Decay Regularization, which helps to prevent the model over-fitting.
 
@@ -97,7 +171,7 @@ class L2Decay(fluid.regularizer.L2Decay):
         loss = 0.5 * coeff * reduce\_sum(square(x))
 
     Args:
-        regularization_coeff(float, optional): regularization coeff. Default:0.0
+        coeff(float, optional): regularization coeff. Default:0.0
 
     Examples:
         .. code-block:: python
@@ -137,4 +211,45 @@ class L2Decay(fluid.regularizer.L2Decay):
     """
 
     def __init__(self, coeff=0.0):
-        super().__init__(coeff)
+        assert coeff is not None
+        super().__init__()
+        self._coeff = coeff
+
+    def __call__(self, param, grad, block):
+        """Add L2 weight decay ops to network
+
+        Adds L2 weight decay ops.
+        L2WeightDecay = reg_coeff * parameter
+
+        Args:
+            param: parameter variable for which regularization is applied
+            block: block in which variable is to be created
+
+        Returns:
+            new variable for weight decay
+        """
+        assert isinstance(param, framework.Variable)
+        assert isinstance(block, framework.Block)
+
+        if framework._non_static_mode():
+            if framework.in_dygraph_mode():
+                return _C_ops.scale(param, self._coeff, 0.0, True)
+            else:
+                return _legacy_C_ops.scale(param, "scale", self._coeff)
+        else:
+            decay = block.create_var(
+                dtype=param.dtype, shape=param.shape, lod_level=param.lod_level
+            )
+
+            # Append Op to calculate decay
+            block.append_op(
+                type='scale',
+                inputs={"X": param},
+                outputs={"Out": decay},
+                attrs={"scale": self._coeff},
+            )
+
+            return decay
+
+    def __str__(self):
+        return "L2Decay, coeff=%f" % self._coeff
diff --git a/test/contrib/test_multi_precision_fp16_train.py b/test/contrib/test_multi_precision_fp16_train.py
index 3672cd2427a86..64dc91585cb98 100644
--- a/test/contrib/test_multi_precision_fp16_train.py
+++ b/test/contrib/test_multi_precision_fp16_train.py
@@ -134,7 +134,7 @@ def train(use_pure_fp16=True, use_nesterov=False, optimizer=""):
                 learning_rate=0.001,
                 momentum=0.9,
                 use_nesterov=use_nesterov,
-                weight_decay=fluid.regularizer.L2Decay(1e-4),
+                weight_decay=paddle.regularizer.L2Decay(1e-4),
                 multi_precision=use_pure_fp16,
             )
 
diff --git a/test/dygraph_to_static/darknet.py b/test/dygraph_to_static/darknet.py
index 111e564f8c182..fa4a391392cfb 100644
--- a/test/dygraph_to_static/darknet.py
+++ b/test/dygraph_to_static/darknet.py
@@ -14,8 +14,8 @@
 
 import paddle
 from paddle.fluid.param_attr import ParamAttr
-from paddle.fluid.regularizer import L2Decay
 from paddle.nn import BatchNorm
+from paddle.regularizer import L2Decay
 
 
 class ConvBNLayer(paddle.nn.Layer):
diff --git a/test/dygraph_to_static/test_bmn.py b/test/dygraph_to_static/test_bmn.py
index 925da4140a239..57aca8f9ada7f 100644
--- a/test/dygraph_to_static/test_bmn.py
+++ b/test/dygraph_to_static/test_bmn.py
@@ -451,9 +451,7 @@ def optimizer(cfg, parameter_list):
     optimizer = fluid.optimizer.Adam(
         fluid.layers.piecewise_decay(boundaries=bd, values=lr),
         parameter_list=parameter_list,
-        regularization=fluid.regularizer.L2DecayRegularizer(
-            regularization_coeff=l2_weight_decay
-        ),
+        regularization=paddle.regularizer.L2Decay(coeff=l2_weight_decay),
     )
     return optimizer
 
diff --git a/test/dygraph_to_static/test_lac.py b/test/dygraph_to_static/test_lac.py
index 9e854f9dda09f..933fc00d73521 100644
--- a/test/dygraph_to_static/test_lac.py
+++ b/test/dygraph_to_static/test_lac.py
@@ -100,9 +100,7 @@ def __init__(self, input_dim, grnn_hidden_dim, init_bound, h_0=None):
                 initializer=paddle.nn.initializer.Uniform(
                     low=-init_bound, high=init_bound
                 ),
-                regularizer=fluid.regularizer.L2DecayRegularizer(
-                    regularization_coeff=1e-4
-                ),
+                regularizer=paddle.regularizer.L2Decay(coeff=1e-4),
             ),
         )
 
@@ -113,9 +111,7 @@ def __init__(self, input_dim, grnn_hidden_dim, init_bound, h_0=None):
                 initializer=paddle.nn.initializer.Uniform(
                     low=-init_bound, high=init_bound
                 ),
-                regularizer=fluid.regularizer.L2DecayRegularizer(
-                    regularization_coeff=1e-4
-                ),
+                regularizer=paddle.regularizer.L2Decay(coeff=1e-4),
             ),
         )
 
@@ -126,9 +122,7 @@ def __init__(self, input_dim, grnn_hidden_dim, init_bound, h_0=None):
                 initializer=paddle.nn.initializer.Uniform(
                     low=-init_bound, high=init_bound
                 ),
-                regularizer=fluid.regularizer.L2DecayRegularizer(
-                    regularization_coeff=1e-4
-                ),
+                regularizer=paddle.regularizer.L2Decay(coeff=1e-4),
             ),
         )
 
@@ -140,9 +134,7 @@ def __init__(self, input_dim, grnn_hidden_dim, init_bound, h_0=None):
                 initializer=paddle.nn.initializer.Uniform(
                     low=-init_bound, high=init_bound
                 ),
-                regularizer=fluid.regularizer.L2DecayRegularizer(
-                    regularization_coeff=1e-4
-                ),
+                regularizer=paddle.regularizer.L2Decay(coeff=1e-4),
             ),
         )
 
@@ -417,9 +409,7 @@ def __init__(self, args, length=None):
                 initializer=paddle.nn.initializer.Uniform(
                     low=-self.init_bound, high=self.init_bound
                 ),
-                regularizer=fluid.regularizer.L2DecayRegularizer(
-                    regularization_coeff=1e-4
-                ),
+                regularizer=paddle.regularizer.L2Decay(coeff=1e-4),
             ),
         )
 
diff --git a/test/dygraph_to_static/test_mobile_net.py b/test/dygraph_to_static/test_mobile_net.py
index 48041926604e7..7f0efd67620be 100644
--- a/test/dygraph_to_static/test_mobile_net.py
+++ b/test/dygraph_to_static/test_mobile_net.py
@@ -448,7 +448,7 @@ def create_optimizer(args, parameter_list):
     optimizer = fluid.optimizer.Momentum(
         learning_rate=args.lr,
         momentum=args.momentum_rate,
-        regularization=fluid.regularizer.L2Decay(args.l2_decay),
+        regularization=paddle.regularizer.L2Decay(args.l2_decay),
         parameter_list=parameter_list,
     )
 
diff --git a/test/dygraph_to_static/test_resnet.py b/test/dygraph_to_static/test_resnet.py
index 7738bcee08e85..9fcc1a803f822 100644
--- a/test/dygraph_to_static/test_resnet.py
+++ b/test/dygraph_to_static/test_resnet.py
@@ -48,7 +48,7 @@ def optimizer_setting(parameter_list=None):
     optimizer = fluid.optimizer.Momentum(
         learning_rate=base_lr,
         momentum=momentum_rate,
-        regularization=fluid.regularizer.L2Decay(l2_decay),
+        regularization=paddle.regularizer.L2Decay(l2_decay),
         parameter_list=parameter_list,
     )
 
diff --git a/test/dygraph_to_static/test_se_resnet.py b/test/dygraph_to_static/test_se_resnet.py
index b6c90418d2910..05b2f6a517039 100644
--- a/test/dygraph_to_static/test_se_resnet.py
+++ b/test/dygraph_to_static/test_se_resnet.py
@@ -82,7 +82,7 @@ def optimizer_setting(params, parameter_list):
             learning_rate=lr, step_each_epoch=step, epochs=num_epochs
         ),
         momentum=momentum_rate,
-        regularization=fluid.regularizer.L2Decay(l2_decay),
+        regularization=paddle.regularizer.L2Decay(l2_decay),
         parameter_list=parameter_list,
     )
 
diff --git a/test/dygraph_to_static/test_tsm.py b/test/dygraph_to_static/test_tsm.py
index c81ec1ba1a287..d0892a50fdd35 100644
--- a/test/dygraph_to_static/test_tsm.py
+++ b/test/dygraph_to_static/test_tsm.py
@@ -281,7 +281,7 @@ def create_optimizer(cfg, params):
     optimizer = fluid.optimizer.Momentum(
         learning_rate=fluid.layers.piecewise_decay(boundaries=bd, values=lr),
         momentum=momentum,
-        regularization=fluid.regularizer.L2Decay(l2_weight_decay),
+        regularization=paddle.regularizer.L2Decay(l2_weight_decay),
         parameter_list=params,
     )
 
diff --git a/test/dygraph_to_static/test_yolov3.py b/test/dygraph_to_static/test_yolov3.py
index 442ce47928b2d..640fe7c3fdade 100644
--- a/test/dygraph_to_static/test_yolov3.py
+++ b/test/dygraph_to_static/test_yolov3.py
@@ -107,7 +107,7 @@ def train(to_static):
 
         optimizer = fluid.optimizer.Momentum(
             learning_rate=lr,
-            regularization=fluid.regularizer.L2Decay(cfg.weight_decay),
+            regularization=paddle.regularizer.L2Decay(cfg.weight_decay),
             momentum=cfg.momentum,
             parameter_list=model.parameters(),
         )
diff --git a/test/dygraph_to_static/yolov3.py b/test/dygraph_to_static/yolov3.py
index fc121c47534ce..b13a7eb8e16ea 100644
--- a/test/dygraph_to_static/yolov3.py
+++ b/test/dygraph_to_static/yolov3.py
@@ -20,8 +20,8 @@
 import paddle
 from paddle import _legacy_C_ops, fluid
 from paddle.fluid.param_attr import ParamAttr
-from paddle.fluid.regularizer import L2Decay
 from paddle.jit.api import to_static
+from paddle.regularizer import L2Decay
 
 
 class AttrDict(dict):
diff --git a/test/prim/model/test_resnet_prim_cinn.py b/test/prim/model/test_resnet_prim_cinn.py
index 5e91de52c7ae6..75238a8192c08 100644
--- a/test/prim/model/test_resnet_prim_cinn.py
+++ b/test/prim/model/test_resnet_prim_cinn.py
@@ -75,7 +75,7 @@ def optimizer_setting(parameter_list=None):
     optimizer = fluid.optimizer.Momentum(
         learning_rate=base_lr,
         momentum=momentum_rate,
-        regularization=fluid.regularizer.L2Decay(l2_decay),
+        regularization=paddle.regularizer.L2Decay(l2_decay),
         parameter_list=parameter_list,
     )
 

From 359f43a9340072ccb494f297f55bfc4bd2e13171 Mon Sep 17 00:00:00 2001
From: Aurelius84 <zhangliujie@baidu.com>
Date: Mon, 15 May 2023 11:33:52 +0800
Subject: [PATCH 320/405] [CI]Fix test_bert_primm_cinn gt loss value (#53796)

---
 test/prim/model/test_bert_prim_cinn.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/test/prim/model/test_bert_prim_cinn.py b/test/prim/model/test_bert_prim_cinn.py
index 3170dba893cf1..ea475730d1d15 100644
--- a/test/prim/model/test_bert_prim_cinn.py
+++ b/test/prim/model/test_bert_prim_cinn.py
@@ -34,16 +34,16 @@
 
 
 DY2ST_PRIM_CINN_GT = [
-    10.976988792419434,
-    10.345282554626465,
-    10.33430004119873,
-    10.275235176086426,
-    10.22808837890625,
-    10.200264930725098,
-    10.161019325256348,
-    10.10872745513916,
-    10.121539115905762,
-    9.990568161010742,
+    11.086677551269531,
+    10.357963562011719,
+    10.33290958404541,
+    10.270476341247559,
+    10.230023384094238,
+    10.196759223937988,
+    10.153482437133789,
+    10.098485946655273,
+    10.119072914123535,
+    9.993000984191895,
 ]
 
 

From a822a0845f8b0e562e6e9a6349da302c1d92c4cf Mon Sep 17 00:00:00 2001
From: risemeup1 <62429225+risemeup1@users.noreply.github.com>
Date: Mon, 15 May 2023 13:03:14 +0800
Subject: [PATCH 321/405] bug fixes (#53798)

---
 paddle/fluid/framework/CMakeLists.txt | 2 +-
 paddle/phi/capi/lib/CMakeLists.txt    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt
index 3cab75f0971dd..c9ca72c051227 100755
--- a/paddle/fluid/framework/CMakeLists.txt
+++ b/paddle/fluid/framework/CMakeLists.txt
@@ -1099,7 +1099,7 @@ endif()
 cc_library(
   prune
   SRCS prune.cc
-  DEPS framework_proto)
+  DEPS framework_proto auto_parallel_proto)
 cc_test(
   prune_test
   SRCS prune_test.cc
diff --git a/paddle/phi/capi/lib/CMakeLists.txt b/paddle/phi/capi/lib/CMakeLists.txt
index 185de92cd20ec..60afb74a6d451 100644
--- a/paddle/phi/capi/lib/CMakeLists.txt
+++ b/paddle/phi/capi/lib/CMakeLists.txt
@@ -6,7 +6,7 @@ cc_library(
 cc_library(
   phi_c_device_context
   SRCS c_device_context.cc
-  DEPS auto_parallel_proto phi_backends)
+  DEPS phi_backends)
 
 cc_library(
   phi_c_int_array

From 515297121a6494f3bf66d1ef233f437dacea36e2 Mon Sep 17 00:00:00 2001
From: Ghost Screaming <mofengshenjieII@163.com>
Date: Mon, 15 May 2023 13:08:44 +0800
Subject: [PATCH 322/405] Fix bug of hybrid_parallel_optimizer, amp use
 scaler.minimize(), (#53773)

however it can't deal with group of parameter_list of dict.
---
 .../dygraph_optimizer/hybrid_parallel_optimizer.py            | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py
index beb3fa0ff6875..a553a8a5d9cc2 100755
--- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py
+++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py
@@ -432,7 +432,9 @@ def minimize(
         # minimize does not support parameters in the form of param_group,
         # so no need use _obtain_optimizer_parameters_list
         parameter_list = (
-            parameters if parameters else self._inner_opt._parameter_list
+            parameters
+            if parameters
+            else _obtain_optimizer_parameters_list(self._inner_opt)
         )
 
         # Here sharding should use global parameter list

From b428e8f35c9809697e296d0c17eabf1a15a35899 Mon Sep 17 00:00:00 2001
From: YuanRisheng <yuanrisheng@baidu.com>
Date: Mon, 15 May 2023 14:03:30 +0800
Subject: [PATCH 323/405] [PHI]Add Filter for get_kernel_signatures.py (#53760)

* delete log

* filter some kernel signature
---
 paddle/phi/tools/get_kernel_signatures.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/paddle/phi/tools/get_kernel_signatures.py b/paddle/phi/tools/get_kernel_signatures.py
index ccc3c78a33c50..33af4213b8337 100644
--- a/paddle/phi/tools/get_kernel_signatures.py
+++ b/paddle/phi/tools/get_kernel_signatures.py
@@ -190,8 +190,6 @@ def get_kernel_signatures():
     KernelSignatureSearcher.filter["kernel_name"].
     """
     Paddle_path = osp.abspath(osp.join(osp.dirname(__file__), '../../..'))
-    if Paddle_path.split('/')[-1] != 'Paddle':
-        raise ValueError('Paddle path error.')
     build_path = osp.join(Paddle_path, 'build')
     os.makedirs(build_path, exist_ok=True)
     KernelSignatureSearcher.build_path = build_path
@@ -204,8 +202,9 @@ def get_kernel_signatures():
     # subdirs, so we need to search them separately.
     independent_subdir = [
         'fusion',
-        'legacy',
-        'selected_rows',
+        # Currently, we need filter legacy dir and selected_rows dir.
+        # 'legacy',
+        # 'selected_rows',
         'sparse',
         'strings',
     ]

From 3d4d7c1951626ae3738d8183978858b45c17e654 Mon Sep 17 00:00:00 2001
From: Aurelius84 <zhangliujie@baidu.com>
Date: Mon, 15 May 2023 14:46:59 +0800
Subject: [PATCH 324/405] [UnitTest]Fix deprecate fluid.regularizer
 test=document_fix (#53805)

---
 test/prim/model/test_resnet_cinn.py | 2 +-
 test/prim/model/test_resnet_prim.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/prim/model/test_resnet_cinn.py b/test/prim/model/test_resnet_cinn.py
index db20dc5a605e1..963b311b5b098 100644
--- a/test/prim/model/test_resnet_cinn.py
+++ b/test/prim/model/test_resnet_cinn.py
@@ -75,7 +75,7 @@ def optimizer_setting(parameter_list=None):
     optimizer = fluid.optimizer.Momentum(
         learning_rate=base_lr,
         momentum=momentum_rate,
-        regularization=fluid.regularizer.L2Decay(l2_decay),
+        regularization=paddle.regularizer.L2Decay(l2_decay),
         parameter_list=parameter_list,
     )
 
diff --git a/test/prim/model/test_resnet_prim.py b/test/prim/model/test_resnet_prim.py
index 79c169c2272b3..d55f795286a4e 100644
--- a/test/prim/model/test_resnet_prim.py
+++ b/test/prim/model/test_resnet_prim.py
@@ -75,7 +75,7 @@ def optimizer_setting(parameter_list=None):
     optimizer = fluid.optimizer.Momentum(
         learning_rate=base_lr,
         momentum=momentum_rate,
-        regularization=fluid.regularizer.L2Decay(l2_decay),
+        regularization=paddle.regularizer.L2Decay(l2_decay),
         parameter_list=parameter_list,
     )
 

From 3d6bd6a4fc655c6d73dfea61bca362e825d06e5f Mon Sep 17 00:00:00 2001
From: Charles-hit <56987902+Charles-hit@users.noreply.github.com>
Date: Mon, 15 May 2023 14:54:26 +0800
Subject: [PATCH 325/405] add check ops for prim (#52302)

* add check ops for prim

* fix pow and concat composite registration

* modify log

* add note and remove useless code

* remove useless code

* modify program to check

* remove useless note
---
 .../fluid/tests/unittests/prim_op_test.py     | 54 +++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/python/paddle/fluid/tests/unittests/prim_op_test.py b/python/paddle/fluid/tests/unittests/prim_op_test.py
index 6fa612ccba4b1..22f050fc8f00b 100644
--- a/python/paddle/fluid/tests/unittests/prim_op_test.py
+++ b/python/paddle/fluid/tests/unittests/prim_op_test.py
@@ -605,6 +605,11 @@ def check_static_comp(self):
             )
             ret = flatten(_as_list(self.public_python_api(*args)))
             primapi.to_prim(main_program.blocks)
+        # ensure the operator not in program if check_prim is True
+        forward_ops = [op.type for op in main_program.blocks[0].ops]
+        assert self.op_type not in forward_ops, (
+            "%s shouldn't appear in program when check_prim is True"
+        ) % (self.op_type)
         exe = paddle.static.Executor(self.place)
         exe.run(startup_program)
         ret = exe.run(main_program, feed=feed, fetch_list=ret)
@@ -675,6 +680,16 @@ def check_jit_comp(self):
         )
         net = PrimNet(self.public_python_api)
         net = apply_to_static(net, False)
+        # ensure the operator not in program if check_prim is True
+        forward_ops = [
+            op.type
+            for op in net.forward.get_concrete_program(args)[1]
+            .forward_program.block(0)
+            .ops
+        ]
+        assert self.op_type not in forward_ops, (
+            "%s shouldn't appear in program when check_prim is True"
+        ) % (self.op_type)
         ret = flatten(_as_list(net(args)))
         ret = paddle.utils.map_structure(lambda x: x.numpy(), ret)
         if OpTestUtils.is_bfloat16_type(self.dtype):
@@ -761,6 +776,16 @@ def check_jit_comp_with_cinn(self):
         net = apply_to_static(
             net, core.is_compiled_with_cinn() and self.enable_cinn
         )
+        # check the operator not in program if check prim is True
+        forward_ops = [
+            op.type
+            for op in net.forward.get_concrete_program(args)[1]
+            .forward_program.block(0)
+            .ops
+        ]
+        assert self.op_type not in forward_ops, (
+            "%s shouldn't appear in program when check_prim is True"
+        ) % (self.op_type)
         ret = flatten(_as_list(net(args)))
         ret = paddle.utils.map_structure(lambda x: x.numpy(), ret)
         if OpTestUtils.is_bfloat16_type(self.dtype):
@@ -1055,6 +1080,12 @@ def check_static_comp(self):
                 var_dict={**inputs_dict, **outputs_dict}
             )
             ret = paddle.static.gradients(ys, xs, vs, no_grad_set=no_grad_vars)
+        # check the backward operator not in program when check_prim is True
+        ops = [op.type for op in main_program.blocks[0].ops]
+        backward_op_type = self.op_type + "_grad"
+        assert backward_op_type not in ops, (
+            "%s shouldn't appear in program when check_prim is True"
+        ) % (backward_op_type)
         exe = paddle.static.Executor(self.place)
         exe.run(startup_program)
         actual_ret = exe.run(main_program, feed=feed, fetch_list=ret)
@@ -1140,6 +1171,17 @@ def check_jit_comp(self):
         )
         net = PrimNet(self.public_python_api)
         net = apply_to_static(net, False)
+        # check the backward operator not in program when check_prim is True
+        ops = [
+            op.type
+            for op in net.forward.get_concrete_program(args)[1]
+            .backward_program.block(0)
+            .ops
+        ]
+        backward_op_type = self.op_type + "_grad"
+        assert backward_op_type not in ops, (
+            "%s shouldn't appear in program when check_prim is True"
+        ) % (backward_op_type)
         out = _as_list(net(args))
         if hasattr(self.op_test, "python_out_sig"):
             outputs_sig = self.op_test.python_out_sig
@@ -1259,6 +1301,18 @@ def check_jit_comp_with_cinn(self):
         net = apply_to_static(
             net, core.is_compiled_with_cinn() and self.enable_cinn
         )
+        # check the backward operator not in program when check_prim is True
+        ops = [
+            op.type
+            for op in net.forward.get_concrete_program(args)[1]
+            .backward_program.block(0)
+            .ops
+        ]
+        backward_op_type = self.op_type + "_grad"
+        assert backward_op_type not in ops, (
+            "%s shouldn't appear in program when check_prim is True"
+        ) % (backward_op_type)
+
         out = _as_list(net(args))
         if hasattr(self.op_test, "python_out_sig"):
             outputs_sig = self.op_test.python_out_sig

From a9c3e32d81bb8bfce75e497a05a93643425a8000 Mon Sep 17 00:00:00 2001
From: zyfncg <zhangyunfei07@baidu.com>
Date: Mon, 15 May 2023 14:56:54 +0800
Subject: [PATCH 326/405] fix bug of test_pad_op for cinn (#53772)

---
 .../prim/api/composite_backward/composite_backward_api.h  | 8 ++++----
 python/paddle/fluid/tests/unittests/test_pad_op.py        | 1 -
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
index cd88139af23a7..6eb366dc355ed 100644
--- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
+++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
@@ -1949,15 +1949,15 @@ void pad_grad(const Tensor& input,
     size_t rank = input.dims().size();
     auto out_dims = out_grad.dims();
 
-    std::vector<int> starts(rank, 0);
+    std::vector<int64_t> starts(rank, 0);
     std::vector<int64_t> ends(rank, 0);
     std::vector<int64_t> axes(rank, 0);
     std::vector<int64_t> infer_flags(rank, 1);
     std::vector<int64_t> decrease_axis({});
     for (size_t i = 0; i < rank; ++i) {
-      starts.push_back(static_cast<int>(paddings[2 * i]));
-      ends.push_back(static_cast<int64_t>(out_dims[i] - paddings[2 * i + 1]));
-      axes.push_back(i);
+      starts[i] = static_cast<int64_t>(paddings[2 * i]);
+      ends[i] = static_cast<int64_t>(out_dims[i] - paddings[2 * i + 1]);
+      axes[i] = i;
     }
     auto out_tmp =
         slice<T>(out_grad, axes, starts, ends, infer_flags, decrease_axis);
diff --git a/python/paddle/fluid/tests/unittests/test_pad_op.py b/python/paddle/fluid/tests/unittests/test_pad_op.py
index 5fd186338444d..a25956cfb0697 100644
--- a/python/paddle/fluid/tests/unittests/test_pad_op.py
+++ b/python/paddle/fluid/tests/unittests/test_pad_op.py
@@ -51,7 +51,6 @@ def setUp(self):
         }
         self.prim_op_type = "prim"
         self.public_python_api = pad_wrapper
-        self.enable_cinn = False
 
     def get_dtype(self):
         return np.float64

From 96188fc160c5241c9fbb32eec3f0e7f5fd39edab Mon Sep 17 00:00:00 2001
From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com>
Date: Mon, 15 May 2023 15:37:57 +0800
Subject: [PATCH 327/405] remove some [-Wunused-paramter]warning (#53681)

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop
---
 .../fluid/operators/collective/send_v2_op.h   |  2 +-
 .../fused/mkldnn/fusion_gru_mkldnn_op.cc      |  4 +--
 .../fused/mkldnn/fusion_lstm_mkldnn_op.cc     |  6 ++--
 paddle/fluid/operators/memcpy_h2d_op.h        |  2 +-
 paddle/fluid/operators/memcpy_op.h            |  2 +-
 paddle/phi/core/tensor_utils.cc               | 10 +++----
 paddle/phi/kernels/cpu/diag_grad_kernel.cc    |  2 +-
 .../kernels/cpu/fill_diagonal_grad_kernel.cc  |  2 +-
 .../phi/kernels/cpu/gather_nd_grad_kernel.cc  |  2 +-
 .../phi/kernels/cpu/index_add_grad_kernel.cc  |  2 +-
 .../kernels/cpu/index_select_grad_kernel.cc   |  2 +-
 .../phi/kernels/cpu/mean_all_grad_kernel.cc   |  2 +-
 .../kernels/cpu/put_along_axis_grad_kernel.cc |  4 +--
 .../kernels/cpu/scatter_nd_add_grad_kernel.cc |  2 +-
 .../impl/elementwise_grad_kernel_impl.h       | 30 ++++++++++++-------
 paddle/phi/kernels/impl/eye_kernel_impl.h     |  2 +-
 paddle/phi/kernels/impl/fill_kernel_impl.h    |  2 +-
 .../phi/kernels/onednn/interpolate_kernel.cc  |  4 +--
 paddle/phi/kernels/unsqueeze_kernel.h         |  2 +-
 19 files changed, 47 insertions(+), 37 deletions(-)

diff --git a/paddle/fluid/operators/collective/send_v2_op.h b/paddle/fluid/operators/collective/send_v2_op.h
index 047796dfe2430..7f51861008942 100644
--- a/paddle/fluid/operators/collective/send_v2_op.h
+++ b/paddle/fluid/operators/collective/send_v2_op.h
@@ -28,7 +28,7 @@ namespace operators {
 template <typename T, typename DeviceContext>
 class SendOpV2CPUKernel : public framework::OpKernel<T> {
  public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
+  void Compute(const framework::ExecutionContext& ctx UNUSED) const override {
     PADDLE_THROW(platform::errors::Unavailable(
         "Do not support send for cpu kernel now."));
   }
diff --git a/paddle/fluid/operators/fused/mkldnn/fusion_gru_mkldnn_op.cc b/paddle/fluid/operators/fused/mkldnn/fusion_gru_mkldnn_op.cc
index 806c883228035..05d1e64f92ae7 100644
--- a/paddle/fluid/operators/fused/mkldnn/fusion_gru_mkldnn_op.cc
+++ b/paddle/fluid/operators/fused/mkldnn/fusion_gru_mkldnn_op.cc
@@ -33,7 +33,7 @@ class GRUMKLDNNHandler : public RNNMKLDNNHandler<T, dnnl::gru_forward, T_out> {
   GRUMKLDNNHandler(const paddle::framework::ExecutionContext& ctx,
                    const OneDNNContext& dev_ctx,
                    const dnnl::engine onednn_engine,
-                   platform::Place cpu_place,
+                   platform::Place cpu_place UNUSED,
                    const phi::DenseTensor* input,
                    const phi::DenseTensor* weight_h,
                    const phi::DenseTensor* h0,
@@ -42,7 +42,7 @@ class GRUMKLDNNHandler : public RNNMKLDNNHandler<T, dnnl::gru_forward, T_out> {
                    const int64_t Ti,
                    const int64_t IC,
                    const int64_t OC,
-                   const std::string& unique_name)
+                   const std::string& unique_name UNUSED)
       : RNNMKLDNNHandler<T, dnnl::gru_forward, T_out>(
             ctx,
             dev_ctx,
diff --git a/paddle/fluid/operators/fused/mkldnn/fusion_lstm_mkldnn_op.cc b/paddle/fluid/operators/fused/mkldnn/fusion_lstm_mkldnn_op.cc
index bda5eab2d725f..d973c5e89a626 100644
--- a/paddle/fluid/operators/fused/mkldnn/fusion_lstm_mkldnn_op.cc
+++ b/paddle/fluid/operators/fused/mkldnn/fusion_lstm_mkldnn_op.cc
@@ -33,17 +33,17 @@ class LSTMMKLDNNHandler
   LSTMMKLDNNHandler(const paddle::framework::ExecutionContext& ctx,
                     const OneDNNContext& dev_ctx,
                     const dnnl::engine onednn_engine,
-                    platform::Place cpu_place,
+                    platform::Place cpu_place UNUSED,
                     const phi::DenseTensor* input,
                     const phi::DenseTensor* weight_h,
                     const phi::DenseTensor* h0,
-                    const phi::DenseTensor* c0,
+                    const phi::DenseTensor* c0 UNUSED,
                     const bool is_reverse,
                     const int64_t N,
                     const int64_t Ti,
                     const int64_t IC,
                     const int64_t OC,
-                    const std::string& unique_name)
+                    const std::string& unique_name UNUSED)
       : RNNMKLDNNHandler<T, dnnl::lstm_forward, T_out>(
             ctx,
             dev_ctx,
diff --git a/paddle/fluid/operators/memcpy_h2d_op.h b/paddle/fluid/operators/memcpy_h2d_op.h
index 9946e80bbc30f..5f480461d77cd 100644
--- a/paddle/fluid/operators/memcpy_h2d_op.h
+++ b/paddle/fluid/operators/memcpy_h2d_op.h
@@ -66,7 +66,7 @@ class MemcpyH2DFunctor {
   }
 
   template <typename T>
-  void operator()(const T &v) const {
+  void operator()(const T &v UNUSED) const {
     PADDLE_ENFORCE_EQ(
         true,
         false,
diff --git a/paddle/fluid/operators/memcpy_op.h b/paddle/fluid/operators/memcpy_op.h
index 3c989b151a315..1cb077ee8a4b9 100644
--- a/paddle/fluid/operators/memcpy_op.h
+++ b/paddle/fluid/operators/memcpy_op.h
@@ -81,7 +81,7 @@ class MemcpyFunctor {
   }
 
   template <typename T>
-  void operator()(const T &v) const {
+  void operator()(const T &v UNUSED) const {
     PADDLE_ENFORCE_EQ(
         true,
         false,
diff --git a/paddle/phi/core/tensor_utils.cc b/paddle/phi/core/tensor_utils.cc
index c73d6dc369bb8..abe44d3e2550b 100644
--- a/paddle/phi/core/tensor_utils.cc
+++ b/paddle/phi/core/tensor_utils.cc
@@ -318,11 +318,11 @@ void Copy(const Context& dev_ctx,
 }
 
 template <typename Context>
-void Copy(const Context& dev_ctx,
-          const TensorArray& src,
-          Place dst_place,
-          bool blocking,
-          TensorArray* dst) {
+void Copy(const Context& dev_ctx UNUSED,
+          const TensorArray& src UNUSED,
+          Place dst_place UNUSED,
+          bool blocking UNUSED,
+          TensorArray* dst UNUSED) {
   // NOTE(Ruibiao): implements Copy() for TensorArray when needed.
   PADDLE_THROW(errors::Unimplemented("Copy for TensorArray is unimplemented."));
 }
diff --git a/paddle/phi/kernels/cpu/diag_grad_kernel.cc b/paddle/phi/kernels/cpu/diag_grad_kernel.cc
index 13d3d679ff006..5a2f15d11428a 100644
--- a/paddle/phi/kernels/cpu/diag_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/diag_grad_kernel.cc
@@ -23,7 +23,7 @@ namespace phi {
 
 template <typename T, typename Context>
 void DiagGradKernel(const Context& dev_ctx,
-                    const DenseTensor& x,
+                    const DenseTensor& x UNUSED,
                     const DenseTensor& out_grad,
                     int offset,
                     DenseTensor* x_grad) {
diff --git a/paddle/phi/kernels/cpu/fill_diagonal_grad_kernel.cc b/paddle/phi/kernels/cpu/fill_diagonal_grad_kernel.cc
index 0fe9c50dc15e8..204c544e2d95f 100644
--- a/paddle/phi/kernels/cpu/fill_diagonal_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/fill_diagonal_grad_kernel.cc
@@ -22,7 +22,7 @@ namespace phi {
 template <typename T, typename Context>
 void FillDiagonalGradKernel(const Context& ctx,
                             const DenseTensor& out_grad,
-                            float value,
+                            float value UNUSED,
                             int offset,
                             bool wrap,
                             DenseTensor* x_grad) {
diff --git a/paddle/phi/kernels/cpu/gather_nd_grad_kernel.cc b/paddle/phi/kernels/cpu/gather_nd_grad_kernel.cc
index 88a288afd318e..5aaec6f6139e5 100644
--- a/paddle/phi/kernels/cpu/gather_nd_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/gather_nd_grad_kernel.cc
@@ -23,7 +23,7 @@ namespace phi {
 
 template <typename T, typename Context>
 void GatherNdGradKernel(const Context &ctx,
-                        const DenseTensor &x,
+                        const DenseTensor &x UNUSED,
                         const DenseTensor &index,
                         const DenseTensor &out_grad,
                         DenseTensor *x_grad) {
diff --git a/paddle/phi/kernels/cpu/index_add_grad_kernel.cc b/paddle/phi/kernels/cpu/index_add_grad_kernel.cc
index 007d8927377f8..a60d52f2005a4 100644
--- a/paddle/phi/kernels/cpu/index_add_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/index_add_grad_kernel.cc
@@ -23,7 +23,7 @@ namespace phi {
 template <typename T, typename Context>
 void IndexAddGradKernel(const Context& ctx,
                         const DenseTensor& index,
-                        const DenseTensor& add_value,
+                        const DenseTensor& add_value UNUSED,
                         const DenseTensor& out_grad,
                         int axis,
                         DenseTensor* x_grad,
diff --git a/paddle/phi/kernels/cpu/index_select_grad_kernel.cc b/paddle/phi/kernels/cpu/index_select_grad_kernel.cc
index cf8176687eab2..fc2e88b2f9906 100644
--- a/paddle/phi/kernels/cpu/index_select_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/index_select_grad_kernel.cc
@@ -22,7 +22,7 @@ namespace phi {
 
 template <typename T, typename Context>
 void IndexSelectGradKernel(const Context& ctx,
-                           const DenseTensor& x,
+                           const DenseTensor& x UNUSED,
                            const DenseTensor& index,
                            const DenseTensor& out_grad,
                            int dim,
diff --git a/paddle/phi/kernels/cpu/mean_all_grad_kernel.cc b/paddle/phi/kernels/cpu/mean_all_grad_kernel.cc
index 00321b010598f..81556643284f0 100644
--- a/paddle/phi/kernels/cpu/mean_all_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/mean_all_grad_kernel.cc
@@ -21,7 +21,7 @@ namespace phi {
 
 template <typename T, typename Context>
 void MeanAllGradKernel(const Context& dev_ctx,
-                       const DenseTensor& x,
+                       const DenseTensor& x UNUSED,
                        const DenseTensor& out_grad,
                        DenseTensor* x_grad) {
   PADDLE_ENFORCE_EQ(out_grad.numel(),
diff --git a/paddle/phi/kernels/cpu/put_along_axis_grad_kernel.cc b/paddle/phi/kernels/cpu/put_along_axis_grad_kernel.cc
index 5dce970f32ef0..d44af05357a9a 100644
--- a/paddle/phi/kernels/cpu/put_along_axis_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/put_along_axis_grad_kernel.cc
@@ -25,11 +25,11 @@ namespace phi {
 
 template <typename T, typename Context>
 void PutAlongAxisGradKernel(const Context& dev_ctx,
-                            const DenseTensor& x,
+                            const DenseTensor& x UNUSED,
                             const DenseTensor& index,
                             const DenseTensor& out_grad,
                             int axis,
-                            const std::string& reduce,
+                            const std::string& reduce UNUSED,
                             DenseTensor* x_grad,
                             DenseTensor* value_grad) {
   PADDLE_ENFORCE_EQ(
diff --git a/paddle/phi/kernels/cpu/scatter_nd_add_grad_kernel.cc b/paddle/phi/kernels/cpu/scatter_nd_add_grad_kernel.cc
index 844e6370caf73..d8009bc85881c 100644
--- a/paddle/phi/kernels/cpu/scatter_nd_add_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/scatter_nd_add_grad_kernel.cc
@@ -24,7 +24,7 @@ namespace phi {
 template <typename T, typename Context>
 void ScatterNdAddGradKernel(const Context &ctx,
                             const DenseTensor &index,
-                            const DenseTensor &updates,
+                            const DenseTensor &updates UNUSED,
                             const DenseTensor &out_grad,
                             DenseTensor *x_grad,
                             DenseTensor *updates_grad) {
diff --git a/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h b/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h
index 3ce1e721b968e..098514df85e47 100644
--- a/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h
@@ -364,13 +364,18 @@ void ElementwiseFMinGradKernel(const Context& dev_ctx,
 
 template <typename T>
 struct MulGradDX {
-  HOSTDEVICE T operator()(T x, T y, T out, T dout) const { return dout * y; }
+  HOSTDEVICE T operator()(T x UNUSED, T y, T out UNUSED, T dout) const {
+    return dout * y;
+  }
 };
 
 // avoid [-Wint-in-bool-context] warning
 template <>
 struct MulGradDX<bool> {
-  HOSTDEVICE bool operator()(bool x, bool y, bool out, bool dout) const {
+  HOSTDEVICE bool operator()(bool x UNUSED,
+                             bool y,
+                             bool out UNUSED,
+                             bool dout) const {
     return dout && y;
   }
 };
@@ -395,13 +400,18 @@ struct MulGradDX<phi::dtype::complex<T>> {
 
 template <typename T>
 struct MulGradDY {
-  HOSTDEVICE T operator()(T x, T y, T out, T dout) const { return dout * x; }
+  HOSTDEVICE T operator()(T x, T y UNUSED, T out UNUSED, T dout) const {
+    return dout * x;
+  }
 };
 
 // avoid [-Wint-in-bool-context] warning
 template <>
 struct MulGradDY<bool> {
-  HOSTDEVICE bool operator()(bool x, bool y, bool out, bool dout) const {
+  HOSTDEVICE bool operator()(bool x,
+                             bool y UNUSED,
+                             bool out UNUSED,
+                             bool dout) const {
     return dout && x;
   }
 };
@@ -826,14 +836,14 @@ void MultiplyTripleGradKernel(const Context& dev_ctx,
 
 template <typename T>
 struct MaxGradDx {
-  HOSTDEVICE T operator()(T x, T y, T out, T dout) const {
+  HOSTDEVICE T operator()(T x, T y, T out UNUSED, T dout) const {
     return dout * static_cast<T>(x > y);
   }
 };
 
 template <typename T>
 struct MaxGradDy {
-  HOSTDEVICE T operator()(T x, T y, T out, T dout) const {
+  HOSTDEVICE T operator()(T x, T y, T out UNUSED, T dout) const {
     return dout * static_cast<T>(x <= y);
   }
 };
@@ -845,14 +855,14 @@ struct MaxGradDy {
 */
 template <typename T>
 struct MinGradDx {
-  HOSTDEVICE T operator()(T x, T y, T out, T dout) const {
+  HOSTDEVICE T operator()(T x, T y, T out UNUSED, T dout) const {
     return dout * static_cast<T>(x < y);
   }
 };
 
 template <typename T>
 struct MinGradDy {
-  HOSTDEVICE T operator()(T x, T y, T out, T dout) const {
+  HOSTDEVICE T operator()(T x, T y, T out UNUSED, T dout) const {
     return dout * static_cast<T>(x >= y);
   }
 };
@@ -924,14 +934,14 @@ compute_pow_grad_dy(T x, T y, T out, T dout) {
 }
 #else
 template <typename T, typename MPType>
-HOSTDEVICE T compute_pow_grad_dx(T x, T y, T out, T dout) {
+HOSTDEVICE T compute_pow_grad_dx(T x, T y, T out UNUSED, T dout) {
   MPType x_val = static_cast<MPType>(x);
   MPType y_val = static_cast<MPType>(y);
   return static_cast<T>(static_cast<MPType>(dout) * y_val *
                         std::pow(x_val, y_val - 1));
 }
 template <typename T, typename MPType>
-HOSTDEVICE T compute_pow_grad_dy(T x, T y, T out, T dout) {
+HOSTDEVICE T compute_pow_grad_dy(T x, T y, T out UNUSED, T dout) {
   MPType x_val = static_cast<MPType>(x);
   MPType y_val = static_cast<MPType>(y);
   return static_cast<T>(static_cast<MPType>(dout) * std::log(x_val) *
diff --git a/paddle/phi/kernels/impl/eye_kernel_impl.h b/paddle/phi/kernels/impl/eye_kernel_impl.h
index 2d373f99a277a..c93ac260dd874 100644
--- a/paddle/phi/kernels/impl/eye_kernel_impl.h
+++ b/paddle/phi/kernels/impl/eye_kernel_impl.h
@@ -37,7 +37,7 @@ template <typename T, typename Context>
 void EyeKernel(const Context& ctx,
                const Scalar& num_rows,
                const Scalar& num_columns,
-               DataType dtype,
+               DataType dtype UNUSED,
                DenseTensor* out) {
   auto columns = num_columns.to<int64_t>();
   auto rows = num_rows.to<int64_t>();
diff --git a/paddle/phi/kernels/impl/fill_kernel_impl.h b/paddle/phi/kernels/impl/fill_kernel_impl.h
index 7d10ea42bd6b6..6894204cd06a4 100644
--- a/paddle/phi/kernels/impl/fill_kernel_impl.h
+++ b/paddle/phi/kernels/impl/fill_kernel_impl.h
@@ -24,7 +24,7 @@ namespace phi {
 
 template <typename T, typename Context>
 void FillKernel(const Context& dev_ctx,
-                const DenseTensor& x,
+                const DenseTensor& x UNUSED,
                 const Scalar& value,
                 DenseTensor* out) {
   T fill_var = value.to<T>();
diff --git a/paddle/phi/kernels/onednn/interpolate_kernel.cc b/paddle/phi/kernels/onednn/interpolate_kernel.cc
index b9a8672d792ce..508cdcc329a2e 100644
--- a/paddle/phi/kernels/onednn/interpolate_kernel.cc
+++ b/paddle/phi/kernels/onednn/interpolate_kernel.cc
@@ -239,8 +239,8 @@ void NearestInterpKernel(
     int out_w,
     const std::vector<float>& scale,
     const std::string& interp_method,
-    bool align_corners,
-    int align_mode,
+    bool align_corners UNUSED,
+    int align_mode UNUSED,
     DenseTensor* output) {
   InterpolateKernel<T, Context>(ctx,
                                 x,
diff --git a/paddle/phi/kernels/unsqueeze_kernel.h b/paddle/phi/kernels/unsqueeze_kernel.h
index bb190e14179c0..e6ad4dc75d3d5 100644
--- a/paddle/phi/kernels/unsqueeze_kernel.h
+++ b/paddle/phi/kernels/unsqueeze_kernel.h
@@ -39,7 +39,7 @@ void Unsqueeze(const Context& dev_ctx,
                const DenseTensor& x,
                const IntArray& axes,
                DenseTensor* out,
-               DenseTensor* xshape) {
+               DenseTensor* xshape UNUSED) {
   MetaTensor meta_out(out);
   UnsqueezeInferMeta(x, axes, &meta_out);
   UnsqueezeInferKernel<T, Context>(dev_ctx, x, axes, out);

From ca2ea162d2ba46d1e8de34ba6f7240ffa8b6889b Mon Sep 17 00:00:00 2001
From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com>
Date: Mon, 15 May 2023 15:38:13 +0800
Subject: [PATCH 328/405] remove some [-Wunsed-parameter]warning (#53679)

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop
---
 paddle/fluid/inference/api/helper.h           |  2 +-
 paddle/fluid/operators/assign_op.h            |  2 +-
 paddle/phi/core/kernel_registry.h             | 20 ++++++-------
 .../kernels/check_memory_continue_kernel.cc   |  2 +-
 paddle/phi/kernels/cpu/cum_kernel.cc          |  2 +-
 .../phi/kernels/cpu/diagonal_grad_kernel.cc   |  2 +-
 paddle/phi/kernels/cpu/elementwise_grad.h     | 12 ++++++--
 paddle/phi/kernels/cpu/gather_grad_kernel.cc  |  2 +-
 .../kernels/cpu/overlap_add_grad_kernel.cc    |  2 +-
 paddle/phi/kernels/cpu/pad3d_kernel.cc        | 28 +++++++++----------
 paddle/phi/kernels/cpu/roll_grad_kernel.cc    |  2 +-
 paddle/phi/kernels/funcs/activation_functor.h |  4 +--
 .../kernels/funcs/selected_rows_functor.cc    |  2 +-
 paddle/phi/kernels/funcs/squared_l2_norm.h    |  2 +-
 .../phi/kernels/impl/fill_grad_kernel_impl.h  |  4 +--
 .../phi/kernels/impl/frame_grad_kernel_impl.h |  2 +-
 .../phi/kernels/impl/pad_grad_kernel_impl.h   |  2 +-
 .../phi/kernels/impl/trace_grad_kernel_impl.h |  2 +-
 .../phi/kernels/sparse/sparse_utils_kernel.h  |  2 +-
 19 files changed, 51 insertions(+), 45 deletions(-)

diff --git a/paddle/fluid/inference/api/helper.h b/paddle/fluid/inference/api/helper.h
index 92b6f12338f79..d379930517dac 100644
--- a/paddle/fluid/inference/api/helper.h
+++ b/paddle/fluid/inference/api/helper.h
@@ -339,7 +339,7 @@ static bool CompareTensor(const PaddleTensor &a, const PaddleTensor &b) {
 }
 
 static std::string DescribeTensor(const PaddleTensor &tensor,
-                                  int max_num_of_data = 15) {
+                                  int max_num_of_data UNUSED = 15) {
   std::stringstream os;
   os << "Tensor [" << tensor.name << "]\n";
   os << " - type: ";
diff --git a/paddle/fluid/operators/assign_op.h b/paddle/fluid/operators/assign_op.h
index f45d00ba82c77..6efc621120929 100644
--- a/paddle/fluid/operators/assign_op.h
+++ b/paddle/fluid/operators/assign_op.h
@@ -60,7 +60,7 @@ class AssignFunctor {
   }
 
   template <typename T>
-  void operator()(const T &v) const {
+  void operator()(const T &v UNUSED) const {
     PADDLE_ENFORCE_EQ(
         true,
         false,
diff --git a/paddle/phi/core/kernel_registry.h b/paddle/phi/core/kernel_registry.h
index 3ca455da1dd40..f4e021f7269a7 100644
--- a/paddle/phi/core/kernel_registry.h
+++ b/paddle/phi/core/kernel_registry.h
@@ -1433,16 +1433,16 @@ struct KernelRegistrar {
     reg_type, kernel_name, layout, meta_kernel_fn, N, ...)        \
   static void __PD_KERNEL_args_def_FN_##kernel_name##_##layout(   \
       const ::phi::KernelKey& kernel_key, ::phi::Kernel* kernel); \
-  PD_EXPAND(                                                      \
-      PD_CONCATENATE(_PD_FOR_ALL_BACKEND_DTYPE_, N)(              \
-          reg_type,                                               \
-          kernel_name,                                            \
-          layout,                                                 \
-          meta_kernel_fn,                                         \
-          __PD_KERNEL_args_def_FN_##kernel_name##_##layout,       \
-          __VA_ARGS__) void                                       \
-          __PD_KERNEL_args_def_FN_##kernel_name##_##layout(       \
-              const ::phi::KernelKey& kernel_key, ::phi::Kernel* kernel))
+  PD_EXPAND(PD_CONCATENATE(_PD_FOR_ALL_BACKEND_DTYPE_, N)(        \
+      reg_type,                                                   \
+      kernel_name,                                                \
+      layout,                                                     \
+      meta_kernel_fn,                                             \
+      __PD_KERNEL_args_def_FN_##kernel_name##_##layout,           \
+      __VA_ARGS__) void                                           \
+                __PD_KERNEL_args_def_FN_##kernel_name##_##layout( \
+                    const ::phi::KernelKey& kernel_key UNUSED,    \
+                    ::phi::Kernel* kernel UNUSED))
 #ifndef _WIN32
 #define ___PD_REGISTER_KERNEL_FOR_ALL_BACKEND_DTYPE(                  \
     reg_type, kernel_name, backend, layout, kernel_fn, args_def_fn)   \
diff --git a/paddle/phi/kernels/check_memory_continue_kernel.cc b/paddle/phi/kernels/check_memory_continue_kernel.cc
index 819bb30712bf0..6e496a355302f 100644
--- a/paddle/phi/kernels/check_memory_continue_kernel.cc
+++ b/paddle/phi/kernels/check_memory_continue_kernel.cc
@@ -28,7 +28,7 @@ template <typename T, typename Context>
 void CheckMemoryContinueKernel(const Context &dev_ctx,
                                const std::vector<const DenseTensor *> &input,
                                DenseTensor *output,
-                               std::vector<DenseTensor *> xout) {
+                               std::vector<DenseTensor *> xout UNUSED) {
   int64_t size_of_dtype = sizeof(T);
   auto dtype = input.at(0)->dtype();
   int64_t numel = 0;
diff --git a/paddle/phi/kernels/cpu/cum_kernel.cc b/paddle/phi/kernels/cpu/cum_kernel.cc
index f7ec5bbbf9e84..ec6c3cfd0baab 100644
--- a/paddle/phi/kernels/cpu/cum_kernel.cc
+++ b/paddle/phi/kernels/cpu/cum_kernel.cc
@@ -52,7 +52,7 @@ template <typename T, typename Context, typename Reducer>
 void ScanKernel(const Context& dev_ctx,
                 const DenseTensor& x,
                 int axis,
-                bool flatten,
+                bool flatten UNUSED,
                 bool exclusive,
                 bool reverse,
                 Reducer reducer,
diff --git a/paddle/phi/kernels/cpu/diagonal_grad_kernel.cc b/paddle/phi/kernels/cpu/diagonal_grad_kernel.cc
index f5d6ee2dce674..151ec3df10356 100644
--- a/paddle/phi/kernels/cpu/diagonal_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/diagonal_grad_kernel.cc
@@ -22,7 +22,7 @@ namespace phi {
 
 template <typename T, typename Context>
 void DiagonalGradKernel(const Context& dev_ctx,
-                        const DenseTensor& x,
+                        const DenseTensor& x UNUSED,
                         const DenseTensor& out_grad,
                         int offset,
                         int axis1,
diff --git a/paddle/phi/kernels/cpu/elementwise_grad.h b/paddle/phi/kernels/cpu/elementwise_grad.h
index 05c02f167b6a2..0602d529de842 100644
--- a/paddle/phi/kernels/cpu/elementwise_grad.h
+++ b/paddle/phi/kernels/cpu/elementwise_grad.h
@@ -75,7 +75,9 @@ void ElemwiseExplicitGradCompute(const CPUContext& dev_ctx,
 */
 template <typename T>
 struct IdentityGrad {
-  HOSTDEVICE T operator()(T x, T y, T out, T dout) const { return dout; }
+  HOSTDEVICE T operator()(T x UNUSED, T y UNUSED, T out UNUSED, T dout) const {
+    return dout;
+  }
 };
 
 template <typename T>
@@ -120,12 +122,16 @@ ElementwiseAddGrad(const CPUContext& ctx,
 
 template <typename T>
 struct SubGradDX {
-  HOSTDEVICE T operator()(T x, T y, T out, T dout) const { return dout; }
+  HOSTDEVICE T operator()(T x UNUSED, T y UNUSED, T out UNUSED, T dout) const {
+    return dout;
+  }
 };
 
 template <typename T>
 struct SubGradDY {
-  HOSTDEVICE T operator()(T x, T y, T out, T dout) const { return -dout; }
+  HOSTDEVICE T operator()(T x UNUSED, T y UNUSED, T out UNUSED, T dout) const {
+    return -dout;
+  }
 };
 
 template <typename T>
diff --git a/paddle/phi/kernels/cpu/gather_grad_kernel.cc b/paddle/phi/kernels/cpu/gather_grad_kernel.cc
index 0d0341234e60b..456c7ea633cde 100644
--- a/paddle/phi/kernels/cpu/gather_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/gather_grad_kernel.cc
@@ -24,7 +24,7 @@ namespace phi {
 
 template <typename T, typename Context>
 void GatherGradKernel(const Context& dev_ctx,
-                      const DenseTensor& x,
+                      const DenseTensor& x UNUSED,
                       const DenseTensor& index,
                       const DenseTensor& out_grad,
                       const Scalar& axis,
diff --git a/paddle/phi/kernels/cpu/overlap_add_grad_kernel.cc b/paddle/phi/kernels/cpu/overlap_add_grad_kernel.cc
index 4f6dd6e14829a..bee17b3d3b969 100644
--- a/paddle/phi/kernels/cpu/overlap_add_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/overlap_add_grad_kernel.cc
@@ -21,7 +21,7 @@ namespace phi {
 
 template <typename T, typename Context>
 void OverlapAddGradKernel(const Context& dev_ctx,
-                          const DenseTensor& x,
+                          const DenseTensor& x UNUSED,
                           const DenseTensor& out_grad,
                           int hop_length,
                           int axis,
diff --git a/paddle/phi/kernels/cpu/pad3d_kernel.cc b/paddle/phi/kernels/cpu/pad3d_kernel.cc
index aa44491a2a967..d26e2fc7ea986 100644
--- a/paddle/phi/kernels/cpu/pad3d_kernel.cc
+++ b/paddle/phi/kernels/cpu/pad3d_kernel.cc
@@ -26,7 +26,7 @@ void ConstPad3DFuncNCDHW(const T* in_data,
                          const int in_depth,
                          const int in_height,
                          const int in_width,
-                         const int out_depth,
+                         const int out_depth UNUSED,
                          const int out_height,
                          const int out_width,
                          const int pad_front,
@@ -53,7 +53,7 @@ void ConstPad3DFuncNDHWC(const T* in_data,
                          const int in_depth,
                          const int in_height,
                          const int in_width,
-                         const int out_depth,
+                         const int out_depth UNUSED,
                          const int out_height,
                          const int out_width,
                          const int pad_front,
@@ -88,7 +88,7 @@ void ReflectPad3DFuncNCDHW(const T* in_data,
                            const int in_depth,
                            const int in_height,
                            const int in_width,
-                           const int out_depth,
+                           const int out_depth UNUSED,
                            const int out_height,
                            const int out_width,
                            const int pad_front,
@@ -97,7 +97,7 @@ void ReflectPad3DFuncNCDHW(const T* in_data,
                            const int out_d,
                            const int out_h,
                            const int out_w,
-                           const T value) {
+                           const T value UNUSED) {
   int in_d = out_d - pad_front;
   int in_h = out_h - pad_top;
   int in_w = out_w - pad_left;
@@ -120,7 +120,7 @@ void ReflectPad3DFuncNDHWC(const T* in_data,
                            const int in_depth,
                            const int in_height,
                            const int in_width,
-                           const int out_depth,
+                           const int out_depth UNUSED,
                            const int out_height,
                            const int out_width,
                            const int pad_front,
@@ -129,7 +129,7 @@ void ReflectPad3DFuncNDHWC(const T* in_data,
                            const int out_d,
                            const int out_h,
                            const int out_w,
-                           const T value) {
+                           const T value UNUSED) {
   int in_d = out_d - pad_front;
   int in_h = out_h - pad_top;
   int in_w = out_w - pad_left;
@@ -156,7 +156,7 @@ void ReplicatePad3DFuncNCDHW(const T* in_data,
                              const int in_depth,
                              const int in_height,
                              const int in_width,
-                             const int out_depth,
+                             const int out_depth UNUSED,
                              const int out_height,
                              const int out_width,
                              const int pad_front,
@@ -165,7 +165,7 @@ void ReplicatePad3DFuncNCDHW(const T* in_data,
                              const int out_d,
                              const int out_h,
                              const int out_w,
-                             const T value) {
+                             const T value UNUSED) {
   int in_d = std::min(in_depth - 1, std::max(out_d - pad_front, 0));
   int in_h = std::min(in_height - 1, std::max(out_h - pad_top, 0));
   int in_w = std::min(in_width - 1, std::max(out_w - pad_left, 0));
@@ -181,7 +181,7 @@ void ReplicatePad3DFuncNDHWC(const T* in_data,
                              const int in_depth,
                              const int in_height,
                              const int in_width,
-                             const int out_depth,
+                             const int out_depth UNUSED,
                              const int out_height,
                              const int out_width,
                              const int pad_front,
@@ -190,7 +190,7 @@ void ReplicatePad3DFuncNDHWC(const T* in_data,
                              const int out_d,
                              const int out_h,
                              const int out_w,
-                             const T value) {
+                             const T value UNUSED) {
   int in_d = std::min(in_depth - 1, std::max(out_d - pad_front, 0));
   int in_h = std::min(in_height - 1, std::max(out_h - pad_top, 0));
   int in_w = std::min(in_width - 1, std::max(out_w - pad_left, 0));
@@ -210,7 +210,7 @@ void CircularPad3DFuncNCDHW(const T* in_data,
                             const int in_depth,
                             const int in_height,
                             const int in_width,
-                            const int out_depth,
+                            const int out_depth UNUSED,
                             const int out_height,
                             const int out_width,
                             const int pad_front,
@@ -219,7 +219,7 @@ void CircularPad3DFuncNCDHW(const T* in_data,
                             const int out_d,
                             const int out_h,
                             const int out_w,
-                            const T value) {
+                            const T value UNUSED) {
   int in_d = ((out_d - pad_front) % in_depth + in_depth) % in_depth;
   int in_h = ((out_h - pad_top) % in_height + in_height) % in_height;
   int in_w = ((out_w - pad_left) % in_width + in_width) % in_width;
@@ -235,7 +235,7 @@ void CircularPad3DFuncNDHWC(const T* in_data,
                             const int in_depth,
                             const int in_height,
                             const int in_width,
-                            const int out_depth,
+                            const int out_depth UNUSED,
                             const int out_height,
                             const int out_width,
                             const int pad_front,
@@ -244,7 +244,7 @@ void CircularPad3DFuncNDHWC(const T* in_data,
                             const int out_d,
                             const int out_h,
                             const int out_w,
-                            const T value) {
+                            const T value UNUSED) {
   int in_d = ((out_d - pad_front) % in_depth + in_depth) % in_depth;
   int in_h = ((out_h - pad_top) % in_height + in_height) % in_height;
   int in_w = ((out_w - pad_left) % in_width + in_width) % in_width;
diff --git a/paddle/phi/kernels/cpu/roll_grad_kernel.cc b/paddle/phi/kernels/cpu/roll_grad_kernel.cc
index c348bfe30045e..bee30ad8e49de 100644
--- a/paddle/phi/kernels/cpu/roll_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/roll_grad_kernel.cc
@@ -22,7 +22,7 @@ namespace phi {
 
 template <typename T, typename Context>
 void RollGradKernel(const Context& dev_ctx,
-                    const DenseTensor& x,
+                    const DenseTensor& x UNUSED,
                     const DenseTensor& out_grad,
                     const IntArray& shifts,
                     const std::vector<int64_t>& axis,
diff --git a/paddle/phi/kernels/funcs/activation_functor.h b/paddle/phi/kernels/funcs/activation_functor.h
index ccc2c474499d7..df63c22c94600 100644
--- a/paddle/phi/kernels/funcs/activation_functor.h
+++ b/paddle/phi/kernels/funcs/activation_functor.h
@@ -1112,7 +1112,7 @@ struct Expm1GradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x UNUSED, Out out, dOut dout, dX dx) const {
     dx.device(d) = dout * out + dout;
   }
 
@@ -1412,7 +1412,7 @@ struct LeakyReluGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     auto temp1 =
         static_cast<T>(alpha) * (x < static_cast<T>(0)).template cast<T>();
     auto temp2 = (x >= static_cast<T>(0)).template cast<T>();
diff --git a/paddle/phi/kernels/funcs/selected_rows_functor.cc b/paddle/phi/kernels/funcs/selected_rows_functor.cc
index efeb742cfd2ae..0c91d250d2313 100644
--- a/paddle/phi/kernels/funcs/selected_rows_functor.cc
+++ b/paddle/phi/kernels/funcs/selected_rows_functor.cc
@@ -288,7 +288,7 @@ template struct SelectedRowsSumTo<phi::CPUContext, double>;
 
 template <typename T>
 struct SelectedRowsAddToTensor<phi::CPUContext, T> {
-  void operator()(const phi::CPUContext& context,
+  void operator()(const phi::CPUContext& context UNUSED,
                   const phi::SelectedRows& input1,
                   phi::DenseTensor* input2) {
     if (UNLIKELY(input1.rows().size() == 0)) {
diff --git a/paddle/phi/kernels/funcs/squared_l2_norm.h b/paddle/phi/kernels/funcs/squared_l2_norm.h
index ccb208791b849..c77552822bbfb 100644
--- a/paddle/phi/kernels/funcs/squared_l2_norm.h
+++ b/paddle/phi/kernels/funcs/squared_l2_norm.h
@@ -36,7 +36,7 @@ void SquaredL2Norm(const phi::CPUContext& ctx,
                    const T1* x,
                    T2* y,
                    size_t numel,
-                   memory_utils::Buffer* buffer = nullptr) {
+                   memory_utils::Buffer* buffer UNUSED = nullptr) {
   if (std::is_same<T1, T2>::value) {
     using EigenT = typename phi::EigenTensor<T1, 1>::Type;
     using ConstEigenT = typename phi::EigenTensor<T1, 1>::ConstType;
diff --git a/paddle/phi/kernels/impl/fill_grad_kernel_impl.h b/paddle/phi/kernels/impl/fill_grad_kernel_impl.h
index dffb81fbea4e3..a8a67a2b080c6 100644
--- a/paddle/phi/kernels/impl/fill_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/fill_grad_kernel_impl.h
@@ -24,8 +24,8 @@ namespace phi {
 
 template <typename T, typename Context>
 void FillGradKernel(const Context& dev_ctx,
-                    const DenseTensor& out_grad,
-                    const Scalar& value,
+                    const DenseTensor& out_grad UNUSED,
+                    const Scalar& value UNUSED,
                     DenseTensor* in_grad) {
   if (in_grad) {
     dev_ctx.template Alloc<T>(in_grad);
diff --git a/paddle/phi/kernels/impl/frame_grad_kernel_impl.h b/paddle/phi/kernels/impl/frame_grad_kernel_impl.h
index 9d1bfe453d09a..9f6ceee24f183 100644
--- a/paddle/phi/kernels/impl/frame_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/frame_grad_kernel_impl.h
@@ -20,7 +20,7 @@
 namespace phi {
 template <typename T, typename Context>
 void FrameGradKernel(const Context& dev_ctx,
-                     const DenseTensor& x,
+                     const DenseTensor& x UNUSED,
                      const DenseTensor& dout,
                      int frame_length,
                      int hop_length,
diff --git a/paddle/phi/kernels/impl/pad_grad_kernel_impl.h b/paddle/phi/kernels/impl/pad_grad_kernel_impl.h
index 4939a41c25a69..875045f8c8e8f 100644
--- a/paddle/phi/kernels/impl/pad_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/pad_grad_kernel_impl.h
@@ -22,7 +22,7 @@ template <typename T, typename Context>
 void PadGradKernel(const Context& dev_ctx,
                    const DenseTensor& d_out,
                    const std::vector<int>& paddings,
-                   const Scalar& pad_value,
+                   const Scalar& pad_value UNUSED,
                    DenseTensor* d_x) {
   if (d_x == nullptr) {
     return;
diff --git a/paddle/phi/kernels/impl/trace_grad_kernel_impl.h b/paddle/phi/kernels/impl/trace_grad_kernel_impl.h
index 1099f27f3622e..640fd07a92a2b 100644
--- a/paddle/phi/kernels/impl/trace_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/trace_grad_kernel_impl.h
@@ -82,7 +82,7 @@ struct TraceGradFunctor {
 
 template <typename T, typename Context>
 void TraceGradKernel(const Context& ctx,
-                     const DenseTensor& x,
+                     const DenseTensor& x UNUSED,
                      const DenseTensor& out_grad,
                      int offset,
                      int axis1,
diff --git a/paddle/phi/kernels/sparse/sparse_utils_kernel.h b/paddle/phi/kernels/sparse/sparse_utils_kernel.h
index e06391d9404ef..8d92b312bd857 100644
--- a/paddle/phi/kernels/sparse/sparse_utils_kernel.h
+++ b/paddle/phi/kernels/sparse/sparse_utils_kernel.h
@@ -165,7 +165,7 @@ void IndicesCooKernel(const Context& dev_ctx UNUSED,
 }
 
 template <typename T, typename Context>
-void SparseCooTensorKernel(const Context& dev_ctx,
+void SparseCooTensorKernel(const Context& dev_ctx UNUSED,
                            const DenseTensor& values,
                            const DenseTensor& indices,
                            const std::vector<int64_t>& shape,

From 8ed01e8d254d6dd46c77d6eb585802d72e133167 Mon Sep 17 00:00:00 2001
From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com>
Date: Mon, 15 May 2023 15:38:23 +0800
Subject: [PATCH 329/405] remove some [-Wunsed-parameter] warning (#53687)

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop
---
 paddle/phi/core/utils/unroll_array_ops.h           |  2 +-
 paddle/phi/kernels/cpu/amp_kernel.cc               |  2 +-
 paddle/phi/kernels/cpu/argsort_grad_kernel.cc      |  2 +-
 paddle/phi/kernels/cpu/elementwise.h               |  8 ++++----
 paddle/phi/kernels/cpu/hsigmoid_loss_grad.h        |  3 ++-
 paddle/phi/kernels/cpu/index_sample_grad_kernel.cc |  2 +-
 paddle/phi/kernels/cpu/index_select_impl.h         |  2 +-
 paddle/phi/kernels/cpu/kthvalue_grad_kernel.cc     |  2 +-
 .../phi/kernels/cpu/masked_select_grad_kernel.cc   |  2 +-
 paddle/phi/kernels/cpu/nanmedian_grad_kernel.cc    |  4 ++--
 paddle/phi/kernels/cpu/nanmedian_kernel.cc         |  2 +-
 paddle/phi/kernels/cpu/randperm_kernel.cc          |  2 +-
 .../kernels/cpu/repeat_interleave_grad_kernel.cc   |  4 ++--
 paddle/phi/kernels/cpu/rnn_grad_kernel.cc          | 12 ++++++------
 paddle/phi/kernels/cpu/rnn_kernel.cc               | 14 +++++++-------
 paddle/phi/kernels/cpu/scatter_grad_kernel.cc      |  4 ++--
 paddle/phi/kernels/cpu/top_k_grad_kernel.cc        |  4 ++--
 paddle/phi/kernels/cpu/trunc_grad_kernel.cc        |  2 +-
 paddle/phi/kernels/cpu/where_grad_kernel.cc        |  4 ++--
 19 files changed, 39 insertions(+), 38 deletions(-)

diff --git a/paddle/phi/core/utils/unroll_array_ops.h b/paddle/phi/core/utils/unroll_array_ops.h
index 665c2ae512829..ea9d6273e4a4e 100644
--- a/paddle/phi/core/utils/unroll_array_ops.h
+++ b/paddle/phi/core/utils/unroll_array_ops.h
@@ -32,7 +32,7 @@ struct UnrollFillConstant {
 template <size_t kStart, size_t kEnd>
 struct UnrollFillConstant<kStart, kEnd, true> {
   template <typename T>
-  HOSTDEVICE inline static void Run(T *data, T val) {}
+  HOSTDEVICE inline static void Run(T *data UNUSED, T val UNUSED) {}
 };
 
 template <size_t kStart, size_t kEnd, bool kStop>
diff --git a/paddle/phi/kernels/cpu/amp_kernel.cc b/paddle/phi/kernels/cpu/amp_kernel.cc
index ae0b01707cece..97c71290573cb 100644
--- a/paddle/phi/kernels/cpu/amp_kernel.cc
+++ b/paddle/phi/kernels/cpu/amp_kernel.cc
@@ -31,7 +31,7 @@ namespace phi {
 template <typename T, bool IsFoundInfOnCPU>
 class UpdateLossScalingFunctor<phi::CPUContext, T, IsFoundInfOnCPU> {
  public:
-  void operator()(const phi::CPUContext& ctx,
+  void operator()(const phi::CPUContext& ctx UNUSED,
                   const bool* found_inf_data,
                   const T* pre_loss_scaling_data,
                   const int* good_in_data,
diff --git a/paddle/phi/kernels/cpu/argsort_grad_kernel.cc b/paddle/phi/kernels/cpu/argsort_grad_kernel.cc
index 81616dafc0a8b..7be48f896b2f3 100644
--- a/paddle/phi/kernels/cpu/argsort_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/argsort_grad_kernel.cc
@@ -55,7 +55,7 @@ void ArgsortGradKernel(const Context& dev_ctx,
                        const DenseTensor& input,
                        const DenseTensor& out_grad,
                        int axis,
-                       bool descending,
+                       bool descending UNUSED,
                        DenseTensor* in_grad) {
   auto in_dims = indices.dims();
   auto rank = input.dims().size();
diff --git a/paddle/phi/kernels/cpu/elementwise.h b/paddle/phi/kernels/cpu/elementwise.h
index 255dae7da014d..918eb2f003062 100644
--- a/paddle/phi/kernels/cpu/elementwise.h
+++ b/paddle/phi/kernels/cpu/elementwise.h
@@ -122,10 +122,10 @@ struct SameDimsDivideFunctor<
     DevCtx,
     T,
     typename std::enable_if<!std::is_floating_point<T>::value>::type> {
-  void operator()(const DevCtx& dev_ctx,
-                  const DenseTensor& x,
-                  const DenseTensor& y,
-                  DenseTensor* z) {
+  void operator()(const DevCtx& dev_ctx UNUSED,
+                  const DenseTensor& x UNUSED,
+                  const DenseTensor& y UNUSED,
+                  DenseTensor* z UNUSED) {
     phi::errors::InvalidArgument(
         "If use SameDimsDivideFunctor, template args(T) must be floating "
         "point. ");
diff --git a/paddle/phi/kernels/cpu/hsigmoid_loss_grad.h b/paddle/phi/kernels/cpu/hsigmoid_loss_grad.h
index f4b35c9101836..9dda9785e0d73 100644
--- a/paddle/phi/kernels/cpu/hsigmoid_loss_grad.h
+++ b/paddle/phi/kernels/cpu/hsigmoid_loss_grad.h
@@ -31,7 +31,8 @@ void HSigmoidLossGradKernelImpl(const Context& ctx,
                                 const DenseTensor& label,
                                 const paddle::optional<DenseTensor>& path,
                                 const paddle::optional<DenseTensor>& code,
-                                const paddle::optional<DenseTensor>& bias,
+                                const paddle::optional<DenseTensor>& bias
+                                    UNUSED,
                                 const DenseTensor& pre_out,
                                 const DenseTensor& out_grad,
                                 int num_classes,
diff --git a/paddle/phi/kernels/cpu/index_sample_grad_kernel.cc b/paddle/phi/kernels/cpu/index_sample_grad_kernel.cc
index 0ac688f48fa5b..a0b2e8aef34cc 100644
--- a/paddle/phi/kernels/cpu/index_sample_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/index_sample_grad_kernel.cc
@@ -69,7 +69,7 @@ void IndexSampleGradInner(const Context& context,
 
 template <typename T, typename Context>
 void IndexSampleGradKernel(const Context& ctx,
-                           const DenseTensor& x,
+                           const DenseTensor& x UNUSED,
                            const DenseTensor& index,
                            const DenseTensor& out_grad,
                            DenseTensor* x_grad) {
diff --git a/paddle/phi/kernels/cpu/index_select_impl.h b/paddle/phi/kernels/cpu/index_select_impl.h
index 1b2842ee66634..7c20ab8ea5dfd 100644
--- a/paddle/phi/kernels/cpu/index_select_impl.h
+++ b/paddle/phi/kernels/cpu/index_select_impl.h
@@ -26,7 +26,7 @@ namespace phi {
 
 template <typename Context, typename T, class Enable = void>
 struct IndexSelectAdd {
-  void operator()(const Context& ctx,
+  void operator()(const Context& ctx UNUSED,
                   int slice_size,
                   const T* src_pointer,
                   const T* p_pointer,
diff --git a/paddle/phi/kernels/cpu/kthvalue_grad_kernel.cc b/paddle/phi/kernels/cpu/kthvalue_grad_kernel.cc
index 459c66fae09e4..2987158710c27 100644
--- a/paddle/phi/kernels/cpu/kthvalue_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/kthvalue_grad_kernel.cc
@@ -49,7 +49,7 @@ void KthvalueGradKernel(const Context& dev_ctx,
                         const DenseTensor& x,
                         const DenseTensor& indices,
                         const DenseTensor& d_out,
-                        int k,
+                        int k UNUSED,
                         int axis,
                         bool keepdim,
                         DenseTensor* d_x) {
diff --git a/paddle/phi/kernels/cpu/masked_select_grad_kernel.cc b/paddle/phi/kernels/cpu/masked_select_grad_kernel.cc
index f615fb2e0bc3f..8b1ca87b4a270 100644
--- a/paddle/phi/kernels/cpu/masked_select_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/masked_select_grad_kernel.cc
@@ -21,7 +21,7 @@ namespace phi {
 
 template <typename T, typename Context>
 void MaskedSelectGradKernel(const Context& dev_ctx,
-                            const DenseTensor& x,
+                            const DenseTensor& x UNUSED,
                             const DenseTensor& mask,
                             const DenseTensor& out_grad,
                             DenseTensor* x_grad) {
diff --git a/paddle/phi/kernels/cpu/nanmedian_grad_kernel.cc b/paddle/phi/kernels/cpu/nanmedian_grad_kernel.cc
index f8639a0d10fee..bce234375e529 100644
--- a/paddle/phi/kernels/cpu/nanmedian_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/nanmedian_grad_kernel.cc
@@ -26,7 +26,7 @@ void CalcMedianGradKernel(const Context& dev_ctx,
                           const DenseTensor& x,
                           const DenseTensor& median_index,
                           const DenseTensor& out_grad,
-                          const IntArray& axes,
+                          const IntArray& axes UNUSED,
                           DenseTensor* x_grad,
                           T* x_grad_ptr) {
   phi::funcs::SetConstant<Context, T> set_zero;
@@ -83,7 +83,7 @@ void NanmedianGradKernel(const Context& dev_ctx,
                          const DenseTensor& median_index,
                          const DenseTensor& out_grad,
                          const IntArray& axes,
-                         bool keep_dim,
+                         bool keep_dim UNUSED,
                          DenseTensor* x_grad) {
   BaseMedianGradKernel<T, Context>(
       dev_ctx, input, median_index, out_grad, axes, x_grad);
diff --git a/paddle/phi/kernels/cpu/nanmedian_kernel.cc b/paddle/phi/kernels/cpu/nanmedian_kernel.cc
index 595cf868db094..660c115f92f89 100644
--- a/paddle/phi/kernels/cpu/nanmedian_kernel.cc
+++ b/paddle/phi/kernels/cpu/nanmedian_kernel.cc
@@ -192,7 +192,7 @@ template <typename T, typename Context>
 void NanmedianKernel(const Context& dev_ctx,
                      const DenseTensor& x,
                      const IntArray& axes,
-                     bool keepdim,
+                     bool keepdim UNUSED,
                      DenseTensor* out,
                      DenseTensor* median_index) {
   BaseMedianKernel<T, Context>(dev_ctx, x, axes, out, median_index, true);
diff --git a/paddle/phi/kernels/cpu/randperm_kernel.cc b/paddle/phi/kernels/cpu/randperm_kernel.cc
index b536f75a71578..a8dac2bf4b095 100644
--- a/paddle/phi/kernels/cpu/randperm_kernel.cc
+++ b/paddle/phi/kernels/cpu/randperm_kernel.cc
@@ -21,7 +21,7 @@ namespace phi {
 template <typename T, typename Context>
 void RandpermKernel(const Context& dev_ctx,
                     int n,
-                    DataType dtype,
+                    DataType dtype UNUSED,
                     DenseTensor* out) {
   T* out_data = dev_ctx.template Alloc<T>(out);
   int seed = 0;
diff --git a/paddle/phi/kernels/cpu/repeat_interleave_grad_kernel.cc b/paddle/phi/kernels/cpu/repeat_interleave_grad_kernel.cc
index d97e9b5603797..05f19ac36107e 100644
--- a/paddle/phi/kernels/cpu/repeat_interleave_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/repeat_interleave_grad_kernel.cc
@@ -26,7 +26,7 @@ namespace phi {
 template <typename T, typename Context>
 void RepeatInterleaveWithTensorIndexGradKernel(
     const Context& ctx,
-    const DenseTensor& x,
+    const DenseTensor& x UNUSED,
     const DenseTensor& repeats_tensor,
     const DenseTensor& out_grad,
     int dim,
@@ -74,7 +74,7 @@ void RepeatInterleaveWithTensorIndexGradKernel(
 
 template <typename T, typename Context>
 void RepeatInterleaveGradKernel(const Context& ctx,
-                                const DenseTensor& x,
+                                const DenseTensor& x UNUSED,
                                 const DenseTensor& out_grad,
                                 int repeats,
                                 int dim,
diff --git a/paddle/phi/kernels/cpu/rnn_grad_kernel.cc b/paddle/phi/kernels/cpu/rnn_grad_kernel.cc
index cb479cb755cfa..3f5395d514369 100644
--- a/paddle/phi/kernels/cpu/rnn_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/rnn_grad_kernel.cc
@@ -164,19 +164,19 @@ template <typename T, template <typename> class EigenActivationBackwardFunctor>
 struct SimpleRNNGradCell : GradCell<T> {
   void operator()(const CPUContext& dev_ctx,
                   DenseTensor* gate_tensor,
-                  DenseTensor* state_tensor,
-                  DenseTensor* act_state_tensor,
+                  DenseTensor* state_tensor UNUSED,
+                  DenseTensor* act_state_tensor UNUSED,
                   DenseTensor* hidden_tensor,
                   const DenseTensor* weight_hh,
                   DenseTensor* pre_hidden,
-                  DenseTensor* pre_state,
+                  DenseTensor* pre_state UNUSED,
                   DenseTensor* grad_hidden,
-                  DenseTensor* grad_state,
+                  DenseTensor* grad_state UNUSED,
                   DenseTensor* grad_gate,
                   DenseTensor* grad_weight_hh,
                   DenseTensor* grad_pre_hidden,
-                  DenseTensor* grad_pre_state,
-                  DenseTensor* grad_bias_hh,
+                  DenseTensor* grad_pre_state UNUSED,
+                  DenseTensor* grad_bias_hh UNUSED,
                   const DenseTensor& mask_tensor,
                   bool has_sequence_length) const override {
     DenseTensor grad_pre_hidden_bak;
diff --git a/paddle/phi/kernels/cpu/rnn_kernel.cc b/paddle/phi/kernels/cpu/rnn_kernel.cc
index de1caf16f621e..5163f319c05b6 100644
--- a/paddle/phi/kernels/cpu/rnn_kernel.cc
+++ b/paddle/phi/kernels/cpu/rnn_kernel.cc
@@ -57,13 +57,13 @@ struct SimpleRNNCell : Cell<T> {
                   DenseTensor* input,
                   const DenseTensor* weight_hh,
                   const DenseTensor* init_h,
-                  const DenseTensor* init_c,
-                  DenseTensor* last_h,
-                  DenseTensor* last_c,
-                  DenseTensor* last_c_act,
-                  DenseTensor* output,
-                  const DenseTensor* bias_hh,
-                  DenseTensor* weight_hh_gru) const override {
+                  const DenseTensor* init_c UNUSED,
+                  DenseTensor* last_h UNUSED,
+                  DenseTensor* last_c UNUSED,
+                  DenseTensor* last_c_act UNUSED,
+                  DenseTensor* output UNUSED,
+                  const DenseTensor* bias_hh UNUSED,
+                  DenseTensor* weight_hh_gru UNUSED) const override {
     auto blas = phi::funcs::GetBlas<CPUContext, T>(*dev_ctx);
     auto mat_dim_a =
         phi::funcs::CreateMatrixDescriptor(init_h->dims(), 0, false);
diff --git a/paddle/phi/kernels/cpu/scatter_grad_kernel.cc b/paddle/phi/kernels/cpu/scatter_grad_kernel.cc
index 9fb1136e766e6..d47e1c54976b2 100644
--- a/paddle/phi/kernels/cpu/scatter_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/scatter_grad_kernel.cc
@@ -25,9 +25,9 @@ namespace phi {
 template <typename T, typename Context>
 void ScatterGradKernel(const Context &ctx,
                        const DenseTensor &index,
-                       const DenseTensor &updates,
+                       const DenseTensor &updates UNUSED,
                        const DenseTensor &out_grad,
-                       bool overwrite,
+                       bool overwrite UNUSED,
                        DenseTensor *x_grad,
                        DenseTensor *updates_grad) {
   const auto &index_type = index.dtype();
diff --git a/paddle/phi/kernels/cpu/top_k_grad_kernel.cc b/paddle/phi/kernels/cpu/top_k_grad_kernel.cc
index a65c2324cc768..1bdcd4dd2547d 100644
--- a/paddle/phi/kernels/cpu/top_k_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/top_k_grad_kernel.cc
@@ -56,8 +56,8 @@ void TopkGradKernel(const Context& dev_ctx,
                     const DenseTensor& out_grad,
                     const Scalar& k_scalar,
                     int axis,
-                    bool largest,
-                    bool sorted,
+                    bool largest UNUSED,
+                    bool sorted UNUSED,
                     DenseTensor* x_grad) {
   const auto& in_dims = x.dims();
   const auto& out_dims = indices.dims();
diff --git a/paddle/phi/kernels/cpu/trunc_grad_kernel.cc b/paddle/phi/kernels/cpu/trunc_grad_kernel.cc
index 24fc389256222..fe14eb77f5afc 100644
--- a/paddle/phi/kernels/cpu/trunc_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/trunc_grad_kernel.cc
@@ -21,7 +21,7 @@ namespace phi {
 
 template <typename T, typename Context>
 void TruncGradKernel(const Context& dev_ctx,
-                     const DenseTensor& out_grad,
+                     const DenseTensor& out_grad UNUSED,
                      DenseTensor* in_grad) {
   T* dx_data = dev_ctx.template Alloc<T>(in_grad);
 
diff --git a/paddle/phi/kernels/cpu/where_grad_kernel.cc b/paddle/phi/kernels/cpu/where_grad_kernel.cc
index a9cdbd7ad77cc..19b2ee570ca30 100644
--- a/paddle/phi/kernels/cpu/where_grad_kernel.cc
+++ b/paddle/phi/kernels/cpu/where_grad_kernel.cc
@@ -21,8 +21,8 @@ namespace phi {
 template <typename T, typename Context>
 void WhereGradKernel(const Context& ctx,
                      const DenseTensor& condition,
-                     const DenseTensor& x,
-                     const DenseTensor& y,
+                     const DenseTensor& x UNUSED,
+                     const DenseTensor& y UNUSED,
                      const DenseTensor& out_grad,
                      DenseTensor* x_grad,
                      DenseTensor* y_grad) {

From 3e1fffea8f3dab5074c1d1bc7b79c02d63fa2c94 Mon Sep 17 00:00:00 2001
From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com>
Date: Mon, 15 May 2023 15:38:39 +0800
Subject: [PATCH 330/405] remove some [-Wunsed-parameter] warning (#53689)

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop
---
 .../collective/process_group_with_stream.h    | 102 +++++++++---------
 .../distributed/ps/service/heter_server.h     |  16 +--
 paddle/fluid/eager/eager_layout_auto_tune.h   |   4 +-
 paddle/fluid/framework/fleet/metrics.h        |   8 +-
 paddle/fluid/framework/op_registry.h          |   6 +-
 paddle/fluid/operators/activation_op.h        |   2 +-
 .../fluid/operators/cuda_graph_with_in_out.h  |   2 +-
 paddle/fluid/operators/math/beam_search.cc    |   2 +-
 paddle/fluid/operators/memcpy_d2h_op.h        |   2 +-
 paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc |   3 +-
 .../operators/mkldnn/matmul_mkldnn_op.cc      |   2 +-
 .../sequence_ops/sequence_expand_op.h         |   2 +-
 paddle/phi/backends/onednn/onednn_reuse.h     |   2 +-
 paddle/phi/kernels/funcs/blas/blas_impl.cu.h  |   2 +-
 .../phi/kernels/onednn/interpolate_kernel.cc  |   4 +-
 paddle/phi/kernels/onednn/pad3d_kernel.cc     |   4 +-
 paddle/phi/kernels/reshape_grad_kernel.cc     |   2 +-
 paddle/phi/kernels/reshape_kernel.cc          |   2 +-
 paddle/phi/kernels/share_buffer_kernel.cc     |   6 +-
 19 files changed, 89 insertions(+), 84 deletions(-)

diff --git a/paddle/fluid/distributed/collective/process_group_with_stream.h b/paddle/fluid/distributed/collective/process_group_with_stream.h
index cfdb7b16553f3..0cea9bb3ed87e 100644
--- a/paddle/fluid/distributed/collective/process_group_with_stream.h
+++ b/paddle/fluid/distributed/collective/process_group_with_stream.h
@@ -101,12 +101,12 @@ class ProcessGroupWithStream : public ProcessGroup {
   }
 
   std::shared_ptr<ProcessGroup::Task> AllGather(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      int64_t offset,
-      int64_t numel,
-      bool sync_op,
-      bool use_calc_stream) override {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      int64_t offset UNUSED,
+      int64_t numel UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) override {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroupWithStream (%s) does not support all_gather.",
         GetBackendName()));
@@ -125,11 +125,11 @@ class ProcessGroupWithStream : public ProcessGroup {
   }
 
   std::shared_ptr<ProcessGroup::Task> AllReduce(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const AllreduceOptions& opts,
-      bool sync_op,
-      bool use_calc_stream) override {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const AllreduceOptions& opts UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) override {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroupWithStream (%s) does not support all_reduce.",
         GetBackendName()));
@@ -150,12 +150,12 @@ class ProcessGroupWithStream : public ProcessGroup {
   }
 
   std::shared_ptr<ProcessGroup::Task> AllToAll(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const std::vector<int64_t>& out_size_each_rank,
-      const std::vector<int64_t>& in_size_each_rank,
-      bool sync_op,
-      bool use_calc_stream) override {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const std::vector<int64_t>& out_size_each_rank UNUSED,
+      const std::vector<int64_t>& in_size_each_rank UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) override {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroupWithStream (%s) does not support all_to_all.",
         GetBackendName()));
@@ -174,11 +174,11 @@ class ProcessGroupWithStream : public ProcessGroup {
   }
 
   std::shared_ptr<ProcessGroup::Task> Broadcast(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const BroadcastOptions& opts,
-      bool sync_op,
-      bool use_calc_stream) override {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const BroadcastOptions& opts UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) override {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroupWithStream (%s) does not support broadcast.",
         GetBackendName()));
@@ -195,11 +195,12 @@ class ProcessGroupWithStream : public ProcessGroup {
                   /*use_calc_stream*/ false);
   }
 
-  std::shared_ptr<ProcessGroup::Task> Reduce(phi::DenseTensor* out_tensor,
-                                             const phi::DenseTensor& in_tensor,
-                                             const ReduceOptions& opts,
-                                             bool sync_op,
-                                             bool use_calc_stream) override {
+  std::shared_ptr<ProcessGroup::Task> Reduce(
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const ReduceOptions& opts UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) override {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroupWithStream (%s) does not support reduce.",
         GetBackendName()));
@@ -218,11 +219,11 @@ class ProcessGroupWithStream : public ProcessGroup {
   }
 
   std::shared_ptr<ProcessGroup::Task> ReduceScatter(
-      phi::DenseTensor* out_tensor,
-      const phi::DenseTensor& in_tensor,
-      const ReduceScatterOptions& opts,
-      bool sync_op,
-      bool use_calc_stream) override {
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const ReduceScatterOptions& opts UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) override {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroupWithStream (%s) does not support reduce_scatter.",
         GetBackendName()));
@@ -239,11 +240,12 @@ class ProcessGroupWithStream : public ProcessGroup {
                    /*use_calc_stream*/ false);
   }
 
-  std::shared_ptr<ProcessGroup::Task> Scatter(phi::DenseTensor* out_tensor,
-                                              const phi::DenseTensor& in_tensor,
-                                              const ScatterOptions& opts,
-                                              bool sync_op,
-                                              bool use_calc_stream) override {
+  std::shared_ptr<ProcessGroup::Task> Scatter(
+      phi::DenseTensor* out_tensor UNUSED,
+      const phi::DenseTensor& in_tensor UNUSED,
+      const ScatterOptions& opts UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) override {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroupWithStream (%s) does not support scatter.",
         GetBackendName()));
@@ -284,12 +286,13 @@ class ProcessGroupWithStream : public ProcessGroup {
                 use_calc_stream);
   }
 
-  std::shared_ptr<ProcessGroup::Task> Recv(phi::DenseTensor* tensor,
-                                           int src_rank,
-                                           int64_t offset,
-                                           int64_t numel,
-                                           bool sync_op,
-                                           bool use_calc_stream) override {
+  std::shared_ptr<ProcessGroup::Task> Recv(phi::DenseTensor* tensor UNUSED,
+                                           int src_rank UNUSED,
+                                           int64_t offset UNUSED,
+                                           int64_t numel UNUSED,
+                                           bool sync_op UNUSED,
+                                           bool use_calc_stream
+                                               UNUSED) override {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroupWithStream (%s) does not support recv.",
         GetBackendName()));
@@ -330,12 +333,13 @@ class ProcessGroupWithStream : public ProcessGroup {
                 use_calc_stream);
   }
 
-  std::shared_ptr<ProcessGroup::Task> Send(const phi::DenseTensor& tensor,
-                                           int dst_rank,
-                                           int64_t offset,
-                                           int64_t numel,
-                                           bool sync_op,
-                                           bool use_calc_stream) override {
+  std::shared_ptr<ProcessGroup::Task> Send(
+      const phi::DenseTensor& tensor UNUSED,
+      int dst_rank UNUSED,
+      int64_t offset UNUSED,
+      int64_t numel UNUSED,
+      bool sync_op UNUSED,
+      bool use_calc_stream UNUSED) override {
     PADDLE_THROW(phi::errors::Unimplemented(
         "ProcessGroupWithStream (%s) does not support send.",
         GetBackendName()));
diff --git a/paddle/fluid/distributed/ps/service/heter_server.h b/paddle/fluid/distributed/ps/service/heter_server.h
index 88d37e6022ed5..c4e9d05ac1352 100755
--- a/paddle/fluid/distributed/ps/service/heter_server.h
+++ b/paddle/fluid/distributed/ps/service/heter_server.h
@@ -535,25 +535,25 @@ class HeterService : public PsService {
   bool IsExit() { return is_exit_; }
 
  private:
-  int32_t stop_profiler(const PsRequestMessage& request,
-                        PsResponseMessage& response,  // NOLINT
-                        brpc::Controller* cntl) {
+  int32_t stop_profiler(const PsRequestMessage& request UNUSED,
+                        PsResponseMessage& response UNUSED,  // NOLINT
+                        brpc::Controller* cntl UNUSED) {
     platform::DisableProfiler(
         platform::EventSortingKey::kDefault,
         string::Sprintf("heter_worker_%s_profile", endpoint_));
     return 0;
   }
 
-  int32_t start_profiler(const PsRequestMessage& request,
-                         PsResponseMessage& response,  // NOLINT
-                         brpc::Controller* cntl) {
+  int32_t start_profiler(const PsRequestMessage& request UNUSED,
+                         PsResponseMessage& response UNUSED,  // NOLINT
+                         brpc::Controller* cntl UNUSED) {
     platform::EnableProfiler(platform::ProfilerState::kAll);
     return 0;
   }
 
   int32_t stop_heter_worker(const PsRequestMessage& request,
-                            PsResponseMessage& response,  // NOLINT
-                            brpc::Controller* cntl) {
+                            PsResponseMessage& response UNUSED,  // NOLINT
+                            brpc::Controller* cntl UNUSED) {
     auto client_id = request.client_id();
     stop_cpu_worker_set_.insert(client_id);
     if (stop_cpu_worker_set_.size() == fan_in_) {
diff --git a/paddle/fluid/eager/eager_layout_auto_tune.h b/paddle/fluid/eager/eager_layout_auto_tune.h
index 61a5218d36ce3..f5e77113ba8ea 100644
--- a/paddle/fluid/eager/eager_layout_auto_tune.h
+++ b/paddle/fluid/eager/eager_layout_auto_tune.h
@@ -65,7 +65,7 @@ inline std::shared_ptr<EagerLayoutTransformer> EagerLayoutAutotune(
     const std::string& op_name,
     const paddle::small_vector<std::vector<paddle::Tensor>,
                                kSlotSmallVectorSize>& tensors_vector,
-    T* attr) {
+    T* attr UNUSED) {
   // For lightly op like reduce
   if (!(DesiredLayout() == phi::DataLayout::UNDEFINED)) {
     VLOG(4) << "LayoutAutotune was unstarted. Current op :" << op_name;
@@ -81,7 +81,7 @@ inline std::shared_ptr<EagerLayoutTransformer> EagerLayoutAutotune(
     const paddle::small_vector<std::vector<paddle::Tensor>,
                                kSlotSmallVectorSize>& tensors_vector,
     T1* axis,
-    T2* keep_dim) {
+    T2* keep_dim UNUSED) {
   // For lightly op like argmax
   return EagerLayoutAutotune<T1>(op_name, tensors_vector, axis);
 }
diff --git a/paddle/fluid/framework/fleet/metrics.h b/paddle/fluid/framework/fleet/metrics.h
index 6212eec1abb47..6542e1063b212 100644
--- a/paddle/fluid/framework/fleet/metrics.h
+++ b/paddle/fluid/framework/fleet/metrics.h
@@ -226,7 +226,7 @@ class Metric {
                    const std::string& pred_varname,
                    const std::string& uid_varname,
                    int metric_phase,
-                   int bucket_size = 1000000) {
+                   int bucket_size UNUSED = 1000000) {
       label_varname_ = label_varname;
       pred_varname_ = pred_varname;
       uid_varname_ = uid_varname;
@@ -297,7 +297,7 @@ class Metric {
     }
     virtual ~MultiTaskMetricMsg() {}
     void add_data(const Scope* exe_scope,
-                  const paddle::platform::Place& place) override {
+                  const paddle::platform::Place& place UNUSED) override {
       std::vector<int64_t> cmatch_rank_data;
       get_data<int64_t>(exe_scope, cmatch_rank_varname_, &cmatch_rank_data);
       std::vector<int64_t> label_data;
@@ -378,7 +378,7 @@ class Metric {
     }
     virtual ~CmatchRankMetricMsg() {}
     void add_data(const Scope* exe_scope,
-                  const paddle::platform::Place& place) override {
+                  const paddle::platform::Place& place UNUSED) override {
       std::vector<int64_t> cmatch_rank_data;
       get_data<int64_t>(exe_scope, cmatch_rank_varname_, &cmatch_rank_data);
       std::vector<int64_t> label_data;
@@ -502,7 +502,7 @@ class Metric {
     }
     virtual ~CmatchRankMaskMetricMsg() {}
     void add_data(const Scope* exe_scope,
-                  const paddle::platform::Place& place) override {
+                  const paddle::platform::Place& place UNUSED) override {
       std::vector<int64_t> cmatch_rank_data;
       get_data<int64_t>(exe_scope, cmatch_rank_varname_, &cmatch_rank_data);
       std::vector<int64_t> label_data;
diff --git a/paddle/fluid/framework/op_registry.h b/paddle/fluid/framework/op_registry.h
index 1d2a372684339..8cb29a0d5df4c 100644
--- a/paddle/fluid/framework/op_registry.h
+++ b/paddle/fluid/framework/op_registry.h
@@ -257,9 +257,9 @@ struct OpKernelRegistrarFunctorEx<PlaceType,
                                   true,
                                   I,
                                   DataTypeAndKernelType...> {
-  void operator()(const char* op_type,
-                  const char* library_type,
-                  int customized_type_value) const {}
+  void operator()(const char* op_type UNUSED,
+                  const char* library_type UNUSED,
+                  int customized_type_value UNUSED) const {}
 };
 
 template <typename PlaceType, size_t I, typename... DataTypeAndKernelType>
diff --git a/paddle/fluid/operators/activation_op.h b/paddle/fluid/operators/activation_op.h
index 609938f1e6674..8280c817b706a 100644
--- a/paddle/fluid/operators/activation_op.h
+++ b/paddle/fluid/operators/activation_op.h
@@ -334,7 +334,7 @@ struct SoftReluGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x UNUSED, Out out, dOut dout, dX dx) const {
     auto tmp = static_cast<T>(threshold);
     auto temp = ((out > -tmp) * (out < tmp)).template cast<T>();
     dx.device(d) = dout * (static_cast<T>(1) - (-out).exp()) * temp;
diff --git a/paddle/fluid/operators/cuda_graph_with_in_out.h b/paddle/fluid/operators/cuda_graph_with_in_out.h
index 37be7bad1b3cb..3f65450d30c0e 100644
--- a/paddle/fluid/operators/cuda_graph_with_in_out.h
+++ b/paddle/fluid/operators/cuda_graph_with_in_out.h
@@ -110,7 +110,7 @@ static std::unique_ptr<CUDAGraphWithInOuts> CaptureCUDAGraph(
     inputs.insert(inputs.end(), input_tensors.begin(), input_tensors.end());
   }
 
-  auto func = [&](const std::vector<const phi::DenseTensor *> &inputs) {
+  auto func = [&](const std::vector<const phi::DenseTensor *> &inputs UNUSED) {
     callable(ctx);
     std::vector<phi::DenseTensor *> outputs;
     for (const auto &name : output_names) {
diff --git a/paddle/fluid/operators/math/beam_search.cc b/paddle/fluid/operators/math/beam_search.cc
index 6e5d9b87175cc..2b6599e4f39ff 100644
--- a/paddle/fluid/operators/math/beam_search.cc
+++ b/paddle/fluid/operators/math/beam_search.cc
@@ -25,7 +25,7 @@ namespace math {
 template <typename T>
 class BeamSearchFunctor<phi::CPUContext, T> {
  public:
-  void operator()(const phi::CPUContext &context,
+  void operator()(const phi::CPUContext &context UNUSED,
                   const phi::DenseTensor *pre_ids,
                   const phi::DenseTensor *pre_scores,
                   const phi::DenseTensor *ids,
diff --git a/paddle/fluid/operators/memcpy_d2h_op.h b/paddle/fluid/operators/memcpy_d2h_op.h
index fa2ba1cdfc5e5..4f948e4482f8a 100644
--- a/paddle/fluid/operators/memcpy_d2h_op.h
+++ b/paddle/fluid/operators/memcpy_d2h_op.h
@@ -58,7 +58,7 @@ class MemcpyD2HFunctor {
   }
 
   template <typename T>
-  void operator()(const T &v) const {
+  void operator()(const T &v UNUSED) const {
     PADDLE_ENFORCE_EQ(
         true,
         false,
diff --git a/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
index d408514e83921..e175e848dee3f 100644
--- a/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
@@ -403,7 +403,8 @@ class FCMKLDNNKernel : public framework::OpKernel<T_in> {
                              }));
   }
 
-  void PrepareSrcMem(const std::shared_ptr<dnnl::inner_product_forward>& fc_p,
+  void PrepareSrcMem(const std::shared_ptr<dnnl::inner_product_forward>& fc_p
+                         UNUSED,
                      const std::shared_ptr<dnnl::memory>& src_mem,
                      const phi::DenseTensor* x,
                      const dnnl::engine& engine) const {
diff --git a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc
index 04338fc866954..ffd72b9681233 100644
--- a/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/matmul_mkldnn_op.cc
@@ -411,7 +411,7 @@ class MatMulMKLDNNKernel : public paddle::framework::OpKernel<T> {
   }
 
  private:
-  void CalculateMatrixDims(const ExecutionContext &ctx,
+  void CalculateMatrixDims(const ExecutionContext &ctx UNUSED,
                            const std::vector<int64_t> &x_dims,
                            const std::vector<int64_t> &y_dims,
                            std::vector<int64_t> *x_bd_dims,
diff --git a/paddle/fluid/operators/sequence_ops/sequence_expand_op.h b/paddle/fluid/operators/sequence_ops/sequence_expand_op.h
index ecaf260b1c576..767942b4db874 100644
--- a/paddle/fluid/operators/sequence_ops/sequence_expand_op.h
+++ b/paddle/fluid/operators/sequence_ops/sequence_expand_op.h
@@ -48,7 +48,7 @@ struct SequenceExpandGradFunctor {
 
 template <typename T>
 struct SequenceExpandFunctor<phi::CPUContext, T> {
-  void operator()(const phi::CPUContext& context,
+  void operator()(const phi::CPUContext& context UNUSED,
                   const phi::DenseTensor& x,
                   const phi::Vector<size_t>& x_lod,   /*expand source lod*/
                   const phi::Vector<size_t>& ref_lod, /*expand referenced lod*/
diff --git a/paddle/phi/backends/onednn/onednn_reuse.h b/paddle/phi/backends/onednn/onednn_reuse.h
index 330f0318a6e09..24181e6267ba5 100644
--- a/paddle/phi/backends/onednn/onednn_reuse.h
+++ b/paddle/phi/backends/onednn/onednn_reuse.h
@@ -1115,7 +1115,7 @@ class PReluOneDNNHandler
                      const DenseTensor& x,
                      const DenseTensor& weights,
                      const std::string& mode,
-                     const std::string& data_format,
+                     const std::string& data_format UNUSED,
                      const bool is_test)
       : OneDNNHandlerNoCachingT<T, dnnl::prelu_forward, dnnl::prelu_backward>(
             engine, cpu_place) {
diff --git a/paddle/phi/kernels/funcs/blas/blas_impl.cu.h b/paddle/phi/kernels/funcs/blas/blas_impl.cu.h
index e39c860bc8096..c08903e7d37c6 100644
--- a/paddle/phi/kernels/funcs/blas/blas_impl.cu.h
+++ b/paddle/phi/kernels/funcs/blas/blas_impl.cu.h
@@ -189,7 +189,7 @@ struct CUBlas<double> {
   }
 
   template <typename... ARGS>
-  static void GEMM_EX(ARGS... args) {
+  static void GEMM_EX(ARGS... args UNUSED) {
     PADDLE_THROW(
         phi::errors::Unimplemented("Currently there are not cublasDgemmEx."));
   }
diff --git a/paddle/phi/kernels/onednn/interpolate_kernel.cc b/paddle/phi/kernels/onednn/interpolate_kernel.cc
index 508cdcc329a2e..0481453a91cbd 100644
--- a/paddle/phi/kernels/onednn/interpolate_kernel.cc
+++ b/paddle/phi/kernels/onednn/interpolate_kernel.cc
@@ -209,8 +209,8 @@ void BilinearInterpKernel(
     int out_w,
     const std::vector<float>& scale,
     const std::string& interp_method,
-    bool align_corners,
-    int align_mode,
+    bool align_corners UNUSED,
+    int align_mode UNUSED,
     DenseTensor* output) {
   InterpolateKernel<T, Context>(ctx,
                                 x,
diff --git a/paddle/phi/kernels/onednn/pad3d_kernel.cc b/paddle/phi/kernels/onednn/pad3d_kernel.cc
index b0db38ccec1ab..6bb3c950057b0 100644
--- a/paddle/phi/kernels/onednn/pad3d_kernel.cc
+++ b/paddle/phi/kernels/onednn/pad3d_kernel.cc
@@ -24,9 +24,9 @@ template <typename T, typename Context>
 void Pad3dKernel(const Context& dev_ctx,
                  const DenseTensor& x,
                  const IntArray& paddings,
-                 const std::string& mode,
+                 const std::string& mode UNUSED,
                  float pad_value,
-                 const std::string& data_format,
+                 const std::string& data_format UNUSED,
                  DenseTensor* out) {
   PadOpKernel<T, Context>(dev_ctx, x, paddings.GetData(), pad_value, out);
 }
diff --git a/paddle/phi/kernels/reshape_grad_kernel.cc b/paddle/phi/kernels/reshape_grad_kernel.cc
index ef4c7d4a6f7e1..28a8e6b5e056b 100644
--- a/paddle/phi/kernels/reshape_grad_kernel.cc
+++ b/paddle/phi/kernels/reshape_grad_kernel.cc
@@ -53,7 +53,7 @@ void ReshapeGradKernel<phi::XPUContext>(const XPUContext& dev_ctx,
 
 template <typename Context>
 void ReshapeDoubleGradKernel(const Context& dev_ctx,
-                             const DenseTensor& out_grad,
+                             const DenseTensor& out_grad UNUSED,
                              const DenseTensor& x_grad_grad,
                              DenseTensor* out_grad_grad) {
   ReshapeGradKernel(dev_ctx, x_grad_grad, out_grad_grad);
diff --git a/paddle/phi/kernels/reshape_kernel.cc b/paddle/phi/kernels/reshape_kernel.cc
index 47ebf8cf2925a..934653928e10f 100644
--- a/paddle/phi/kernels/reshape_kernel.cc
+++ b/paddle/phi/kernels/reshape_kernel.cc
@@ -81,7 +81,7 @@ void ReshapeKernel(const Context& dev_ctx,
                    const DenseTensor& x,
                    const IntArray& shape,
                    DenseTensor* out,
-                   DenseTensor* xshape) {
+                   DenseTensor* xshape UNUSED) {
   ReshapeInferKernel(dev_ctx, x, shape, out);
 }
 
diff --git a/paddle/phi/kernels/share_buffer_kernel.cc b/paddle/phi/kernels/share_buffer_kernel.cc
index dd4272a591bf4..9ffcddae0c1e4 100644
--- a/paddle/phi/kernels/share_buffer_kernel.cc
+++ b/paddle/phi/kernels/share_buffer_kernel.cc
@@ -22,11 +22,11 @@
 namespace phi {
 
 template <typename Context>
-void ShareBufferKernel(const Context &dev_ctx,
+void ShareBufferKernel(const Context &dev_ctx UNUSED,
                        const std::vector<const DenseTensor *> &x,
-                       const std::vector<bool> &share_dims_and_dtype,
+                       const std::vector<bool> &share_dims_and_dtype UNUSED,
                        std::vector<DenseTensor *> out,
-                       std::vector<DenseTensor *> xout) {
+                       std::vector<DenseTensor *> xout UNUSED) {
   PADDLE_ENFORCE_EQ(
       x.size(),
       out.size(),

From 972daa46086f197ad31530bd1ee04e76d609602d Mon Sep 17 00:00:00 2001
From: zhwesky2010 <1183042833@qq.com>
Date: Mon, 15 May 2023 16:17:49 +0800
Subject: [PATCH 331/405] [BUG] fix windows kernel dispatch of _lzcnt bug
 (#53728)

---
 paddle/phi/api/lib/kernel_dispatch.cc     |  5 ++++-
 paddle/phi/kernels/gpu/randperm_kernel.cu |  2 --
 paddle/scripts/paddle_build.bat           | 20 +++++++++-----------
 3 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/paddle/phi/api/lib/kernel_dispatch.cc b/paddle/phi/api/lib/kernel_dispatch.cc
index 90a0afcf1e6d3..81b90769e819e 100644
--- a/paddle/phi/api/lib/kernel_dispatch.cc
+++ b/paddle/phi/api/lib/kernel_dispatch.cc
@@ -78,7 +78,10 @@ std::size_t CountLeadingZeros(uint32_t val) {
 #if defined(__clang__) || defined(__GNUC__)
   return __builtin_clz(val);
 #elif defined(_MSC_VER)
-  return __lzcnt(val);
+  // windows don't have built-in clz/ctz function
+  DWORD Index;
+  _BitScanReverse(&Index, val);
+  return (uint32_t)Index ^ 31;
 #else
   if (val == 0) {
     return 32;
diff --git a/paddle/phi/kernels/gpu/randperm_kernel.cu b/paddle/phi/kernels/gpu/randperm_kernel.cu
index 456d541451e64..2ad512701e097 100644
--- a/paddle/phi/kernels/gpu/randperm_kernel.cu
+++ b/paddle/phi/kernels/gpu/randperm_kernel.cu
@@ -35,8 +35,6 @@ namespace cub = hipcub;
 #include "paddle/phi/kernels/funcs/for_range.h"
 #include "paddle/phi/kernels/randint_kernel.h"
 
-DECLARE_bool(use_curand);
-
 namespace phi {
 
 template <typename keyT, typename dataT>
diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat
index 16e1befa025d2..b0ea6905c94cf 100644
--- a/paddle/scripts/paddle_build.bat
+++ b/paddle/scripts/paddle_build.bat
@@ -211,7 +211,7 @@ if "%WITH_SCCACHE%"=="ON" (
 :install_sccache
 echo There is not sccache in this PC, will install sccache.
 echo Download package from https://paddle-ci.gz.bcebos.com/window_requirement/sccache.exe
-%PYTHON_ROOT%\python.exe -c "import wget;wget.download('https://paddle-ci.gz.bcebos.com/window_requirement/sccache.exe')"
+python -c "import wget;wget.download('https://paddle-ci.gz.bcebos.com/window_requirement/sccache.exe')"
 xcopy sccache.exe %SCCACHE_ROOT%\ /Y
 del sccache.exe
 goto:eof
@@ -429,7 +429,7 @@ if not exist %THIRD_PARTY_PATH% (
     if not exist %THIRD_PARTY_HOME% mkdir "%THIRD_PARTY_HOME%"
     cd /d %THIRD_PARTY_HOME%
     echo Getting third party: downloading ...
-    %PYTHON_ROOT%\python.exe -c "import wget;wget.download('https://paddle-windows.bj.bcebos.com/third_party/%sub_dir%/%md5%.tar.gz')" 2>nul
+    python -c "import wget;wget.download('https://paddle-windows.bj.bcebos.com/third_party/%sub_dir%/%md5%.tar.gz')" 2>nul
     if !ERRORLEVEL! EQU 0 (
         echo Getting third party: extracting ...
         tar -xf %md5%.tar.gz
@@ -572,17 +572,15 @@ if %ERRORLEVEL% NEQ 0 (
 )
 
 if "%UPLOAD_TP_FILE%"=="ON" (
-    set BCE_FILE=%cache_dir%\bce-python-sdk-0.8.33\BosClient.py
+    set BCE_FILE=%cache_dir%\bce-python-sdk-new\BosClient.py
     echo Uploading third_party: checking bce ...
-    if not exist %cache_dir%\bce-python-sdk-0.8.33 (
+    if not exist %cache_dir%\bce-python-sdk-new (
         echo There is no bce in this PC, will install bce.
         cd /d %cache_dir%
-        echo Download package from https://paddle-windows.bj.bcebos.com/bce-python-sdk-0.8.33.tar.gz
-        %PYTHON_ROOT%\python.exe -c "import wget;wget.download('https://paddle-windows.bj.bcebos.com/bce-python-sdk-0.8.33.tar.gz')"
-        %PYTHON_ROOT%\python.exe -c "import shutil;shutil.unpack_archive('bce-python-sdk-0.8.33.tar.gz', extract_dir='./',format='gztar')"
-        cd /d %cache_dir%\bce-python-sdk-0.8.33
-        %PYTHON_ROOT%\python.exe setup.py install 1>nul
-        del %cache_dir%\bce-python-sdk-0.8.33.tar.gz
+        echo Download package from https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz
+        python -c "import wget;wget.download('https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz')"
+        python -c "import shutil;shutil.unpack_archive('bos_new.tar.gz', extract_dir='./bce-python-sdk-new',format='gztar')"
+        python -m pip install bce-python-sdk
     )
     if !errorlevel! EQU 0 (
         cd /d %THIRD_PARTY_HOME%
@@ -590,7 +588,7 @@ if "%UPLOAD_TP_FILE%"=="ON" (
         tar -zcf %md5%.tar.gz %md5%
         if !errorlevel! EQU 0 (
             echo Uploading third_party: uploading ...
-            %PYTHON_ROOT%\python.exe !BCE_FILE! %md5%.tar.gz paddle-windows/third_party/%sub_dir% 1>nul
+            python !BCE_FILE! %md5%.tar.gz paddle-windows/third_party/%sub_dir% 1>nul
             if !errorlevel! EQU 0 (
                 echo Upload third party %md5% to bos paddle-windows/third_party/%sub_dir% successfully.
             ) else (

From 0ef51804ef6cd755475e6e6fbd15cc51fb240b6e Mon Sep 17 00:00:00 2001
From: chalsliu <45041955+chalsliu@users.noreply.github.com>
Date: Mon, 15 May 2023 16:19:38 +0800
Subject: [PATCH 332/405] Reduce inference library size and compile time
 (#53369)

* Reduce inference library size and compile time

* resolve conflicts
---
 cmake/phi.cmake                    |   5 ++
 paddle/phi/kernels/CMakeLists.txt  |   9 ++
 tools/reduce_lib_size_util.py      | 128 +++++++++++++++++++++++++++++
 tools/remove_grad_op_and_kernel.py |  50 ++++++++---
 4 files changed, 182 insertions(+), 10 deletions(-)
 create mode 100644 tools/reduce_lib_size_util.py

diff --git a/cmake/phi.cmake b/cmake/phi.cmake
index 96760969adf90..f97d5d3f8f74f 100644
--- a/cmake/phi.cmake
+++ b/cmake/phi.cmake
@@ -64,6 +64,11 @@ function(generate_unify_header DIR_NAME)
       endif()
     endif()
   endforeach()
+  if(DEFINED REDUCE_INFERENCE_LIB_SIZE)
+    if(${kernel_name} MATCHES ".*_grad")
+      continue()
+    endif()
+  endif()
   # append header into extension.h
   string(REPLACE "${PADDLE_SOURCE_DIR}\/" "" header_file "${header_file}")
   file(APPEND ${phi_extension_header_file} "#include \"${header_file}\"\n")
diff --git a/paddle/phi/kernels/CMakeLists.txt b/paddle/phi/kernels/CMakeLists.txt
index e617dac99e99c..932091b087c09 100644
--- a/paddle/phi/kernels/CMakeLists.txt
+++ b/paddle/phi/kernels/CMakeLists.txt
@@ -118,6 +118,11 @@ file(
   "strings/gpu/*.cu"
   "fusion/gpu/*.cu")
 
+if(DEFINED REDUCE_INFERENCE_LIB_SIZE)
+  list(FILTER kernel_cu EXCLUDE REGEX ".*_grad_kernel\\.cc$")
+  list(FILTER kernel_cu EXCLUDE REGEX ".*_grad_kernel\\.cu$")
+endif()
+
 if(WITH_CUTLASS)
   execute_process(
     COMMAND ${CMAKE_COMMAND} -E make_directory
@@ -184,6 +189,10 @@ else()
     "fusion/cpu/*.cc")
 endif()
 
+if(DEFINED REDUCE_INFERENCE_LIB_SIZE)
+  list(FILTER kernel_cc EXCLUDE REGEX ".*_grad_kernel\\.cc$")
+endif()
+
 file(
   GLOB
   kernel_xpu
diff --git a/tools/reduce_lib_size_util.py b/tools/reduce_lib_size_util.py
new file mode 100644
index 0000000000000..0815e7b1ddc4d
--- /dev/null
+++ b/tools/reduce_lib_size_util.py
@@ -0,0 +1,128 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This script simply removes grad kernels. You should use this script
+when cmake ON_INFER=ON, which can greatly reduce the volume of the inference library.
+"""
+
+import glob
+import os
+
+
+def is_balanced(content):
+    """
+    Check whether sequence contains valid parenthesis.
+    Args:
+       content (str): content of string.
+
+    Returns:
+        boolean: True if sequence contains valid parenthesis.
+    """
+
+    if content.find('{') == -1:
+        return False
+    stack = []
+    push_chars, pop_chars = '({', ')}'
+    for c in content:
+        if c in push_chars:
+            stack.append(c)
+        elif c in pop_chars:
+            if not len(stack):
+                return False
+            else:
+                stack_top = stack.pop()
+                balancing_bracket = push_chars[pop_chars.index(c)]
+                if stack_top != balancing_bracket:
+                    return False
+    return not stack
+
+
+def grad_kernel_definition(content, kernel_pattern, grad_pattern):
+    """
+    Args:
+       content(str): file content
+       kernel_pattern(str): kernel pattern
+       grad_pattern(str): grad pattern
+
+    Returns:
+        (list, int): grad kernel definitions in file and count.
+    """
+
+    results = []
+    count = 0
+    start = 0
+    lens = len(content)
+    while True:
+        index = content.find(kernel_pattern, start)
+        if index == -1:
+            return results, count
+        i = index + 1
+        while i <= lens:
+            check_str = content[index:i]
+            if is_balanced(check_str):
+                if check_str.find(grad_pattern) != -1:
+                    results.append(check_str)
+                    count += 1
+                start = i
+                break
+            i += 1
+        else:
+            return results, count
+
+
+def remove_grad_kernels(dry_run=False):
+    """
+    Args:
+       dry_run(bool): whether just print
+
+    Returns:
+        int: number of kernel(grad) removed
+    """
+
+    pd_kernel_pattern = 'PD_REGISTER_STRUCT_KERNEL'
+    register_op_pd_kernel_count = 0
+    matches = []
+
+    tool_dir = os.path.dirname(os.path.abspath(__file__))
+    all_op = glob.glob(
+        os.path.join(tool_dir, '../paddle/fluid/operators/**/*.cc'),
+        recursive=True,
+    )
+    all_op += glob.glob(
+        os.path.join(tool_dir, '../paddle/fluid/operators/**/*.cu'),
+        recursive=True,
+    )
+
+    for op_file in all_op:
+        with open(op_file, 'r', encoding='utf-8') as f:
+            content = ''.join(f.readlines())
+
+            pd_kernel, pd_kernel_count = grad_kernel_definition(
+                content, pd_kernel_pattern, '_grad,'
+            )
+
+            register_op_pd_kernel_count += pd_kernel_count
+
+            matches.extend(pd_kernel)
+
+        for to_remove in matches:
+            content = content.replace(to_remove, '')
+            if dry_run:
+                print(op_file, to_remove)
+
+        if not dry_run:
+            with open(op_file, 'w', encoding='utf-8') as f:
+                f.write(content)
+
+    return register_op_pd_kernel_count
diff --git a/tools/remove_grad_op_and_kernel.py b/tools/remove_grad_op_and_kernel.py
index 870640347187d..823707f904d9d 100644
--- a/tools/remove_grad_op_and_kernel.py
+++ b/tools/remove_grad_op_and_kernel.py
@@ -12,14 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-This script simply removes all grad ops and kernels. You should use this script
-when cmake ON_INFER=ON, which can greatly reduce the volume of the prediction library.
+This script simply removes grad ops and kernels. You should use this script
+when cmake ON_INFER=ON, which can greatly reduce the volume of the inference library.
 """
 
+import argparse
 import glob
 import os
 import re
 
+import reduce_lib_size_util
+
+
+def parse_args():
+    """Parse input arguments."""
+    parser = argparse.ArgumentParser(description='Remove grad op and kernels.')
+    parser.add_argument('--only_kernel', action='store_true', default=False)
+    parser.add_argument('--dry_run', action='store_true', default=False)
+
+    args = parser.parse_args()
+    return args
+
 
 def find_type_files(cur_dir, file_type, file_list=[]):
     next_level_dirs = os.listdir(cur_dir)
@@ -42,6 +55,10 @@ def remove_grad_op_and_kernel(content, pattern1, pattern2):
 
 
 def update_operator_cmake(cmake_file):
+    """Update operator cmake.
+    Args:
+        cmake_file (str): cmake file path.
+    """
     pat1 = 'add_subdirectory(optimizers)'
     pat2 = r'register_operators\(EXCLUDES.*?py_func_op.*?\)'
 
@@ -66,6 +83,8 @@ def update_operator_cmake(cmake_file):
 
 if __name__ == '__main__':
 
+    args = parse_args()
+
     tool_dir = os.path.dirname(os.path.abspath(__file__))
 
     all_op = glob.glob(
@@ -92,14 +111,17 @@ def update_operator_cmake(cmake_file):
         # remove all grad op
         op_pattern1 = r'REGISTER_OPERATOR\(.*?\);?'
         op_pattern2 = r'REGISTER_OPERATOR\(.*?_grad,.*?\);?'
+        if args.only_kernel:
+            op_pattern1 = 'DISABLE_REMOVE_GRAD_OP_' + op_pattern1
+            op_pattern2 = 'DISABLE_REMOVE_GRAD_OP_' + op_pattern2
 
         # remove all cpu grad kernel
-        cpu_kernel_pattern1 = r'REGISTER_OP_CPU_KERNEL\(.*?\);?'
-        cpu_kernel_pattern2 = r'REGISTER_OP_CPU_KERNEL\(.*?_grad,.*?\);?'
+        cpu_kernel_pattern1 = r'REGISTER_OP_CPU_KERNEL\(.*?\);?|REGISTER_OP_CPU_KERNEL_FUNCTOR\(.*?\);?'
+        cpu_kernel_pattern2 = r'REGISTER_OP_CPU_KERNEL\(.*?_grad,.*?\);?|REGISTER_OP_CPU_KERNEL_FUNCTOR\(.*?_grad,.*?\);?'
 
         # remove all gpu grad kernel
-        gpu_kernel_pattern1 = r'REGISTER_OP_CUDA_KERNEL\(.*?\);?'
-        gpu_kernel_pattern2 = r'REGISTER_OP_CUDA_KERNEL\(.*?_grad,.*?\);?'
+        gpu_kernel_pattern1 = r'REGISTER_OP_CUDA_KERNEL\(.*?\);?|REGISTER_OP_CUDA_KERNEL_FUNCTOR\(.*?\);?'
+        gpu_kernel_pattern2 = r'REGISTER_OP_CUDA_KERNEL\(.*?_grad,.*?\);?|REGISTER_OP_CUDA_KERNEL_FUNCTOR\(.*?_grad,.*?\);?'
 
         # remove all xpu grad kernel
         xpu_kernel_pattern1 = r'REGISTER_OP_XPU_KERNEL\(.*?\);?'
@@ -166,11 +188,14 @@ def update_operator_cmake(cmake_file):
             all_matches.extend(op_kernel)
             all_matches.extend(custom_kernel)
 
-        for i in all_matches:
-            content = content.replace(i, '')
+        for to_remove in all_matches:
+            content = content.replace(to_remove, '')
+            if args.dry_run:
+                print(op_file, to_remove)
 
-        with open(op_file, 'w', encoding='utf-8') as f:
-            f.write(content)
+        if not args.dry_run:
+            with open(op_file, 'w', encoding='utf-8') as f:
+                f.write(content)
 
     # 2. update operators/CMakeLists.txt
     cmake_file = os.path.join(
@@ -178,6 +203,10 @@ def update_operator_cmake(cmake_file):
     )
     update_operator_cmake(cmake_file)
 
+    register_pd_kernel_count = reduce_lib_size_util.remove_grad_kernels(
+        args.dry_run
+    )
+
     print('We erase all grad op and kernel for Paddle-Inference lib.')
     print('%50s%10s' % ('type', 'count'))
     print('%50s%10s' % ('REGISTER_OPERATOR', register_op_count))
@@ -194,3 +223,4 @@ def update_operator_cmake(cmake_file):
             register_op_kernel_with_custom_type_count,
         )
     )
+    print('%50s%10s' % ('REGISTER_OP_PD_KERNEL', register_pd_kernel_count))

From 94c38803cec36a54b8fa32900674b6fc919d28e1 Mon Sep 17 00:00:00 2001
From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com>
Date: Mon, 15 May 2023 16:59:39 +0800
Subject: [PATCH 333/405] Silu double grad (#53605)

* add rules

* modify no kernel yaml parse

* success op generate

* success test_silu_double

* modify bug

* modify static error

* modify silu_grad input

* modify kernel signature

* modify kernel signature

* code style

* code style

* review

* delete opinfo modify
---
 .../generator/eager_gen.py                    |   1 +
 .../fluid/operators/generator/generate_op.py  |  15 +
 .../operators/generator/generate_sparse_op.py |   2 +
 .../fluid/operators/generator/parse_utils.py  |  17 +-
 .../operators/generator/templates/op.c.j2     |   2 +
 .../generator/templates/operator_utils.c.j2   |   3 +-
 .../fluid/operators/generator/tests_utils.py  |   4 +
 .../composite_backward_api.h                  | 330 +--------------
 .../composite_double_backward_api.h           | 387 ++++++++++++++++++
 paddle/phi/api/yaml/backward.yaml             |   7 +
 paddle/phi/api/yaml/generator/api_base.py     |  31 +-
 .../api/yaml/generator/backward_api_gen.py    |   2 +-
 paddle/phi/api/yaml/op_compat.yaml            |   2 +-
 test/prim/prim/vjp/test_comp_high_grad.py     |  83 ++++
 14 files changed, 529 insertions(+), 357 deletions(-)
 create mode 100644 paddle/fluid/prim/api/composite_backward/composite_double_backward_api.h

diff --git a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
index 8d28fa2438f23..b556062343d62 100644
--- a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
+++ b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
@@ -68,6 +68,7 @@
     "matmul_double_grad",
     "tanh_double_grad",
     "subtract_double_grad",
+    "silu_double_grad",
 ]
 
 # dict of special api that forward api's output will affect bacward api's output
diff --git a/paddle/fluid/operators/generator/generate_op.py b/paddle/fluid/operators/generator/generate_op.py
index 6f1ea3d8b3cce..dad5df7430d47 100644
--- a/paddle/fluid/operators/generator/generate_op.py
+++ b/paddle/fluid/operators/generator/generate_op.py
@@ -41,6 +41,7 @@
     is_base_op,
     is_composite_op,
     is_initializer_list,
+    is_only_composite_op,
     is_scalar,
     is_vec,
     supports_inplace,
@@ -72,6 +73,7 @@
 env.filters["find_optinal_inputs_name"] = find_optinal_inputs_name
 env.tests["base_op"] = is_base_op
 env.tests["composite_op"] = is_composite_op
+env.tests["only_composite_op"] = is_only_composite_op
 env.tests["vec"] = is_vec
 env.tests["scalar"] = is_scalar
 env.tests["initializer_list"] = is_initializer_list
@@ -165,6 +167,16 @@ def add_composite_info(ops, backward_ops, backward_op_dict):
         else:
             op["backward_composite"] = None
 
+        # add whether only composite
+        if (
+            op["backward_composite"] is not None
+            and "invoke" not in backward_op_dict[op["backward"]]
+            and "kernel" not in backward_op_dict[op["backward"]]
+        ):
+            op["only_backward_composite"] = True
+        else:
+            op["only_backward_composite"] = False
+
 
 # add fluid name in ops and backward ops info
 def add_fluid_name(dict_list):
@@ -248,6 +260,9 @@ def update_common_params_name(
                 for param in op_item['invoke']['args'].split(',')
             ]
             return
+        elif 'composite' in op_item and 'kernel' not in op_item:
+            return
+
         op_item['infer_meta']['param'] = get_param_list_alias(
             op_item['infer_meta']['param'], args_name_map
         )
diff --git a/paddle/fluid/operators/generator/generate_sparse_op.py b/paddle/fluid/operators/generator/generate_sparse_op.py
index 2635f0f67a1c5..9c92aa3bc3c94 100644
--- a/paddle/fluid/operators/generator/generate_sparse_op.py
+++ b/paddle/fluid/operators/generator/generate_sparse_op.py
@@ -40,6 +40,7 @@
     is_base_op,
     is_composite_op,
     is_initializer_list,
+    is_only_composite_op,
     is_scalar,
     is_vec,
     supports_inplace,
@@ -71,6 +72,7 @@
 env.filters["get_infer_var_type_func"] = get_infer_var_type_func
 env.tests["base_op"] = is_base_op
 env.tests["composite_op"] = is_composite_op
+env.tests["only_composite_op"] = is_only_composite_op
 env.tests["vec"] = is_vec
 env.tests["scalar"] = is_scalar
 env.tests["initializer_list"] = is_initializer_list
diff --git a/paddle/fluid/operators/generator/parse_utils.py b/paddle/fluid/operators/generator/parse_utils.py
index 8a8f4935e1e7e..d0ee533539d2c 100644
--- a/paddle/fluid/operators/generator/parse_utils.py
+++ b/paddle/fluid/operators/generator/parse_utils.py
@@ -498,8 +498,12 @@ def parse_op_entry(op_entry: Dict[str, Any], name_field="op"):
         "data_transform": data_trans,
     }
 
-    # invokes another op ?
-    is_base_op = "invoke" not in op_entry
+    # op should be is_base_op or is_invoke_op or is_only_composite_op
+    is_base_op = True
+    if "invoke" in op_entry:
+        is_base_op = False
+    if "composite" in op_entry and "kernel" not in op_entry:
+        is_base_op = False
 
     if is_base_op:
         # kernel
@@ -524,10 +528,11 @@ def parse_op_entry(op_entry: Dict[str, Any], name_field="op"):
                 "inplace": inplace_pairs,
             }
         )
-    else:
-        # invoke
-        invoke = parse_invoke(op_name, op_entry["invoke"])
-        op["invoke"] = invoke
+
+    # has invoke ?
+    if "invoke" in op_entry:
+        invoke_dict = parse_invoke(op_name, op_entry["invoke"])
+        op.update({"invoke": invoke_dict})
 
     # has composite ?
     if "composite" in op_entry:
diff --git a/paddle/fluid/operators/generator/templates/op.c.j2 b/paddle/fluid/operators/generator/templates/op.c.j2
index feeb1dee169ff..68673256a00d4 100644
--- a/paddle/fluid/operators/generator/templates/op.c.j2
+++ b/paddle/fluid/operators/generator/templates/op.c.j2
@@ -38,6 +38,8 @@ using paddle::framework::GradVarName;
 {{backward_op_maker(op, op_dict[op["forward"]["name"]])}}
 
 {{operator(op)}}
+  {% elif op is only_composite_op %}
+
   {% else %}
 {{backward_op_reused_maker(op, op_dict[op["forward"]["name"]], op["invoke"])}}
   {% endif %}
diff --git a/paddle/fluid/operators/generator/templates/operator_utils.c.j2 b/paddle/fluid/operators/generator/templates/operator_utils.c.j2
index 0b0190f4e4e55..16f0ecaa642f3 100644
--- a/paddle/fluid/operators/generator/templates/operator_utils.c.j2
+++ b/paddle/fluid/operators/generator/templates/operator_utils.c.j2
@@ -472,7 +472,8 @@ REGISTER_OPERATOR({{name}}, ops::{{name | to_pascal_case}}Op,
 {% if not "forward" in op %}{# it is a forward op #}
                   ops::{{name | to_pascal_case}}OpMaker,
 {% endif %}
-{% if "backward" in op and op["backward"] is not none %}{# backward #}
+{% if "only_backward_composite" in op and op["only_backward_composite"] is true %}{# backward #}
+{% elif "backward" in op and op["backward"] is not none %}
   {% set backward_name = op["backward"] %}
                   ops::{{backward_name | to_pascal_case}}OpMaker<paddle::framework::OpDesc>,
                   ops::{{backward_name | to_pascal_case}}OpMaker<paddle::imperative::OpBase>,
diff --git a/paddle/fluid/operators/generator/tests_utils.py b/paddle/fluid/operators/generator/tests_utils.py
index 574f3663b7d56..9cea1698f4f6a 100644
--- a/paddle/fluid/operators/generator/tests_utils.py
+++ b/paddle/fluid/operators/generator/tests_utils.py
@@ -54,6 +54,10 @@ def is_base_op(op):
     return "kernel" in op and "infer_meta" in op
 
 
+def is_only_composite_op(op):
+    return "composite" in op and "kernel" not in op and "invoke" not in op
+
+
 def is_composite_op(op):
     return "composite" in op
 
diff --git a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
index 6eb366dc355ed..c5d56dc82b5c4 100644
--- a/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
+++ b/paddle/fluid/prim/api/composite_backward/composite_backward_api.h
@@ -21,6 +21,7 @@
 #include <math.h>
 
 #include "paddle/fluid/prim/api/all.h"
+#include "paddle/fluid/prim/api/composite_backward/composite_double_backward_api.h"
 #include "paddle/fluid/prim/api/generated_prim/prim_generated_api.h"
 #include "paddle/phi/common/amp_type_traits.h"
 #include "paddle/phi/common/int_array.h"
@@ -164,26 +165,6 @@ void tanh_grad(const Tensor& out, const Tensor& grad_out, Tensor* grad_x) {
   set_output<T>(grad_x_tmp, grad_x);
 }
 
-template <typename T>
-void tanh_double_grad(const Tensor& out,
-                      const Tensor& grad_out,
-                      const Tensor& grad_x_grad,
-                      Tensor* out_grad,
-                      Tensor* grad_out_grad) {
-  // tanh grad grad : ddout = (1 - out^2) * ddx, dout = - (dout_old * 2 * out *
-  // ddx)
-  auto out_m_grad_x_grad = out * grad_x_grad;
-  if (out_grad) {
-    auto out_grad_tmp = -2 * grad_out * out_m_grad_x_grad;
-    set_output<T>(out_grad_tmp, out_grad);
-  }
-
-  if (grad_out_grad) {
-    auto grad_out_grad_tmp = grad_x_grad - out * out_m_grad_x_grad;
-    set_output<T>(grad_out_grad_tmp, grad_out_grad);
-  }
-}
-
 template <typename T>
 void reshape_grad(const Tensor& x, const Tensor& grad_out, Tensor* grad_x) {
   if (grad_x) {
@@ -698,315 +679,6 @@ void abs_grad(const Tensor& x, const Tensor& out_grad, Tensor* x_grad) {
   }
 }
 
-template <typename T>
-void matmul_double_grad(const Tensor& x,
-                        const Tensor& y,
-                        const Tensor& grad_out,
-                        const paddle::optional<Tensor>& grad_x_grad,
-                        const paddle::optional<Tensor>& grad_y_grad,
-                        bool transpose_x,
-                        bool transpose_y,
-                        Tensor* x_grad,
-                        Tensor* y_grad,
-                        Tensor* grad_out_grad) {
-  // Get dims from the input x, y, output_grad
-  std::vector<std::int64_t> x_dims = vectorize(x.dims());
-  std::vector<std::int64_t> y_dims = vectorize(y.dims());
-  std::vector<std::int64_t> grad_out_dims = vectorize(grad_out.dims());
-
-  int x_ndim = x_dims.size();
-  int y_ndim = y_dims.size();
-  int dout_ndim = grad_out_dims.size();
-
-  // prepare dims for x_ndim <= 1 || y_ndim <= 1
-  Tensor x_help, y_help, xg_help, yg_help, out_help;
-
-  if (x_ndim == 1 && y_ndim == 1) {
-    transpose_x = false;
-    transpose_y = false;
-    x_help = reshape<T>(x, IntArray(std::vector<int64_t>({1, x_dims[0]})));
-    y_help = reshape<T>(y, IntArray(std::vector<int64_t>({y_dims[0], 1})));
-    if (grad_x_grad) {
-      xg_help = reshape<T>(grad_x_grad.get(),
-                           IntArray(std::vector<int64_t>({1, x_dims[0]})));
-    }
-    if (grad_y_grad) {
-      yg_help = reshape<T>(grad_y_grad.get(),
-                           IntArray(std::vector<int64_t>({y_dims[0], 1})));
-    }
-    out_help = reshape<T>(grad_out, IntArray(std::vector<int64_t>({1, 1})));
-
-  } else if (x_ndim == 1) {
-    transpose_x = false;
-    x_help = reshape<T>(x, IntArray(std::vector<int64_t>({1, x_dims[0]})));
-    y_help = y;
-    if (grad_x_grad) {
-      xg_help = reshape<T>(grad_x_grad.get(),
-                           IntArray(std::vector<int64_t>({1, x_dims[0]})));
-    }
-    if (grad_y_grad) {
-      yg_help = grad_y_grad.get();
-    }
-    auto tmp_grad_out_dims = grad_out_dims;
-    tmp_grad_out_dims.insert(tmp_grad_out_dims.begin(), 1);
-    out_help = reshape<T>(grad_out, IntArray(tmp_grad_out_dims));
-
-  } else if (y_ndim == 1) {
-    transpose_y = false;
-    x_help = x;
-    y_help = reshape<T>(y, IntArray(std::vector<int64_t>({y_dims[0], 1})));
-    if (grad_x_grad) {
-      xg_help = grad_x_grad.get();
-    }
-    if (grad_y_grad) {
-      yg_help = reshape<T>(grad_y_grad.get(),
-                           IntArray(std::vector<int64_t>({y_dims[0], 1})));
-    }
-    auto tmp_grad_out_dims = grad_out_dims;
-    tmp_grad_out_dims.push_back(1);
-    out_help = reshape<T>(grad_out, IntArray(tmp_grad_out_dims));
-
-  } else {
-    x_help = x;
-    y_help = y;
-    if (grad_x_grad) {
-      xg_help = grad_x_grad.get();
-    }
-    if (grad_y_grad) {
-      yg_help = grad_y_grad.get();
-    }
-    out_help = grad_out;
-  }
-
-  bool is_broadcast = true;
-  if (x_ndim <= 2 && y_ndim <= 2) {
-    is_broadcast = false;
-  } else if (x_ndim != y_ndim) {
-    is_broadcast = true;
-  } else {
-    is_broadcast = !std::equal(
-        x_dims.cbegin(), x_dims.cbegin() + x_ndim - 2, y_dims.cbegin());
-  }
-  Tensor dx, dy, ddout_1, ddout_2, ddout;
-  if (!grad_x_grad && !grad_y_grad) {
-    x_grad = nullptr;
-    y_grad = nullptr;
-    grad_out_grad = nullptr;
-    return;
-
-  } else if (!grad_x_grad) {
-    y_grad = nullptr;
-    if (!transpose_x && !transpose_y) {
-      if (x_grad) {
-        dx = matmul<T>(out_help, yg_help, false, true);
-      }
-      if (grad_out_grad) {
-        ddout = matmul<T>(x_help, yg_help, false, false);
-      }
-    } else if (!transpose_x && transpose_y) {
-      if (x_grad) {
-        dx = matmul<T>(out_help, yg_help, false, false);
-      }
-      if (grad_out_grad) {
-        ddout = matmul<T>(x_help, yg_help, false, true);
-      }
-    } else if (transpose_x && !transpose_y) {
-      if (x_grad) {
-        dx = matmul<T>(yg_help, out_help, false, true);
-      }
-      if (grad_out_grad) {
-        ddout = matmul<T>(x_help, yg_help, true, false);
-      }
-    } else {
-      if (x_grad) {
-        dx = matmul<T>(yg_help, out_help, true, true);
-      }
-      if (grad_out_grad) {
-        ddout = matmul<T>(x_help, yg_help, true, true);
-      }
-    }
-
-  } else if (!grad_y_grad) {
-    x_grad = nullptr;
-    if (!transpose_x && !transpose_y) {
-      if (y_grad) {
-        dy = matmul<T>(xg_help, out_help, true, false);
-      }
-      if (grad_out_grad) {
-        ddout = matmul<T>(xg_help, y_help, false, false);
-      }
-    } else if (!transpose_x && transpose_y) {
-      if (y_grad) {
-        dy = matmul<T>(out_help, xg_help, true, false);
-      }
-      if (grad_out_grad) {
-        ddout = matmul<T>(xg_help, y_help, false, true);
-      }
-    } else if (transpose_x && !transpose_y) {
-      if (y_grad) {
-        dy = matmul<T>(xg_help, out_help, false, false);
-      }
-      if (grad_out_grad) {
-        ddout = matmul<T>(xg_help, y_help, true, false);
-      }
-    } else {
-      if (y_grad) {
-        dy = matmul<T>(out_help, xg_help, true, true);
-      }
-      if (grad_out_grad) {
-        ddout = matmul<T>(xg_help, y_help, true, true);
-      }
-    }
-
-  } else {
-    if (!transpose_x && !transpose_y) {
-      if (x_grad) {
-        dx = matmul<T>(out_help, yg_help, false, true);
-      }
-      if (y_grad) {
-        dy = matmul<T>(xg_help, out_help, true, false);
-      }
-      if (grad_out_grad) {
-        ddout_1 = matmul<T>(x_help, yg_help, false, false);
-        ddout_2 = matmul<T>(xg_help, y_help, false, false);
-        ddout = add<T>(ddout_1, ddout_2);
-      }
-    } else if (!transpose_x && transpose_y) {
-      if (x_grad) {
-        dx = matmul<T>(out_help, yg_help, false, false);
-      }
-
-      if (y_grad) {
-        dy = matmul<T>(out_help, xg_help, true, false);
-      }
-      if (grad_out_grad) {
-        ddout_1 = matmul<T>(x_help, yg_help, false, true);
-        ddout_2 = matmul<T>(xg_help, y_help, false, true);
-        ddout = add<T>(ddout_1, ddout_2);
-      }
-    } else if (transpose_x && !transpose_y) {
-      if (x_grad) {
-        dx = matmul<T>(yg_help, out_help, false, true);
-      }
-
-      if (y_grad) {
-        dy = matmul<T>(xg_help, out_help, false, false);
-      }
-      if (grad_out_grad) {
-        ddout_1 = matmul<T>(x_help, yg_help, true, false);
-        ddout_2 = matmul<T>(xg_help, y_help, true, false);
-        ddout = add<T>(ddout_1, ddout_2);
-      }
-    } else {
-      if (x_grad) {
-        dx = matmul<T>(yg_help, out_help, true, true);
-      }
-      if (y_grad) {
-        dy = matmul<T>(out_help, xg_help, true, true);
-      }
-      if (grad_out_grad) {
-        ddout_1 = matmul<T>(x_help, yg_help, true, true);
-        ddout_2 = matmul<T>(xg_help, y_help, true, true);
-        ddout = add<T>(ddout_1, ddout_2);
-      }
-    }
-  }
-
-  if (is_broadcast) {
-    // Case3: broadcast. It need cost much time to reduce sum for the
-    // broadcast and wastes the memory.
-    // So we should avoid the case in reality.
-    VLOG(3) << "It need cost much time to reduce sum for the broadcast and "
-               "wastes the memory. So we should avoid the case in reality";
-    // Reduce sum to get grad by ReduceSum
-    if (x_grad) {
-      auto tx_dims = x_dims;
-      auto tx_ndim = x_ndim;
-      auto tdout_ndim = dout_ndim;
-      if (x_ndim == 1) {
-        tx_dims = std::vector<int64_t>({1, x_dims[0]});
-        tx_ndim = x_ndim + 1;
-        tdout_ndim = dout_ndim + 1;
-      }
-
-      auto x_grad_reduce_dims =
-          get_reduce_dims(dx, tdout_ndim, tx_ndim, &tx_dims);
-
-      if (!x_grad_reduce_dims.empty()) {
-        dx = sum<T>(dx, IntArray(x_grad_reduce_dims), dy.dtype(), true);
-      }
-      reshape<T>(dx, IntArray(tx_dims));
-    }
-
-    if (y_grad) {
-      auto ty_dims = y_dims;
-      auto ty_ndim = y_ndim;
-      auto tdout_ndim = dout_ndim;
-      if (y_ndim == 1) {
-        ty_dims = std::vector<int64_t>({y_dims[0], 1});
-        ty_ndim = y_ndim + 1;
-        tdout_ndim = dout_ndim + 1;
-      }
-
-      auto y_grad_reduce_dims =
-          get_reduce_dims(dy, tdout_ndim, ty_ndim, &ty_dims);
-
-      if (!y_grad_reduce_dims.empty()) {
-        dy = sum<T>(dy, IntArray(y_grad_reduce_dims), dy.dtype(), true);
-      }
-      reshape<T>(dy, IntArray(ty_dims));
-    }
-  }
-
-  // recover the original dim of output (delete 1)
-  std::vector<int64_t> dx_dims =
-      dx.initialized() ? vectorize(dx.dims()) : std::vector<int64_t>({});
-  std::vector<int64_t> dy_dims =
-      dy.initialized() ? vectorize(dy.dims()) : std::vector<int64_t>({});
-  std::vector<int64_t> ddout_dims =
-      ddout.initialized() ? vectorize(ddout.dims()) : std::vector<int64_t>({});
-  if (x_ndim == 1 && y_ndim == 1) {
-    if (dx.initialized() && dx_dims[0] == 1) {
-      dx = reshape<T>(dx, IntArray(x_dims));
-    }
-    if (dy.initialized() && dy_dims.back() == 1) {
-      dy = reshape<T>(dy, IntArray(y_dims));
-    }
-    if (ddout.initialized() && ddout_dims == std::vector<int64_t>({1, 1})) {
-      ddout = reshape<T>(ddout, IntArray(std::vector<int64_t>({1})));
-    }
-  } else if (x_ndim == 1) {
-    if (dx.initialized() && dx_dims[0] == 1) {
-      dx = reshape<T>(dx, IntArray(x_dims));
-    }
-    if (ddout.initialized() && ddout_dims[0] == 1) {
-      ddout = reshape<T>(ddout,
-                         IntArray(std::vector<int64_t>(
-                             {ddout_dims.cbegin() + 1, ddout_dims.cend()})));
-    }
-  } else if (y_ndim == 1) {
-    if (dy.initialized() && dy_dims.back() == 1) {
-      dy = reshape<T>(dy, IntArray(y_dims));
-    }
-    if (ddout.initialized() && ddout_dims.back() == 1) {
-      ddout = reshape<T>(ddout,
-                         IntArray(std::vector<int64_t>(
-                             {ddout_dims.cbegin(),
-                              ddout_dims.cbegin() + ddout_dims.size() - 1})));
-    }
-  }
-
-  if (x_grad) {
-    set_output<T>(dx, x_grad);
-  }
-  if (y_grad) {
-    set_output<T>(dy, y_grad);
-  }
-  if (grad_out_grad) {
-    set_output<T>(ddout, grad_out_grad);
-  }
-}
-
 template <typename T>
 void slice_grad(const Tensor& input,
                 const Tensor& out_grad,
diff --git a/paddle/fluid/prim/api/composite_backward/composite_double_backward_api.h b/paddle/fluid/prim/api/composite_backward/composite_double_backward_api.h
new file mode 100644
index 0000000000000..b97ffeb4b2675
--- /dev/null
+++ b/paddle/fluid/prim/api/composite_backward/composite_double_backward_api.h
@@ -0,0 +1,387 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#ifndef _USE_MATH_DEFINES
+#define _USE_MATH_DEFINES
+#endif
+
+#include <math.h>
+
+#include "paddle/fluid/prim/api/all.h"
+#include "paddle/fluid/prim/api/generated_prim/prim_generated_api.h"
+#include "paddle/phi/common/amp_type_traits.h"
+#include "paddle/phi/common/int_array.h"
+#include "paddle/phi/core/ddim.h"
+
+namespace paddle {
+namespace prim {
+using Tensor = paddle::Tensor;
+using IntArray = paddle::experimental::IntArrayBase<paddle::Tensor>;
+//  This file define high level grad composite api for Higher order
+//  differentiation
+
+template <typename T>
+void tanh_double_grad(const Tensor& out,
+                      const Tensor& grad_out,
+                      const Tensor& grad_x_grad,
+                      Tensor* out_grad,
+                      Tensor* grad_out_grad) {
+  // tanh grad grad : ddout = (1 - out^2) * ddx, dout = - (dout_old * 2 * out *
+  // ddx)
+  auto out_m_grad_x_grad = out * grad_x_grad;
+  if (out_grad) {
+    auto out_grad_tmp = -2 * grad_out * out_m_grad_x_grad;
+    set_output<T>(out_grad_tmp, out_grad);
+  }
+
+  if (grad_out_grad) {
+    auto grad_out_grad_tmp = grad_x_grad - out * out_m_grad_x_grad;
+    set_output<T>(grad_out_grad_tmp, grad_out_grad);
+  }
+}
+
+template <typename T>
+void matmul_double_grad(const Tensor& x,
+                        const Tensor& y,
+                        const Tensor& grad_out,
+                        const paddle::optional<Tensor>& grad_x_grad,
+                        const paddle::optional<Tensor>& grad_y_grad,
+                        bool transpose_x,
+                        bool transpose_y,
+                        Tensor* x_grad,
+                        Tensor* y_grad,
+                        Tensor* grad_out_grad) {
+  // Get dims from the input x, y, output_grad
+  std::vector<std::int64_t> x_dims = vectorize(x.dims());
+  std::vector<std::int64_t> y_dims = vectorize(y.dims());
+  std::vector<std::int64_t> grad_out_dims = vectorize(grad_out.dims());
+
+  int x_ndim = x_dims.size();
+  int y_ndim = y_dims.size();
+  int dout_ndim = grad_out_dims.size();
+
+  // prepare dims for x_ndim <= 1 || y_ndim <= 1
+  Tensor x_help, y_help, xg_help, yg_help, out_help;
+
+  if (x_ndim == 1 && y_ndim == 1) {
+    transpose_x = false;
+    transpose_y = false;
+    x_help = reshape<T>(x, IntArray(std::vector<int64_t>({1, x_dims[0]})));
+    y_help = reshape<T>(y, IntArray(std::vector<int64_t>({y_dims[0], 1})));
+    if (grad_x_grad) {
+      xg_help = reshape<T>(grad_x_grad.get(),
+                           IntArray(std::vector<int64_t>({1, x_dims[0]})));
+    }
+    if (grad_y_grad) {
+      yg_help = reshape<T>(grad_y_grad.get(),
+                           IntArray(std::vector<int64_t>({y_dims[0], 1})));
+    }
+    out_help = reshape<T>(grad_out, IntArray(std::vector<int64_t>({1, 1})));
+
+  } else if (x_ndim == 1) {
+    transpose_x = false;
+    x_help = reshape<T>(x, IntArray(std::vector<int64_t>({1, x_dims[0]})));
+    y_help = y;
+    if (grad_x_grad) {
+      xg_help = reshape<T>(grad_x_grad.get(),
+                           IntArray(std::vector<int64_t>({1, x_dims[0]})));
+    }
+    if (grad_y_grad) {
+      yg_help = grad_y_grad.get();
+    }
+    auto tmp_grad_out_dims = grad_out_dims;
+    tmp_grad_out_dims.insert(tmp_grad_out_dims.begin(), 1);
+    out_help = reshape<T>(grad_out, IntArray(tmp_grad_out_dims));
+
+  } else if (y_ndim == 1) {
+    transpose_y = false;
+    x_help = x;
+    y_help = reshape<T>(y, IntArray(std::vector<int64_t>({y_dims[0], 1})));
+    if (grad_x_grad) {
+      xg_help = grad_x_grad.get();
+    }
+    if (grad_y_grad) {
+      yg_help = reshape<T>(grad_y_grad.get(),
+                           IntArray(std::vector<int64_t>({y_dims[0], 1})));
+    }
+    auto tmp_grad_out_dims = grad_out_dims;
+    tmp_grad_out_dims.push_back(1);
+    out_help = reshape<T>(grad_out, IntArray(tmp_grad_out_dims));
+
+  } else {
+    x_help = x;
+    y_help = y;
+    if (grad_x_grad) {
+      xg_help = grad_x_grad.get();
+    }
+    if (grad_y_grad) {
+      yg_help = grad_y_grad.get();
+    }
+    out_help = grad_out;
+  }
+
+  bool is_broadcast = true;
+  if (x_ndim <= 2 && y_ndim <= 2) {
+    is_broadcast = false;
+  } else if (x_ndim != y_ndim) {
+    is_broadcast = true;
+  } else {
+    is_broadcast = !std::equal(
+        x_dims.cbegin(), x_dims.cbegin() + x_ndim - 2, y_dims.cbegin());
+  }
+  Tensor dx, dy, ddout_1, ddout_2, ddout;
+  if (!grad_x_grad && !grad_y_grad) {
+    x_grad = nullptr;
+    y_grad = nullptr;
+    grad_out_grad = nullptr;
+    return;
+
+  } else if (!grad_x_grad) {
+    y_grad = nullptr;
+    if (!transpose_x && !transpose_y) {
+      if (x_grad) {
+        dx = matmul<T>(out_help, yg_help, false, true);
+      }
+      if (grad_out_grad) {
+        ddout = matmul<T>(x_help, yg_help, false, false);
+      }
+    } else if (!transpose_x && transpose_y) {
+      if (x_grad) {
+        dx = matmul<T>(out_help, yg_help, false, false);
+      }
+      if (grad_out_grad) {
+        ddout = matmul<T>(x_help, yg_help, false, true);
+      }
+    } else if (transpose_x && !transpose_y) {
+      if (x_grad) {
+        dx = matmul<T>(yg_help, out_help, false, true);
+      }
+      if (grad_out_grad) {
+        ddout = matmul<T>(x_help, yg_help, true, false);
+      }
+    } else {
+      if (x_grad) {
+        dx = matmul<T>(yg_help, out_help, true, true);
+      }
+      if (grad_out_grad) {
+        ddout = matmul<T>(x_help, yg_help, true, true);
+      }
+    }
+
+  } else if (!grad_y_grad) {
+    x_grad = nullptr;
+    if (!transpose_x && !transpose_y) {
+      if (y_grad) {
+        dy = matmul<T>(xg_help, out_help, true, false);
+      }
+      if (grad_out_grad) {
+        ddout = matmul<T>(xg_help, y_help, false, false);
+      }
+    } else if (!transpose_x && transpose_y) {
+      if (y_grad) {
+        dy = matmul<T>(out_help, xg_help, true, false);
+      }
+      if (grad_out_grad) {
+        ddout = matmul<T>(xg_help, y_help, false, true);
+      }
+    } else if (transpose_x && !transpose_y) {
+      if (y_grad) {
+        dy = matmul<T>(xg_help, out_help, false, false);
+      }
+      if (grad_out_grad) {
+        ddout = matmul<T>(xg_help, y_help, true, false);
+      }
+    } else {
+      if (y_grad) {
+        dy = matmul<T>(out_help, xg_help, true, true);
+      }
+      if (grad_out_grad) {
+        ddout = matmul<T>(xg_help, y_help, true, true);
+      }
+    }
+
+  } else {
+    if (!transpose_x && !transpose_y) {
+      if (x_grad) {
+        dx = matmul<T>(out_help, yg_help, false, true);
+      }
+      if (y_grad) {
+        dy = matmul<T>(xg_help, out_help, true, false);
+      }
+      if (grad_out_grad) {
+        ddout_1 = matmul<T>(x_help, yg_help, false, false);
+        ddout_2 = matmul<T>(xg_help, y_help, false, false);
+        ddout = add<T>(ddout_1, ddout_2);
+      }
+    } else if (!transpose_x && transpose_y) {
+      if (x_grad) {
+        dx = matmul<T>(out_help, yg_help, false, false);
+      }
+
+      if (y_grad) {
+        dy = matmul<T>(out_help, xg_help, true, false);
+      }
+      if (grad_out_grad) {
+        ddout_1 = matmul<T>(x_help, yg_help, false, true);
+        ddout_2 = matmul<T>(xg_help, y_help, false, true);
+        ddout = add<T>(ddout_1, ddout_2);
+      }
+    } else if (transpose_x && !transpose_y) {
+      if (x_grad) {
+        dx = matmul<T>(yg_help, out_help, false, true);
+      }
+
+      if (y_grad) {
+        dy = matmul<T>(xg_help, out_help, false, false);
+      }
+      if (grad_out_grad) {
+        ddout_1 = matmul<T>(x_help, yg_help, true, false);
+        ddout_2 = matmul<T>(xg_help, y_help, true, false);
+        ddout = add<T>(ddout_1, ddout_2);
+      }
+    } else {
+      if (x_grad) {
+        dx = matmul<T>(yg_help, out_help, true, true);
+      }
+      if (y_grad) {
+        dy = matmul<T>(out_help, xg_help, true, true);
+      }
+      if (grad_out_grad) {
+        ddout_1 = matmul<T>(x_help, yg_help, true, true);
+        ddout_2 = matmul<T>(xg_help, y_help, true, true);
+        ddout = add<T>(ddout_1, ddout_2);
+      }
+    }
+  }
+
+  if (is_broadcast) {
+    // Case3: broadcast. It need cost much time to reduce sum for the
+    // broadcast and wastes the memory.
+    // So we should avoid the case in reality.
+    VLOG(3) << "It need cost much time to reduce sum for the broadcast and "
+               "wastes the memory. So we should avoid the case in reality";
+    // Reduce sum to get grad by ReduceSum
+    if (x_grad) {
+      auto tx_dims = x_dims;
+      auto tx_ndim = x_ndim;
+      auto tdout_ndim = dout_ndim;
+      if (x_ndim == 1) {
+        tx_dims = std::vector<int64_t>({1, x_dims[0]});
+        tx_ndim = x_ndim + 1;
+        tdout_ndim = dout_ndim + 1;
+      }
+
+      auto x_grad_reduce_dims =
+          get_reduce_dims(dx, tdout_ndim, tx_ndim, &tx_dims);
+
+      if (!x_grad_reduce_dims.empty()) {
+        dx = sum<T>(dx, IntArray(x_grad_reduce_dims), dy.dtype(), true);
+      }
+      reshape<T>(dx, IntArray(tx_dims));
+    }
+
+    if (y_grad) {
+      auto ty_dims = y_dims;
+      auto ty_ndim = y_ndim;
+      auto tdout_ndim = dout_ndim;
+      if (y_ndim == 1) {
+        ty_dims = std::vector<int64_t>({y_dims[0], 1});
+        ty_ndim = y_ndim + 1;
+        tdout_ndim = dout_ndim + 1;
+      }
+
+      auto y_grad_reduce_dims =
+          get_reduce_dims(dy, tdout_ndim, ty_ndim, &ty_dims);
+
+      if (!y_grad_reduce_dims.empty()) {
+        dy = sum<T>(dy, IntArray(y_grad_reduce_dims), dy.dtype(), true);
+      }
+      reshape<T>(dy, IntArray(ty_dims));
+    }
+  }
+
+  // recover the original dim of output (delete 1)
+  std::vector<int64_t> dx_dims =
+      dx.initialized() ? vectorize(dx.dims()) : std::vector<int64_t>({});
+  std::vector<int64_t> dy_dims =
+      dy.initialized() ? vectorize(dy.dims()) : std::vector<int64_t>({});
+  std::vector<int64_t> ddout_dims =
+      ddout.initialized() ? vectorize(ddout.dims()) : std::vector<int64_t>({});
+  if (x_ndim == 1 && y_ndim == 1) {
+    if (dx.initialized() && dx_dims[0] == 1) {
+      dx = reshape<T>(dx, IntArray(x_dims));
+    }
+    if (dy.initialized() && dy_dims.back() == 1) {
+      dy = reshape<T>(dy, IntArray(y_dims));
+    }
+    if (ddout.initialized() && ddout_dims == std::vector<int64_t>({1, 1})) {
+      ddout = reshape<T>(ddout, IntArray(std::vector<int64_t>({1})));
+    }
+  } else if (x_ndim == 1) {
+    if (dx.initialized() && dx_dims[0] == 1) {
+      dx = reshape<T>(dx, IntArray(x_dims));
+    }
+    if (ddout.initialized() && ddout_dims[0] == 1) {
+      ddout = reshape<T>(ddout,
+                         IntArray(std::vector<int64_t>(
+                             {ddout_dims.cbegin() + 1, ddout_dims.cend()})));
+    }
+  } else if (y_ndim == 1) {
+    if (dy.initialized() && dy_dims.back() == 1) {
+      dy = reshape<T>(dy, IntArray(y_dims));
+    }
+    if (ddout.initialized() && ddout_dims.back() == 1) {
+      ddout = reshape<T>(ddout,
+                         IntArray(std::vector<int64_t>(
+                             {ddout_dims.cbegin(),
+                              ddout_dims.cbegin() + ddout_dims.size() - 1})));
+    }
+  }
+
+  if (x_grad) {
+    set_output<T>(dx, x_grad);
+  }
+  if (y_grad) {
+    set_output<T>(dy, y_grad);
+  }
+  if (grad_out_grad) {
+    set_output<T>(ddout, grad_out_grad);
+  }
+}
+
+template <typename T>
+void silu_double_grad(const Tensor& x,
+                      const Tensor& out,
+                      const Tensor& out_grad,
+                      const Tensor& grad_x_grad,
+                      Tensor* grad_x,
+                      Tensor* grad_out_grad) {
+  auto sigmoid = out / x;
+  auto tmp1 = 1 - sigmoid;
+  auto tmp2 = 1 + tmp1 * x;
+  if (grad_out_grad) {
+    auto ddout = grad_x_grad * sigmoid * tmp2;
+    set_output<T>(ddout, grad_out_grad);
+  }
+  if (grad_x) {
+    auto dx =
+        sigmoid * grad_x_grad * out_grad * (1 + (tmp2 - x * sigmoid)) * tmp1;
+    set_output<T>(dx, grad_x);
+  }
+}
+
+}  // namespace prim
+}  // namespace paddle
diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index e3ef0460884b1..92ce63e470976 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -1626,6 +1626,7 @@
     param : [x]
   kernel :
     func : silu_grad
+  backward : silu_double_grad
   composite : silu_grad(x, out, out_grad, x_grad)
   inplace : (out_grad -> x_grad)
 
@@ -2106,6 +2107,12 @@
     func : yolo_loss_grad
   optional : gt_score
 
+- backward_op: silu_double_grad
+  forward: silu_grad (Tensor x, Tensor out, Tensor grad_out) -> Tensor(grad_x)
+  args: (Tensor x, Tensor out, Tensor grad_out, Tensor grad_x_grad)
+  output: Tensor(x_grad), Tensor(grad_out_grad)
+  composite: silu_double_grad(x, out, grad_out, grad_x_grad, x_grad, grad_out_grad)
+
 - backward_op: unpool3d_grad
   forward: unpool3d (Tensor x, Tensor indices, int[] ksize, int[] strides={1,1,1}, int[] paddings={0,0,0}, int[] output_size={0,0,0}, str data_format="NCDHW") -> Tensor(out)
   args: (Tensor x, Tensor indices, Tensor out, Tensor out_grad, int[] ksize, int[] strides, int[] paddings, int[] output_size, str data_format)
diff --git a/paddle/phi/api/yaml/generator/api_base.py b/paddle/phi/api/yaml/generator/api_base.py
index 4d41bb522e002..e4ac5726b9c62 100644
--- a/paddle/phi/api/yaml/generator/api_base.py
+++ b/paddle/phi/api/yaml/generator/api_base.py
@@ -41,6 +41,7 @@ def __init__(self, api_item_yaml):
         ) = self.parse_args(self.api, api_item_yaml)
 
         self.is_base_api = True
+        self.is_only_composite_api = False
         if 'invoke' in api_item_yaml:
             self.is_base_api = False
             self.invoke = api_item_yaml['invoke']
@@ -49,7 +50,12 @@ def __init__(self, api_item_yaml):
                 self.infer_meta = self.parse_infer_meta(
                     api_item_yaml['infer_meta']
                 )
-            self.kernel = self.parse_kernel(api_item_yaml['kernel'])
+            if 'composite' in api_item_yaml and 'kernel' not in api_item_yaml:
+                self.is_base_api = False
+                self.is_only_composite_api = True
+                self.kernel = None
+            else:
+                self.kernel = self.parse_kernel(api_item_yaml['kernel'])
             self.data_transform = self.parse_data_transform(api_item_yaml)
             self.inplace_map, self.view_map = {}, {}
 
@@ -1319,23 +1325,10 @@ def gene_api_code(self):
                     api_code = ""
                 api_code = api_code + self.gene_base_api_code(inplace_flag=True)
             return api_code
-
+        elif self.is_only_composite_api:
+            # for composite and invoke api, dygraph use prim::xxx_grad method
+            return ''
         else:
-            invoke_func_name = self.invoke.split('(')[0].strip()
-            if invoke_func_name in self.attrs['names']:
-                # Adjust the param whose name is same with api invoked.
-                pattern = r'\W' + invoke_func_name + '[^A-Za-z0-9_(]'
-
-                def adjust_name(matched):
-                    matched_str = matched.group()
-                    return matched_str[0:-1] + '_val' + matched_str[-1]
-
-                invoke_code = re.sub(pattern, adjust_name, self.invoke)
-                params_code = re.sub(
-                    pattern, adjust_name, self.get_define_args()
-                )
-            else:
-                invoke_code = self.invoke
-                params_code = self.get_define_args()
-
+            invoke_code = self.invoke
+            params_code = self.get_define_args()
             return self.gene_invoke_code(invoke_code, params_code)
diff --git a/paddle/phi/api/yaml/generator/backward_api_gen.py b/paddle/phi/api/yaml/generator/backward_api_gen.py
index 36ac38a88dd8d..432f70f8dbaa0 100644
--- a/paddle/phi/api/yaml/generator/backward_api_gen.py
+++ b/paddle/phi/api/yaml/generator/backward_api_gen.py
@@ -112,7 +112,7 @@ def gene_return_code(self):
         return ""
 
     def gene_api_declaration(self):
-        if not self.is_base_api:
+        if not self.is_base_api and not self.is_only_composite_api:
             invoke_func_name = self.invoke.split('(')[0]
             if (not invoke_func_name.endswith("_grad")) and (
                 not invoke_func_name.endswith('_impl')
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index 74bdb3c53dde1..5ad73f4e1045f 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -2101,7 +2101,7 @@
     out : Out
 
 - op : silu
-  backward : silu_grad
+  backward : silu_grad, silu_double_grad
   inputs :
     x : X
   outputs :
diff --git a/test/prim/prim/vjp/test_comp_high_grad.py b/test/prim/prim/vjp/test_comp_high_grad.py
index 99268b1b58ed9..489ae6e6f1f89 100644
--- a/test/prim/prim/vjp/test_comp_high_grad.py
+++ b/test/prim/prim/vjp/test_comp_high_grad.py
@@ -331,5 +331,88 @@ def test_high_grad(self):
             self.func_double(p)
             self.func_triple(p)
 '''
+
+
+@param.parameterized_class(
+    ('shape1'),
+    [
+        ([2],),
+        ([2, 3],),
+        ([2, 3, 4],),
+        ([2, 3, 3, 4],),
+    ],
+)
+class TestSiluHighGradCheck(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.shape1 = cls.shape1
+
+    def silu_wrapper(self, x):
+        return paddle.nn.functional.silu(x[0])
+
+    @prog_scope()
+    def func_double(self, place):
+        shape1 = self.shape1
+        eps = 0.0005
+        dtype = np.float64
+        x = paddle.static.data('x', shape1, dtype=dtype)
+        x.stop_gradient = False
+        x.persistable = True
+        out = paddle.nn.functional.silu(x)
+        x_arr = np.random.uniform(-1, 1, shape1).astype(dtype)
+        x_arr[np.abs(x_arr) < 0.005] = 0.002
+
+        # silu double grad only has CompositeOpMaker,don't need set prim_flag
+        from paddle.fluid import core
+
+        core._set_prim_backward_enabled(True)
+        gradient_checker.double_grad_check(
+            [x], y=out, x_init=[x_arr], place=place, eps=eps
+        )
+        gradient_checker.double_grad_check_for_dygraph(
+            self.silu_wrapper,
+            [x],
+            y=out,
+            x_init=[x_arr],
+            place=place,
+        )
+        core._set_prim_backward_enabled(False)
+
+    @prog_scope()
+    def func_triple(self, place):
+        shape1 = self.shape1
+        eps = 0.0005
+        dtype = np.float64
+        x = paddle.static.data('x', shape1, dtype=dtype)
+        x.stop_gradient = False
+        x.persistable = True
+        out = paddle.nn.functional.silu(x)
+        x_arr = np.random.uniform(-1, 1, shape1).astype(dtype)
+        x_arr[np.abs(x_arr) < 0.005] = 0.002
+        from paddle.fluid import core
+
+        core._set_prim_backward_enabled(True)
+        gradient_checker.triple_grad_check(
+            [x], y=out, x_init=[x_arr], place=place, eps=eps
+        )
+        gradient_checker.triple_grad_check_for_dygraph(
+            self.silu_wrapper,
+            [x],
+            y=out,
+            x_init=[x_arr],
+            place=place,
+        )
+        core._set_prim_backward_enabled(False)
+
+    def test_high_grad(self):
+        paddle.enable_static()
+        places = [fluid.CPUPlace()]
+        if core.is_compiled_with_cuda():
+            places.append(fluid.CUDAPlace(0))
+        for p in places:
+            self.func_double(p)
+            self.func_triple(p)
+
+
 if __name__ == '__main__':
     unittest.main()

From cc9aedaf28c22d4b4ff7da41bd963c0aef026282 Mon Sep 17 00:00:00 2001
From: bukejiyu <52310069+bukejiyu@users.noreply.github.com>
Date: Mon, 15 May 2023 19:19:28 +0800
Subject: [PATCH 334/405] [inference Zero-Dim][trt] Add Zero-Dim tensor support
 for clip, cast, flatten_contiguous_range (#53769)

* [inference Zero-Dim][trt]clip,cast,flatten_contiguous_range trt op converter support zero dim
---
 .../convert/flatten_contiguous_range_op.cc    |  53 +++----
 paddle/fluid/inference/tensorrt/op_teller.cc  |  28 ++++
 test/ir/inference/test_trt_convert_cast.py    | 136 +++++++++++-------
 test/ir/inference/test_trt_convert_clip.py    |  14 +-
 ...st_trt_convert_flatten_contiguous_range.py | 107 +++++++++-----
 5 files changed, 220 insertions(+), 118 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc b/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc
index 1705a574c12cc..32bee4fca9fa0 100644
--- a/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc
+++ b/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc
@@ -30,7 +30,6 @@ class FlattenContiguousRangeOpConverter : public OpConverter {
     const int dims = input_dim.nbDims;
     int start_axis = PADDLE_GET_CONST(int, op_desc.GetAttr("start_axis"));
     int stop_axis = PADDLE_GET_CONST(int, op_desc.GetAttr("stop_axis"));
-
     nvinfer1::IShuffleLayer* layer =
         TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input);
     if (!engine_->with_dynamic_shape()) {
@@ -58,32 +57,38 @@ class FlattenContiguousRangeOpConverter : public OpConverter {
       }
       layer->setReshapeDimensions(flatten_dim);
     } else {
-      if (start_axis < 0) start_axis += dims;
-      if (stop_axis < 0) stop_axis += dims;
-
-      int dim_prod = 1;
-      int dim_negative = 0;
       nvinfer1::Dims flatten_dim;
-      flatten_dim.nbDims = dims - (stop_axis - start_axis);
       bool need_slice = false;
-      for (int i = 0, j = 0; i < dims; ++i) {
-        int dim_i = input_dim.d[i];
-        if (start_axis <= i && i <= stop_axis) {
-          if (dim_i < 0) {
-            need_slice = true;
-            break;
-          }
-          dim_prod *= dim_i;
-          if (i == stop_axis) {
-            flatten_dim.d[j++] = dim_prod;
-          }
-        } else {
-          if (dim_i < 0) dim_negative++;
-          if (dim_negative > 1) {
-            need_slice = true;
-            break;
+      if (dims == 0) {
+        flatten_dim.nbDims = 1;
+        flatten_dim.d[0] = 1;
+      } else {
+        if (start_axis < 0) start_axis += dims;
+        if (stop_axis < 0) stop_axis += dims;
+
+        int dim_prod = 1;
+        int dim_negative = 0;
+
+        flatten_dim.nbDims = dims - (stop_axis - start_axis);
+        for (int i = 0, j = 0; i < dims; ++i) {
+          int dim_i = input_dim.d[i];
+          if (start_axis <= i && i <= stop_axis) {
+            if (dim_i < 0) {
+              need_slice = true;
+              break;
+            }
+            dim_prod *= dim_i;
+            if (i == stop_axis) {
+              flatten_dim.d[j++] = dim_prod;
+            }
+          } else {
+            if (dim_i < 0) dim_negative++;
+            if (dim_negative > 1) {
+              need_slice = true;
+              break;
+            }
+            flatten_dim.d[j++] = input_dim.d[i];
           }
-          flatten_dim.d[j++] = input_dim.d[i];
         }
       }
 
diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index 28fb2e985cead..c8bc26c9200ee 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -536,6 +536,12 @@ struct SimpleOpTypeSetTeller : public Teller {
         auto* x_var_desc = block->FindVar(x_var_name);
         const auto x_shape = x_var_desc->GetShape();
         int dims = x_shape.size();
+        if (dims == 0) {
+          VLOG(3) << op_type
+                  << " op does not support input's dim is 0 in tensorrt "
+                     "static shape mode.";
+          return false;
+        }
         if (start_axis < 0) start_axis += dims;
         if (start_axis == 0) {
           VLOG(3) << "TRT flatten_contiguous_range not support the "
@@ -2231,6 +2237,12 @@ struct SimpleOpTypeSetTeller : public Teller {
       auto x_var_name = desc.Input("X")[0];
       auto* x_var_desc = block->FindVar(x_var_name);
       const auto x_shape = x_var_desc->GetShape();
+      if (!with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
+        VLOG(3) << op_type
+                << " op does not support input's dim is 1 or 0 in tensorrt "
+                   "static shape mode.";
+        return false;
+      }
     }
 
     if (op_type == "reduce_sum" || op_type == "reduce_mean" ||
@@ -2421,6 +2433,22 @@ struct SimpleOpTypeSetTeller : public Teller {
 #endif
         return false;
       }
+      auto* block = desc.Block();
+      if (block == nullptr) {
+        VLOG(3) << "The block desc is nullptr, we can't continue to analyze. "
+                   "Developers need to check whether block_desc is passed in "
+                   "the pass.";
+        return false;
+      }
+      auto x_var_name = desc.Input("X")[0];
+      auto* x_var_desc = block->FindVar(x_var_name);
+      const auto x_shape = x_var_desc->GetShape();
+      if (!with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
+        VLOG(3) << op_type
+                << " op does not support input's dim is 1 or 0 in tensorrt "
+                   "static shape mode.";
+        return false;
+      }
     }
 
     if (op_type == "set_value") {
diff --git a/test/ir/inference/test_trt_convert_cast.py b/test/ir/inference/test_trt_convert_cast.py
index 08f5756a4fa87..026abc571050a 100644
--- a/test/ir/inference/test_trt_convert_cast.py
+++ b/test/ir/inference/test_trt_convert_cast.py
@@ -53,69 +53,97 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool:
 
     def sample_program_configs(self):
         def generate_input(type):
-            return np.ones([1, 3, 64, 64]).astype(type)
-
-        for in_dtype in [np.bool_, np.int32, np.float32, np.float64, np.int64]:
-            for out_dtype in [
+            if self.dims == 0:
+                return np.ones([]).astype(type)
+            elif self.dims == 1:
+                return np.ones([1]).astype(type)
+            else:
+                return np.ones([1, 3, 64, 64]).astype(type)
+
+        for dims in [0, 1, 4]:
+            self.dims = dims
+            for in_dtype in [
                 np.bool_,
                 np.int32,
                 np.float32,
                 np.float64,
                 np.int64,
             ]:
-                self.has_bool_dtype = (in_dtype == np.bool_) or (
-                    out_dtype == np.bool_
-                )
-                dics = [
-                    {
-                        "in_dtype": convert_np_dtype_to_dtype_(in_dtype),
-                        "out_dtype": convert_np_dtype_to_dtype_(out_dtype),
-                    },
-                    {
-                        "in_dtype": convert_np_dtype_to_dtype_(out_dtype),
-                        "out_dtype": convert_np_dtype_to_dtype_(in_dtype),
-                    },
-                ]
-
-                ops_config = [
-                    {
-                        "op_type": "cast",
-                        "op_inputs": {"X": ["input_data"]},
-                        "op_outputs": {"Out": ["cast_output_data0"]},
-                        "op_attrs": dics[0],
-                        "outputs_dtype": {"cast_output_data0": out_dtype},
-                    },
-                    {
-                        "op_type": "cast",
-                        "op_inputs": {"X": ["cast_output_data0"]},
-                        "op_outputs": {"Out": ["cast_output_data1"]},
-                        "op_attrs": dics[1],
-                        "outputs_dtype": {"cast_output_data1": in_dtype},
-                    },
-                ]
-
-                ops = self.generate_op_config(ops_config)
-
-                program_config = ProgramConfig(
-                    ops=ops,
-                    weights={},
-                    inputs={
-                        "input_data": TensorConfig(
-                            data_gen=partial(generate_input, in_dtype)
-                        )
-                    },
-                    outputs=["cast_output_data1"],
-                )
-
-                yield program_config
+                for out_dtype in [
+                    np.bool_,
+                    np.int32,
+                    np.float32,
+                    np.float64,
+                    np.int64,
+                ]:
+                    self.has_bool_dtype = (in_dtype == np.bool_) or (
+                        out_dtype == np.bool_
+                    )
+                    dics = [
+                        {
+                            "in_dtype": convert_np_dtype_to_dtype_(in_dtype),
+                            "out_dtype": convert_np_dtype_to_dtype_(out_dtype),
+                        },
+                        {
+                            "in_dtype": convert_np_dtype_to_dtype_(out_dtype),
+                            "out_dtype": convert_np_dtype_to_dtype_(in_dtype),
+                        },
+                    ]
+
+                    ops_config = [
+                        {
+                            "op_type": "cast",
+                            "op_inputs": {"X": ["input_data"]},
+                            "op_outputs": {"Out": ["cast_output_data0"]},
+                            "op_attrs": dics[0],
+                            "outputs_dtype": {"cast_output_data0": out_dtype},
+                        },
+                        {
+                            "op_type": "cast",
+                            "op_inputs": {"X": ["cast_output_data0"]},
+                            "op_outputs": {"Out": ["cast_output_data1"]},
+                            "op_attrs": dics[1],
+                            "outputs_dtype": {"cast_output_data1": in_dtype},
+                        },
+                    ]
+
+                    ops = self.generate_op_config(ops_config)
+
+                    program_config = ProgramConfig(
+                        ops=ops,
+                        weights={},
+                        inputs={
+                            "input_data": TensorConfig(
+                                data_gen=partial(generate_input, in_dtype)
+                            )
+                        },
+                        outputs=["cast_output_data1"],
+                    )
+
+                    yield program_config
 
     def sample_predictor_configs(
         self, program_config
     ) -> (paddle_infer.Config, List[int], float):
         def generate_dynamic_shape(attrs):
-            self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 64, 64]}
-            self.dynamic_shape.max_input_shape = {"input_data": [1, 3, 64, 64]}
-            self.dynamic_shape.opt_input_shape = {"input_data": [1, 3, 64, 64]}
+            if self.dims == 0:
+                self.dynamic_shape.min_input_shape = {"input_data": []}
+                self.dynamic_shape.max_input_shape = {"input_data": []}
+                self.dynamic_shape.opt_input_shape = {"input_data": []}
+            elif self.dims == 1:
+                self.dynamic_shape.min_input_shape = {"input_data": [1]}
+                self.dynamic_shape.max_input_shape = {"input_data": [1]}
+                self.dynamic_shape.opt_input_shape = {"input_data": [1]}
+            else:
+                self.dynamic_shape.min_input_shape = {
+                    "input_data": [1, 3, 64, 64]
+                }
+                self.dynamic_shape.max_input_shape = {
+                    "input_data": [1, 3, 64, 64]
+                }
+                self.dynamic_shape.opt_input_shape = {
+                    "input_data": [1, 3, 64, 64]
+                }
 
         def clear_dynamic_shape():
             self.dynamic_shape.min_input_shape = {}
@@ -123,7 +151,9 @@ def clear_dynamic_shape():
             self.dynamic_shape.opt_input_shape = {}
 
         def generate_trt_nodes_num(attrs, dynamic_shape):
-            if not dynamic_shape and self.has_bool_dtype:
+            if not dynamic_shape and (
+                self.has_bool_dtype or self.dims == 1 or self.dims == 0
+            ):
                 return 0, 4
             return 1, 2
 
diff --git a/test/ir/inference/test_trt_convert_clip.py b/test/ir/inference/test_trt_convert_clip.py
index 4354a034b4e88..c6fd638c2f97a 100644
--- a/test/ir/inference/test_trt_convert_clip.py
+++ b/test/ir/inference/test_trt_convert_clip.py
@@ -29,7 +29,9 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool:
 
     def sample_program_configs(self):
         def generate_input1(dims, batch, attrs: List[Dict[str, Any]]):
-            if dims == 1:
+            if dims == 0:
+                return np.ones([]).astype(np.float32)
+            elif dims == 1:
                 return np.ones([32]).astype(np.float32)
             elif dims == 2:
                 return np.ones([3, 32]).astype(np.float32)
@@ -44,7 +46,7 @@ def generate_weight1(attrs: List[Dict[str, Any]]):
         def generate_weight2(attrs: List[Dict[str, Any]]):
             return np.array([np.random.uniform(10, 20)]).astype("float32")
 
-        for dims in [1, 2, 3, 4]:
+        for dims in [0, 1, 2, 3, 4]:
             for batch in [1, 4]:
                 for op_inputs in [
                     {"X": ["input_data"]},
@@ -93,7 +95,11 @@ def generate_weight2(attrs: List[Dict[str, Any]]):
 
     def sample_predictor_configs(self, program_config):
         def generate_dynamic_shape(attrs):
-            if self.dims == 1:
+            if self.dims == 0:
+                self.dynamic_shape.min_input_shape = {"input_data": []}
+                self.dynamic_shape.max_input_shape = {"input_data": []}
+                self.dynamic_shape.opt_input_shape = {"input_data": []}
+            elif self.dims == 1:
                 self.dynamic_shape.min_input_shape = {"input_data": [1]}
                 self.dynamic_shape.max_input_shape = {"input_data": [64]}
                 self.dynamic_shape.opt_input_shape = {"input_data": [32]}
@@ -125,7 +131,7 @@ def generate_trt_nodes_num(attrs, dynamic_shape):
             if self.input_num == 3:
                 return 0, 3
             else:
-                if not dynamic_shape and self.dims == 1:
+                if not dynamic_shape and (self.dims == 1 or self.dims == 0):
                     return 0, 3
                 else:
                     return 1, 2
diff --git a/test/ir/inference/test_trt_convert_flatten_contiguous_range.py b/test/ir/inference/test_trt_convert_flatten_contiguous_range.py
index ce262692be751..f6d5130ed070c 100644
--- a/test/ir/inference/test_trt_convert_flatten_contiguous_range.py
+++ b/test/ir/inference/test_trt_convert_flatten_contiguous_range.py
@@ -29,48 +29,77 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool:
 
     def sample_program_configs(self):
         def generate_input(batch):
-            return np.random.random([2, batch, 4, 8, 3]).astype(np.float32)
-
-        for batch in [1, 2, 4]:
-            for start_axis in range(5):
-                for stop_axis in range(start_axis, 5):
-                    type = "flatten_contiguous_range"
-                    op_outputs = {
-                        "Out": ["output_data"],
-                        "XShape": ["xshape_data"],
-                    }
-                    ops_config = [
-                        {
-                            "op_type": type,
-                            "op_inputs": {"X": ["input_data"]},
-                            "op_outputs": op_outputs,
-                            "op_attrs": {
-                                "start_axis": start_axis,
-                                "stop_axis": stop_axis,
-                            },
+            if self.dims == 0:
+                return np.random.random([]).astype(np.float32)
+            elif self.dims == 1:
+                return np.random.random([2]).astype(np.float32)
+            else:
+                return np.random.random([2, batch, 4, 8, 3]).astype(np.float32)
+
+        for dims in [0, 1, 5]:
+            self.dims = dims
+            if dims == 0:
+                test_dims = 1
+            else:
+                test_dims = dims
+            for batch in [1, 2, 4]:
+                for start_axis in range(0, test_dims):
+                    test_start = start_axis
+                    if dims == 0:
+                        test_start = -1
+                    for stop_axis in range(test_start, dims):
+                        type = "flatten_contiguous_range"
+                        op_outputs = {
+                            "Out": ["output_data"],
+                            "XShape": ["xshape_data"],
                         }
-                    ]
-                    ops = self.generate_op_config(ops_config)
-
-                    program_config = ProgramConfig(
-                        ops=ops,
-                        weights={},
-                        inputs={
-                            "input_data": TensorConfig(
-                                data_gen=partial(generate_input, batch)
-                            )
-                        },
-                        outputs=["output_data"],
-                    )
-                    yield program_config
+                        ops_config = [
+                            {
+                                "op_type": type,
+                                "op_inputs": {"X": ["input_data"]},
+                                "op_outputs": op_outputs,
+                                "op_attrs": {
+                                    "start_axis": start_axis,
+                                    "stop_axis": stop_axis,
+                                },
+                            }
+                        ]
+                        ops = self.generate_op_config(ops_config)
+
+                        program_config = ProgramConfig(
+                            ops=ops,
+                            weights={},
+                            inputs={
+                                "input_data": TensorConfig(
+                                    data_gen=partial(generate_input, batch)
+                                )
+                            },
+                            outputs=["output_data"],
+                        )
+                        yield program_config
 
     def sample_predictor_configs(
         self, program_config
     ) -> (paddle_infer.Config, List[int], float):
         def generate_dynamic_shape(attrs):
-            self.dynamic_shape.min_input_shape = {"input_data": [2, 1, 4, 8, 3]}
-            self.dynamic_shape.max_input_shape = {"input_data": [2, 4, 4, 8, 3]}
-            self.dynamic_shape.opt_input_shape = {"input_data": [2, 2, 4, 8, 3]}
+            if self.dims == 0:
+                self.dynamic_shape.min_input_shape = {"input_data": []}
+                self.dynamic_shape.max_input_shape = {"input_data": []}
+                self.dynamic_shape.opt_input_shape = {"input_data": []}
+            elif self.dims == 1:
+                self.dynamic_shape.min_input_shape = {"input_data": [2]}
+                self.dynamic_shape.max_input_shape = {"input_data": [2]}
+                self.dynamic_shape.opt_input_shape = {"input_data": [2]}
+            else:
+                self.dynamic_shape.min_input_shape = {
+                    "input_data": [2, 1, 4, 8, 3]
+                }
+                self.dynamic_shape.max_input_shape = {
+                    "input_data": [2, 4, 4, 8, 3]
+                }
+                self.dynamic_shape.opt_input_shape = {
+                    "input_data": [2, 2, 4, 8, 3]
+                }
 
         def clear_dynamic_shape():
             self.dynamic_shape.max_input_shape = {}
@@ -83,7 +112,11 @@ def generate_trt_nodes_num(attrs, dynamic_shape):
                 if dynamic_shape:
                     return 1, 2
                 else:
-                    if attrs[0]['start_axis'] == 0:
+                    if (
+                        attrs[0]['start_axis'] == 0
+                        or self.dims == 0
+                        or self.dims == 1
+                    ):
                         return 0, 3
                     else:
                         return 1, 2

From e04f8d4a6de786ea010d6cc1d6273c6c50cbf237 Mon Sep 17 00:00:00 2001
From: ronnywang <ronny1996@163.com>
Date: Mon, 15 May 2023 19:23:48 +0800
Subject: [PATCH 335/405] [CustomDevice] add inference MP support, PART2
 (#53701)

---
 .../operators/collective/c_comm_init_op.cc    |  80 +++---
 .../operators/collective/c_gen_xccl_id_op.cc  | 132 ++++++++++
 .../custom_device_common_op_registry.cc       | 227 ++++++++++++++++++
 paddle/fluid/pybind/CMakeLists.txt            |  11 +
 paddle/fluid/pybind/imperative.cc             |  21 +-
 paddle/fluid/pybind/pybind.cc                 |   2 +
 6 files changed, 442 insertions(+), 31 deletions(-)
 create mode 100644 paddle/fluid/operators/collective/c_gen_xccl_id_op.cc

diff --git a/paddle/fluid/operators/collective/c_comm_init_op.cc b/paddle/fluid/operators/collective/c_comm_init_op.cc
index b32857a27b2d2..5a22ad716e195 100644
--- a/paddle/fluid/operators/collective/c_comm_init_op.cc
+++ b/paddle/fluid/operators/collective/c_comm_init_op.cc
@@ -25,7 +25,7 @@ limitations under the License. */
 #include "paddle/fluid/framework/op_registry.h"
 
 #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) || \
-    defined(PADDLE_WITH_XPU_BKCL)
+    defined(PADDLE_WITH_XPU_BKCL) || defined(PADDLE_WITH_CUSTOM_DEVICE)
 #include "paddle/fluid/platform/collective_helper.h"
 #endif
 
@@ -48,43 +48,67 @@ class CCommInitOp : public framework::OperatorBase {
 
   void RunImpl(const framework::Scope& scope,
                const platform::Place& place) const override {
+    if (platform::is_custom_place(place)) {
+#if defined(PADDLE_WITH_CUSTOM_DEVICE)
+      auto var = scope.FindVar(Input("X"));
+      PADDLE_ENFORCE_NOT_NULL(
+          var, platform::errors::InvalidArgument("Input con not be empty."));
+
+      phi::ccl::CCLRootId* comm_id = var->GetMutable<phi::ccl::CCLRootId>();
+
+      int nranks = Attr<int>("nranks");
+      int rid = Attr<int>("ring_id");
+
+      int device_id = place.device;
+      if (Attr<int>("device_id") >= 0) {
+        device_id = Attr<int>("device_id");
+      }
+      int rank_id = Attr<int>("rank");
+      platform::XCCLCommContext::Instance(place.GetDeviceType())
+          .CreateComm(comm_id, nranks, rank_id, device_id, rid);
+#else
+      PADDLE_THROW(platform::errors::PreconditionNotMet(
+          "PaddlePaddle should compile with custom device."));
+#endif
+    } else {
 // TODO(wangxi): Put this in the unified header file
 #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
-    using UniqueId = ncclUniqueId;
-    using CommContext = platform::NCCLCommContext;
+      using UniqueId = ncclUniqueId;
+      using CommContext = platform::NCCLCommContext;
 #elif defined(PADDLE_WITH_XPU_BKCL)
-    using UniqueId = BKCLUniqueId;
-    using CommContext = platform::BKCLCommContext;
+      using UniqueId = BKCLUniqueId;
+      using CommContext = platform::BKCLCommContext;
 #else
-    PADDLE_THROW(platform::errors::PreconditionNotMet(
-        "PaddlePaddle should be compiled with GPU or XPU."));
+      PADDLE_THROW(platform::errors::PreconditionNotMet(
+          "PaddlePaddle should be compiled with GPU or XPU."));
 #endif
 
-    PADDLE_ENFORCE_EQ(
-        platform::is_gpu_place(place) || platform::is_xpu_place(place),
-        true,
-        platform::errors::PreconditionNotMet(
-            "CCommInitOp can run on gpu or xpu place only."));
+      PADDLE_ENFORCE_EQ(
+          platform::is_gpu_place(place) || platform::is_xpu_place(place),
+          true,
+          platform::errors::PreconditionNotMet(
+              "CCommInitOp can run on gpu or xpu place only."));
 
 #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) || \
     defined(PADDLE_WITH_XPU_BKCL)
-    auto var = scope.FindVar(Input("X"));
-    PADDLE_ENFORCE_NOT_NULL(
-        var, platform::errors::InvalidArgument("Input con not be empty."));
-
-    UniqueId* comm_id = var->GetMutable<UniqueId>();
-
-    int nranks = Attr<int>("nranks");
-    int rid = Attr<int>("ring_id");
-
-    int device_id = place.device;
-    if (Attr<int>("device_id") >= 0) {
-      device_id = Attr<int>("device_id");
-    }
-    int rank_id = Attr<int>("rank");
-    CommContext::Instance().CreateComm(
-        comm_id, nranks, rank_id, device_id, rid);
+      auto var = scope.FindVar(Input("X"));
+      PADDLE_ENFORCE_NOT_NULL(
+          var, platform::errors::InvalidArgument("Input con not be empty."));
+
+      UniqueId* comm_id = var->GetMutable<UniqueId>();
+
+      int nranks = Attr<int>("nranks");
+      int rid = Attr<int>("ring_id");
+
+      int device_id = place.device;
+      if (Attr<int>("device_id") >= 0) {
+        device_id = Attr<int>("device_id");
+      }
+      int rank_id = Attr<int>("rank");
+      CommContext::Instance().CreateComm(
+          comm_id, nranks, rank_id, device_id, rid);
 #endif
+    }
   }
 };
 
diff --git a/paddle/fluid/operators/collective/c_gen_xccl_id_op.cc b/paddle/fluid/operators/collective/c_gen_xccl_id_op.cc
new file mode 100644
index 0000000000000..effe7021b0d7e
--- /dev/null
+++ b/paddle/fluid/operators/collective/c_gen_xccl_id_op.cc
@@ -0,0 +1,132 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#include <string>
+
+#include "paddle/fluid/framework/op_proto_maker.h"
+#include "paddle/fluid/framework/op_registry.h"
+#include "paddle/fluid/framework/operator.h"
+#include "paddle/fluid/framework/scope.h"
+#include "paddle/fluid/framework/var_type_traits.h"
+#include "paddle/fluid/platform/device_context.h"
+#include "paddle/fluid/platform/enforce.h"
+#include "paddle/fluid/platform/gen_comm_id_helper.h"
+#include "paddle/fluid/platform/place.h"
+#include "paddle/phi/backends/device_manager.h"
+
+namespace paddle {
+namespace operators {
+
+#ifdef PADDLE_WITH_CUSTOM_DEVICE
+static void CopyXCCLIDToVar(const std::vector<phi::ccl::CCLRootId>& xccl_ids,
+                            std::function<std::string(size_t)> func,
+                            const framework::Scope& scope) {
+  for (size_t i = 0; i < xccl_ids.size(); ++i) {
+    std::string var_name = func(i);
+    auto var = scope.FindVar(var_name);
+    PADDLE_ENFORCE_NOT_NULL(
+        var,
+        platform::errors::NotFound("Variable with name %s is not found",
+                                   var_name.c_str()));
+    auto xccl_id = var->GetMutable<phi::ccl::CCLRootId>();
+    *xccl_id = xccl_ids[i];
+  }
+}
+
+class CGenXCCLIdOp : public framework::OperatorBase {
+ public:
+  CGenXCCLIdOp(const std::string& type,
+               const framework::VariableNameMap& inputs,
+               const framework::VariableNameMap& outputs,
+               const framework::AttributeMap& attrs)
+      : OperatorBase(type, inputs, outputs, attrs) {}
+
+  void RunImpl(const framework::Scope& scope,
+               const platform::Place& dev_place) const override {
+    int rank = Attr<int>("rank");
+    int ring_id = Attr<int>("ring_id");
+
+    std::function<std::string(size_t)> func = [&](size_t i) -> std::string {
+      return Output("Out");
+    };
+
+    std::string endpoint = Attr<std::string>("endpoint");
+    int server_fd = platform::SocketServer::GetInstance(endpoint).socket();
+
+    std::vector<phi::ccl::CCLRootId> xccl_ids;
+    xccl_ids.resize(1);
+
+    if (rank == 0) {
+      for (size_t i = 0; i < xccl_ids.size(); ++i) {
+        phi::DeviceManager::CCLGetUniqueId(dev_place.GetDeviceType(),
+                                           &xccl_ids[i]);
+      }
+      std::vector<std::string> endpoint_list =
+          Attr<std::vector<std::string>>("other_endpoints");
+      platform::SendBroadCastCommID(endpoint_list, &xccl_ids, ring_id);
+    } else {
+      platform::RecvBroadCastCommID(server_fd, endpoint, &xccl_ids, ring_id);
+    }
+
+    CopyXCCLIDToVar(xccl_ids, func, scope);
+  }
+};
+
+#else
+class CGenXCCLIdOp : public framework::OperatorBase {
+ public:
+  CGenXCCLIdOp(const std::string& type,
+               const framework::VariableNameMap& inputs,
+               const framework::VariableNameMap& outputs,
+               const framework::AttributeMap& attrs)
+      : OperatorBase(type, inputs, outputs, attrs) {}
+
+  void RunImpl(const framework::Scope& scope,
+               const platform::Place& dev_place) const override {}
+};
+
+#endif
+
+class CGenXCCLIdOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+  void Make() override {
+    AddOutput("Out", "Raw variable contains a XCCL UniqueId instaces.");
+    AddComment(R"DOC(
+CGenXCCLId operator
+
+For trainer 0: generate a new UniqueId and send it to all the other trainers.
+For trainer 1~n: start a gRPC server to get the UniqueId, once got, stop the server.
+)DOC");
+    AddAttr<std::string>("endpoint",
+                         "(string), e.g. 127.0.0.1:6175 "
+                         "current listen endpoint");
+    AddAttr<std::vector<std::string>>(
+        "other_endpoints",
+        "['trainer1_ip:port', 'trainer2_ip:port', ...] "
+        "list of other trainer endpoints")
+        .SetDefault({});
+    AddAttr<int>("rank",
+                 "(int default 0) "
+                 "The rank of the trainer in distributed training.")
+        .SetDefault(0);
+    AddAttr<int>("ring_id", "(int default 0) user specified ring id")
+        .SetDefault(0);
+  }
+};
+
+}  // namespace operators
+}  // namespace paddle
+
+namespace ops = paddle::operators;
+
+REGISTER_OPERATOR(c_gen_xccl_id, ops::CGenXCCLIdOp, ops::CGenXCCLIdOpMaker);
diff --git a/paddle/fluid/operators/custom_device_common_op_registry.cc b/paddle/fluid/operators/custom_device_common_op_registry.cc
index a95fc0cd3206a..f4d9aff037497 100644
--- a/paddle/fluid/operators/custom_device_common_op_registry.cc
+++ b/paddle/fluid/operators/custom_device_common_op_registry.cc
@@ -19,6 +19,7 @@ limitations under the License. */
 #include "paddle/fluid/operators/load_combine_op.h"
 #include "paddle/fluid/operators/run_program_op.h"
 #include "paddle/fluid/operators/save_combine_op.h"
+#include "paddle/fluid/platform/collective_helper.h"
 #include "paddle/phi/api/backward/backward_api.h"
 #include "paddle/phi/api/include/api.h"
 #include "paddle/phi/backends/device_manager.h"
@@ -497,6 +498,131 @@ class CSyncCalcStreamCustomDeviceKernel : public framework::OpKernel<T> {
   }
 };
 
+template <typename DeviceContext, typename T, phi::ccl::CCLReduceOp red_type>
+class CAllReduceOpCustomDeviceKernel : public framework::OpKernel<T> {
+ public:
+  void Compute(const framework::ExecutionContext& ctx) const override {
+    auto in = ctx.Input<phi::DenseTensor>("X");
+    auto out = ctx.Output<phi::DenseTensor>("Out");
+    int rid = ctx.Attr<int>("ring_id");
+
+    auto place = ctx.GetPlace();
+    auto dtype = phi::ccl::ToCCLDataType(in->dtype());
+    int64_t numel = in->numel();
+    const void* sendbuff = in->data<T>();
+    out->Resize(in->dims());
+    void* recvbuff = ctx.device_context().Alloc<T>(out);
+
+    auto map = paddle::distributed::ProcessGroupMapFromGid::getInstance();
+    if (map->has(rid)) {
+      // Use ProcessGroup
+      paddle::distributed::ProcessGroup* pg = map->get(rid);
+      std::vector<phi::DenseTensor> in_tensor;
+      std::vector<phi::DenseTensor> out_tensor;
+      in_tensor.push_back(*in);
+      out_tensor.push_back(*out);
+
+      paddle::distributed::AllreduceOptions opts;
+      switch (red_type) {
+        case phi::ccl::CCLReduceOp::SUM:
+          opts.reduce_op = paddle::distributed::ReduceOp::SUM;
+          break;
+
+        case phi::ccl::CCLReduceOp::MAX:
+          opts.reduce_op = paddle::distributed::ReduceOp::MAX;
+          break;
+
+        case phi::ccl::CCLReduceOp::MIN:
+          opts.reduce_op = paddle::distributed::ReduceOp::MIN;
+          break;
+
+        case phi::ccl::CCLReduceOp::PRODUCT:
+          opts.reduce_op = paddle::distributed::ReduceOp::PRODUCT;
+          break;
+
+        default:
+          PADDLE_THROW(paddle::platform::errors::InvalidArgument(
+              "Invalid reduce type: %d", red_type));
+      }
+
+      auto task = pg->AllReduce(in_tensor, out_tensor, opts);
+      task->Wait();
+      return;
+    }
+
+    auto comm =
+        paddle::platform::XCCLCommContext::Instance(place.GetDeviceType())
+            .Get(rid, place);
+
+    std::shared_ptr<phi::stream::Stream> stream;
+    if (ctx.Attr<bool>("use_calc_stream")) {
+      auto dev_ctx = paddle::platform::DeviceContextPool::Instance().Get(place);
+      stream = static_cast<paddle::platform::CustomDeviceContext*>(dev_ctx)
+                   ->GetStream();
+    } else {
+      stream = comm->stream();
+    }
+    phi::DeviceManager::CCLAllReduce(place.GetDeviceType(),
+                                     const_cast<void*>(sendbuff),
+                                     recvbuff,
+                                     numel,
+                                     dtype,
+                                     red_type,
+                                     comm->comm(),
+                                     *stream);
+  }
+};
+
+template <typename T>
+class CBroadcastOpCustomDeviceKernel : public framework::OpKernel<T> {
+ public:
+  void Compute(const framework::ExecutionContext& ctx) const override {
+    auto x = ctx.Input<phi::DenseTensor>("X");
+    auto out = ctx.Output<phi::DenseTensor>("Out");
+    const auto& place = ctx.GetPlace();
+    ctx.device_context().Alloc<T>(out);
+    int root = ctx.Attr<int>("root");
+    int rid = ctx.Attr<int>("ring_id");
+
+    auto stream = static_cast<const phi::CustomContext&>(ctx.device_context())
+                      .GetStream();
+
+    int numel = x->numel();
+    auto dtype = phi::ccl::ToCCLDataType(x->dtype());
+    auto comm = platform::XCCLCommContext::Instance(place.GetDeviceType())
+                    .Get(rid, place);
+    if (root == comm->rank()) {
+      phi::DeviceManager::CCLBroadcast(place.GetDeviceType(),
+                                       const_cast<void*>(x->data()),
+                                       numel,
+                                       dtype,
+                                       root,
+                                       comm->comm(),
+                                       *stream);
+      VLOG(3) << "rank " << comm->rank() << " invoke Bcast. sent "
+              << x->numel();
+      if (out != x) {
+        framework::TensorCopy(
+            *static_cast<const phi::DenseTensor*>(x),
+            place,
+            *platform::DeviceContextPool::Instance().Get(place),
+            static_cast<phi::DenseTensor*>(out));
+      }
+    } else {
+      phi::DeviceManager::CCLBroadcast(place.GetDeviceType(),
+                                       out->data(),
+                                       numel,
+                                       dtype,
+                                       root,
+                                       comm->comm(),
+                                       *stream);
+      VLOG(3) << "rank " << comm->rank() << " invoke Bcast. received "
+              << phi::product(out->dims());
+    }
+    out->set_lod(x->lod());
+  }
+};
+
 template <typename Context>
 void FeedDenseTensorKernel(const Context& dev_ctx,
                            const phi::ExtendedTensor& x,
@@ -636,6 +762,107 @@ void RegisterCustomDeviceCommonKernel(const std::string& dev_type) {
       paddle::operators::CSyncCalcStreamCustomDeviceKernel<
           paddle::platform::CustomDeviceContext,
           paddle::platform::float16>) {}
+  REGISTER_OP_CUSTOM_DEVICE_KERNEL(
+      c_allreduce_sum,
+      device_type,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          float,
+          phi::ccl::CCLReduceOp::SUM>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          double,
+          phi::ccl::CCLReduceOp::SUM>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          paddle::platform::float16,
+          phi::ccl::CCLReduceOp::SUM>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          int32_t,
+          phi::ccl::CCLReduceOp::SUM>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          int64_t,
+          phi::ccl::CCLReduceOp::SUM>) {}
+  REGISTER_OP_CUSTOM_DEVICE_KERNEL(
+      c_allreduce_min,
+      device_type,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          float,
+          phi::ccl::CCLReduceOp::MIN>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          double,
+          phi::ccl::CCLReduceOp::MIN>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          paddle::platform::float16,
+          phi::ccl::CCLReduceOp::MIN>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          int32_t,
+          phi::ccl::CCLReduceOp::MIN>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          int64_t,
+          phi::ccl::CCLReduceOp::MIN>) {}
+  REGISTER_OP_CUSTOM_DEVICE_KERNEL(
+      c_allreduce_max,
+      device_type,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          float,
+          phi::ccl::CCLReduceOp::MAX>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          double,
+          phi::ccl::CCLReduceOp::MAX>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          paddle::platform::float16,
+          phi::ccl::CCLReduceOp::MAX>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          int32_t,
+          phi::ccl::CCLReduceOp::MAX>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          int64_t,
+          phi::ccl::CCLReduceOp::MAX>) {}
+  REGISTER_OP_CUSTOM_DEVICE_KERNEL(
+      c_allreduce_prod,
+      device_type,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          float,
+          phi::ccl::CCLReduceOp::PRODUCT>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          double,
+          phi::ccl::CCLReduceOp::PRODUCT>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          paddle::platform::float16,
+          phi::ccl::CCLReduceOp::PRODUCT>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          int32_t,
+          phi::ccl::CCLReduceOp::PRODUCT>,
+      paddle::operators::CAllReduceOpCustomDeviceKernel<
+          paddle::platform::CustomDeviceContext,
+          int64_t,
+          phi::ccl::CCLReduceOp::PRODUCT>) {}
+  REGISTER_OP_CUSTOM_DEVICE_KERNEL(
+      c_broadcast,
+      device_type,
+      paddle::operators::CBroadcastOpCustomDeviceKernel<int32_t>,
+      paddle::operators::CBroadcastOpCustomDeviceKernel<int64_t>,
+      paddle::operators::CBroadcastOpCustomDeviceKernel<float>,
+      paddle::operators::CBroadcastOpCustomDeviceKernel<double>,
+      paddle::operators::CBroadcastOpCustomDeviceKernel<
+          paddle::platform::float16>) {}
 #endif
 }
 
diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt
index 1faba336682dc..2375c01e6bd1f 100755
--- a/paddle/fluid/pybind/CMakeLists.txt
+++ b/paddle/fluid/pybind/CMakeLists.txt
@@ -91,6 +91,17 @@ if(WITH_XPU_BKCL)
   set(PYBIND_DEPS ${PYBIND_DEPS} heter_ccl_context)
 endif()
 
+if(WITH_CUSTOM_DEVICE)
+  set(PYBIND_DEPS ${PYBIND_DEPS} xccl_context)
+  if(NOT
+     (WITH_NCCL
+      OR WITH_RCCL
+      OR WITH_XPU_BKCL))
+    set(PYBIND_DEPS ${PYBIND_DEPS} reducer)
+    set(PYBIND_DEPS ${PYBIND_DEPS} heter_ccl_context)
+  endif()
+endif()
+
 if(NOT WIN32)
   set(PYBIND_DEPS ${PYBIND_DEPS} data_loader)
   if(WITH_NCCL OR WITH_RCCL)
diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc
index 65eac1e3dc6fd..5bbd66fd09c98 100644
--- a/paddle/fluid/pybind/imperative.cc
+++ b/paddle/fluid/pybind/imperative.cc
@@ -52,6 +52,7 @@ limitations under the License. */
 #include "paddle/fluid/imperative/reducer.h"
 #include "paddle/fluid/imperative/tracer.h"
 #include "paddle/fluid/imperative/type_defs.h"
+#include "paddle/fluid/imperative/xccl_context.h"
 #include "paddle/fluid/memory/allocation/mmap_allocator.h"
 #include "paddle/fluid/operators/utils.h"
 #include "paddle/fluid/pybind/cuda_streams_py.h"
@@ -2476,8 +2477,9 @@ void BindImperative(py::module *m_ptr) {
       },
       py::call_guard<py::gil_scoped_release>());
 
-#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) || \
-    defined(PADDLE_WITH_XPU_BKCL) || defined(PADDLE_WITH_GLOO)
+#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) ||     \
+    defined(PADDLE_WITH_XPU_BKCL) || defined(PADDLE_WITH_GLOO) || \
+    defined(PADDLE_WITH_CUSTOM_DEVICE)
   py::class_<imperative::ParallelContext,
              std::shared_ptr<imperative::ParallelContext>>(m,
                                                            "ParallelContext");
@@ -2517,6 +2519,19 @@ void BindImperative(py::module *m_ptr) {
            py::arg("ring_id"));
 #endif
 
+#if defined(PADDLE_WITH_CUSTOM_DEVICE)
+  py::class_<imperative::XCCLParallelContext,
+             imperative::ParallelContext,
+             std::shared_ptr<imperative::XCCLParallelContext>>(
+      m, "XCCLParallelContext")
+      .def(py::init<const imperative::ParallelStrategy &,
+                    const platform::CustomPlace &>())
+      .def("init", [](imperative::XCCLParallelContext &self) { self.Init(); })
+      .def("init_with_ring_id",
+           &imperative::XCCLParallelContext::InitWithRingID,
+           py::arg("ring_id"));
+#endif
+
 #if defined(PADDLE_WITH_XPU_BKCL)
   py::class_<imperative::BKCLParallelContext,
              imperative::ParallelContext,
@@ -2545,7 +2560,7 @@ void BindImperative(py::module *m_ptr) {
 #endif
 
 #if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) || \
-    defined(PADDLE_WITH_XPU_BKCL)
+    defined(PADDLE_WITH_XPU_BKCL) || defined(PADDLE_WITH_CUSTOM_DEVICE)
   py::class_<imperative::HeterParallelContext,
              imperative::ParallelContext,
              std::shared_ptr<imperative::HeterParallelContext>>(
diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc
index f4dfb133c1c36..a9b1bf398f84e 100644
--- a/paddle/fluid/pybind/pybind.cc
+++ b/paddle/fluid/pybind/pybind.cc
@@ -158,6 +158,7 @@ limitations under the License. */
 
 #ifdef PADDLE_WITH_CUSTOM_DEVICE
 #include "paddle/fluid/operators/custom_device_common_op_registry.h"
+#include "paddle/fluid/platform/collective_helper.h"
 #include "paddle/phi/capi/capi.h"
 #endif
 
@@ -990,6 +991,7 @@ PYBIND11_MODULE(libpaddle, m) {
         []() { phi::KernelFactory::Instance().kernels().clear(); });
   m.def("clear_device_manager", []() {
 #ifdef PADDLE_WITH_CUSTOM_DEVICE
+    platform::XCCLCommContext::Release();
     phi::DeviceManager::Clear();
 #endif
   });

From 56fded1b8c13175e07e7d2bc4ad976751ea36423 Mon Sep 17 00:00:00 2001
From: ronnywang <ronny1996@163.com>
Date: Mon, 15 May 2023 19:24:00 +0800
Subject: [PATCH 336/405] [CustomDevice] add inference MP support, PART3
 (#53703)

---
 paddle/fluid/pybind/custom_device_py.cc       | 13 ++++++++
 .../distributed/auto_parallel/cluster.py      | 27 +++++++++------
 .../auto_parallel/process_group.py            | 13 ++++++++
 .../meta_optimizers/sharding_optimizer.py     |  1 +
 .../fleet/utils/hybrid_parallel_util.py       |  1 +
 .../ps/utils/collective_transpiler.py         | 33 +++++++++++++++++++
 .../distributed/transpiler/collective.py      | 32 ++++++++++++++++++
 python/paddle/hapi/model.py                   | 31 +++++++++++++++++
 .../optimizer/distributed_fused_lamb.py       | 15 +++++++++
 9 files changed, 156 insertions(+), 10 deletions(-)

diff --git a/paddle/fluid/pybind/custom_device_py.cc b/paddle/fluid/pybind/custom_device_py.cc
index d3b4183f2f4f0..d138115c45ee0 100644
--- a/paddle/fluid/pybind/custom_device_py.cc
+++ b/paddle/fluid/pybind/custom_device_py.cc
@@ -29,6 +29,19 @@ namespace pybind {
 void BindCustomDevicePy(py::module *m_ptr) {
   auto &m = *m_ptr;
   // Bind Methods
+  m.def(
+      "_get_device_total_memory",
+      [](const std::string &device_type, int device_id) {
+        auto place = paddle::platform::CustomPlace(
+            device_type,
+            device_id == -1 ? phi::DeviceManager::GetDevice(device_type)
+                            : device_id);
+        size_t total = 0, free = 0;
+        phi::DeviceManager::MemoryStats(place, &total, &free);
+        return total;
+      },
+      py::arg("device_type"),
+      py::arg("device_id") = -1);
   m.def(
       "_get_current_custom_device_stream",
       [](const std::string &device_type, int device_id) {
diff --git a/python/paddle/distributed/auto_parallel/cluster.py b/python/paddle/distributed/auto_parallel/cluster.py
index 25324f4883334..6a35894900afe 100644
--- a/python/paddle/distributed/auto_parallel/cluster.py
+++ b/python/paddle/distributed/auto_parallel/cluster.py
@@ -883,17 +883,24 @@ def is_by_json_config(json_config):
             assert global_device_count % local_device_count == 0
             node_count = int(global_device_count) // local_device_count
 
-        gpu_info = paddle.device.cuda.get_device_properties()
-        assert gpu_info, "Auto parallel just runs on gpu now."
-
-        gpu_name = gpu_info.name
-        try:
-            re_result = re.split(r'[ , -]', gpu_name)
-            gpu_model = re_result[1]
-            memory = int(re_result[-1][:-2])
-        except:
-            memory = int(gpu_info.total_memory) // (1000**3)
+        if os.getenv("PADDLE_DISTRI_BACKEND", None) == "xccl":
+            gpu_name = os.getenv("PADDLE_XCCL_BACKEND", None)
             gpu_model = gpu_name
+            memory = int(
+                paddle.fluid.core._get_device_total_memory(gpu_name)
+            ) // (1000**3)
+        else:
+            gpu_info = paddle.device.cuda.get_device_properties()
+            assert gpu_info, "Auto parallel just runs on gpu now."
+
+            gpu_name = gpu_info.name
+            try:
+                re_result = re.split(r'[ , -]', gpu_name)
+                gpu_model = re_result[1]
+                memory = int(re_result[-1][:-2])
+            except:
+                memory = int(gpu_info.total_memory) // (1000**3)
+                gpu_model = gpu_name
 
     logger.info(
         "Node Count: {}, Local Device Size: {}, GPU Model: {}, GPU Memory: {}GB, World size: {}, EndPoint: {}.".format(
diff --git a/python/paddle/distributed/auto_parallel/process_group.py b/python/paddle/distributed/auto_parallel/process_group.py
index 8c300cbcd53b6..b5652353311a2 100644
--- a/python/paddle/distributed/auto_parallel/process_group.py
+++ b/python/paddle/distributed/auto_parallel/process_group.py
@@ -148,6 +148,11 @@ def instantiate(self):
                 core.BKCLParallelContext(strategy, place).init_with_ring_id(
                     ring_id
                 )
+            elif genv.device_type in core.get_all_custom_device_type():
+                place = core.CustomPlace(genv.device_type, genv.device_id)
+                core.XCCLParallelContext(strategy, place).init_with_ring_id(
+                    ring_id
+                )
             else:
                 raise AssertionError('No CUDA device found')
 
@@ -162,6 +167,14 @@ def instantiate(self):
                 paddle.set_device(
                     'xpu:%d' % paddle.distributed.ParallelEnv().dev_id
                 )
+            elif genv.device_type in core.get_all_custom_device_type():
+                paddle.set_device(
+                    '%s:%d'
+                    % (
+                        paddle.distributed.ParallelEnv().device_type,
+                        paddle.distributed.ParallelEnv().dev_id,
+                    ),
+                )
             tmp = (
                 paddle.to_tensor([1], dtype="int32")
                 if in_dygraph_mode()
diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py
index 29be8a87fa7d8..59f511084424f 100755
--- a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py
+++ b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py
@@ -1040,6 +1040,7 @@ def _prune_main_program(self, block, shard, rings):
                 "c_calc_comm_stream",
                 "c_gen_nccl_id",
                 "c_gen_bkcl_id",
+                "c_gen_xccl_id",
                 "c_comm_init",
                 'send_v2',
                 'recv_v2',
diff --git a/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py b/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py
index 04093ebcb35fd..cd0455c80697b 100644
--- a/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py
+++ b/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py
@@ -167,6 +167,7 @@ def broadcast_input_data(hcg, *inputs, **kwargs):
         place = paddle.CUDAPlace(dev_idx)
     elif dev in paddle.device.get_all_custom_device_type():
         place = paddle.CustomPlace(dev, dev_idx)
+        dev = 'custom'
     else:
         place = eval(f"paddle.{dev.upper()}Place")(dev_idx)
 
diff --git a/python/paddle/distributed/ps/utils/collective_transpiler.py b/python/paddle/distributed/ps/utils/collective_transpiler.py
index 5219301d9c738..f5278d05367fd 100644
--- a/python/paddle/distributed/ps/utils/collective_transpiler.py
+++ b/python/paddle/distributed/ps/utils/collective_transpiler.py
@@ -14,6 +14,7 @@
 
 import os
 
+import paddle
 from paddle.distributed.fleet.base.private_helper_function import (
     wait_server_ready,
 )
@@ -204,6 +205,38 @@ def _init_communicator(
                         self.op_role_key: OpRole.Forward,
                     },
                 )
+        elif (
+            paddle.distributed.ParallelEnv().device_type
+            in paddle.device.get_all_custom_device_type()
+        ):
+            xccl_id_var = block.create_var(
+                name=unique_name.generate('xccl_id'),
+                persistable=True,
+                type=core.VarDesc.VarType.RAW,
+            )
+            endpoint_to_index_map = {e: idx for idx, e in enumerate(endpoints)}
+            block.append_op(
+                type='c_gen_xccl_id',
+                inputs={},
+                outputs={'Out': xccl_id_var},
+                attrs={
+                    'rank': rank,
+                    'endpoint': current_endpoint,
+                    'other_endpoints': other_endpoints,
+                    self.op_role_key: OpRole.Forward,
+                },
+            )
+            block.append_op(
+                type='c_comm_init',
+                inputs={'X': xccl_id_var},
+                outputs={},
+                attrs={
+                    'nranks': nranks,
+                    'rank': rank,
+                    'ring_id': ring_id,
+                    self.op_role_key: OpRole.Forward,
+                },
+            )
 
     def _broadcast_params(self):
         block = self.startup_program.global_block()
diff --git a/python/paddle/distributed/transpiler/collective.py b/python/paddle/distributed/transpiler/collective.py
index 173f89b91e24b..1fb1cf474a701 100644
--- a/python/paddle/distributed/transpiler/collective.py
+++ b/python/paddle/distributed/transpiler/collective.py
@@ -14,6 +14,7 @@
 
 import os
 
+import paddle
 from paddle.distributed.fleet.base.private_helper_function import (
     wait_server_ready,
 )
@@ -200,6 +201,37 @@ def _init_communicator(
                     self.op_role_key: OpRole.Forward,
                 },
             )
+        elif (
+            paddle.distributed.ParallelEnv().device_type
+            in paddle.device.get_all_custom_device_type()
+        ):
+            xccl_id_var = block.create_var(
+                name=unique_name.generate('xccl_id'),
+                persistable=True,
+                type=core.VarDesc.VarType.RAW,
+            )
+            block.append_op(
+                type='c_gen_xccl_id',
+                inputs={},
+                outputs={'Out': xccl_id_var},
+                attrs={
+                    'rank': rank,
+                    'endpoint': current_endpoint,
+                    'other_endpoints': other_endpoints,
+                    self.op_role_key: OpRole.Forward,
+                },
+            )
+            block.append_op(
+                type='c_comm_init',
+                inputs={'X': xccl_id_var},
+                outputs={},
+                attrs={
+                    'nranks': nranks,
+                    'rank': rank,
+                    'ring_id': ring_id,
+                    self.op_role_key: OpRole.Forward,
+                },
+            )
 
     def _broadcast_params(self):
         block = self.startup_program.global_block()
diff --git a/python/paddle/hapi/model.py b/python/paddle/hapi/model.py
index 3f2db38474eb3..4169902401fcf 100644
--- a/python/paddle/hapi/model.py
+++ b/python/paddle/hapi/model.py
@@ -184,6 +184,37 @@ def init_communicator(
                 'ring_id': 0,
             },
         )
+    elif (
+        paddle.distributed.ParallelEnv().device_type
+        in paddle.device.get_all_custom_device_type()
+    ):
+        xccl_id_var = block.create_var(
+            name=fluid.unique_name.generate('xccl_id'),
+            persistable=True,
+            type=fluid.core.VarDesc.VarType.RAW,
+        )
+
+        block.append_op(
+            type='c_gen_xccl_id',
+            inputs={},
+            outputs={'Out': xccl_id_var},
+            attrs={
+                'rank': rank,
+                'endpoint': current_endpoint,
+                'other_endpoints': other_endpoints,
+            },
+        )
+
+        block.append_op(
+            type='c_comm_init',
+            inputs={'X': xccl_id_var},
+            outputs={},
+            attrs={
+                'nranks': nranks,
+                'rank': rank,
+                'ring_id': 0,
+            },
+        )
 
 
 def prepare_distributed_context(place=None):
diff --git a/python/paddle/incubate/optimizer/distributed_fused_lamb.py b/python/paddle/incubate/optimizer/distributed_fused_lamb.py
index 9a76db3be3d92..a9d9e941d25cc 100644
--- a/python/paddle/incubate/optimizer/distributed_fused_lamb.py
+++ b/python/paddle/incubate/optimizer/distributed_fused_lamb.py
@@ -58,6 +58,21 @@ def init_communicator(block, rank, ranks, ring_id):
                 'ring_id': ring_id,
             },
         )
+    elif (
+        paddle.distributed.ParallelEnv().device_type
+        in paddle.device.get_all_custom_device_type()
+    ):
+        block.append_op(
+            type='c_gen_xccl_id',
+            inputs={},
+            outputs={'Out': comm_id_var},
+            attrs={
+                'rank': local_rank,
+                'endpoint': cur_ep,
+                'other_endpoints': other_eps,
+                'ring_id': ring_id,
+            },
+        )
     block.append_op(
         type='c_comm_init',
         inputs={'X': comm_id_var},

From efd410c86f70066d1c7e0888d2a295364e54423e Mon Sep 17 00:00:00 2001
From: huangjiyi <43315610+huangjiyi@users.noreply.github.com>
Date: Mon, 15 May 2023 21:34:33 +0800
Subject: [PATCH 337/405] move dequantize kernel to phi (#53739)

* update

* fix bug

* fix output type def
---
 .../operators/mkldnn/dequantize_mkldnn_op.cc  | 96 -------------------
 paddle/phi/kernels/dequantize_kernel.h        | 28 ++++++
 .../phi/kernels/onednn/dequantize_kernel.cc   | 86 +++++++++++++++++
 paddle/phi/ops/compat/dequantize_sig.cc       | 26 +++++
 4 files changed, 140 insertions(+), 96 deletions(-)
 delete mode 100644 paddle/fluid/operators/mkldnn/dequantize_mkldnn_op.cc
 create mode 100644 paddle/phi/kernels/dequantize_kernel.h
 create mode 100644 paddle/phi/kernels/onednn/dequantize_kernel.cc
 create mode 100644 paddle/phi/ops/compat/dequantize_sig.cc

diff --git a/paddle/fluid/operators/mkldnn/dequantize_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/dequantize_mkldnn_op.cc
deleted file mode 100644
index 6ed5aefd66d1b..0000000000000
--- a/paddle/fluid/operators/mkldnn/dequantize_mkldnn_op.cc
+++ /dev/null
@@ -1,96 +0,0 @@
-/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include "paddle/fluid/operators/dequantize_op.h"
-
-#include "paddle/fluid/framework/tensor.h"
-#include "paddle/fluid/platform/mkldnn_helper.h"
-#include "paddle/phi/backends/onednn/onednn_reuse.h"
-#include "paddle/phi/core/errors.h"
-
-namespace paddle {
-namespace operators {
-
-using dnnl::memory;
-using dnnl::primitive;
-using dnnl::reorder;
-using dnnl::stream;
-
-template <typename T>
-class DeQuantOpKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
-    auto* x = ctx.Input<phi::DenseTensor>("Input");
-    const auto quantization_scale = ctx.Attr<float>("Scale");
-    const auto quantization_shift = ctx.Attr<float>("Shift");
-    const bool with_shift = quantization_shift != 0.0f;
-    auto* out = ctx.Output<phi::DenseTensor>("Output");
-
-    PADDLE_ENFORCE(quantization_scale != 0.0f,
-                   phi::errors::InvalidArgument(
-                       "Dequantization scale must be different than 0.0f"));
-
-    PADDLE_ENFORCE(quantization_shift <= 255 && quantization_shift >= 0,
-                   phi::errors::InvalidArgument(
-                       "Dequantization shift must be lower or equal to ",
-                       "255 and greater or equal to 0, but got %f",
-                       quantization_shift));
-
-    auto& dev_ctx = ctx.template device_context<phi::OneDNNContext>();
-
-    auto x_tz = phi::vectorize<int64_t>(x->dims());
-    auto x_type = phi::funcs::ToOneDNNDataType(x->dtype());
-    auto out_type = phi::funcs::ToOneDNNDataType(out->dtype());
-
-    dnnl::primitive_attr attrs;
-    static constexpr int32_t mask = 0;  // same shift and scale for whole tensor
-
-    const float reorder_scale = 1. / quantization_scale;
-    attrs.set_output_scales(mask, {reorder_scale});
-
-    if (with_shift) {
-      attrs.set_zero_points(
-          DNNL_ARG_SRC, mask, {static_cast<int32_t>(quantization_shift)});
-    }
-
-    phi::funcs::ReorderOneDNNHandler reorder_handler(
-        x_tz, x->dtype(), x_type, out->dtype(), out_type, dev_ctx.GetEngine());
-
-    auto reorder_src_memory_p = reorder_handler.AcquireSrcMemory(
-        x->mem_desc(), phi::funcs::to_void_cast(x->data<T>()));
-    auto reorder_dst_memory_p = reorder_handler.AcquireDstMemory(
-        out, x->mem_desc(), dev_ctx.GetPlace());
-
-    auto reorder_p = reorder_handler.AcquireReorder(
-        reorder_dst_memory_p, reorder_src_memory_p, attrs);
-
-    auto& astream = phi::OneDNNContext::tls().get_stream();
-    reorder_p->execute(astream, *reorder_src_memory_p, *reorder_dst_memory_p);
-    astream.wait();
-
-    out->set_mem_desc(reorder_dst_memory_p->get_desc());
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-
-REGISTER_OP_KERNEL(dequantize,
-                   MKLDNN,
-                   ::phi::CPUPlace,
-                   ops::DeQuantOpKernel<uint8_t>,
-                   ops::DeQuantOpKernel<int8_t>,
-                   ops::DeQuantOpKernel<paddle::platform::bfloat16>);
diff --git a/paddle/phi/kernels/dequantize_kernel.h b/paddle/phi/kernels/dequantize_kernel.h
new file mode 100644
index 0000000000000..62f76bba897ec
--- /dev/null
+++ b/paddle/phi/kernels/dequantize_kernel.h
@@ -0,0 +1,28 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void DeQuantKernel(const Context& dev_ctx,
+                   const DenseTensor& x,
+                   const float quantization_scale,
+                   const float quantization_shift,
+                   DenseTensor* out);
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/onednn/dequantize_kernel.cc b/paddle/phi/kernels/onednn/dequantize_kernel.cc
new file mode 100644
index 0000000000000..9c04ec528f6c5
--- /dev/null
+++ b/paddle/phi/kernels/onednn/dequantize_kernel.cc
@@ -0,0 +1,86 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/dequantize_kernel.h"
+
+#include "paddle/phi/backends/onednn/onednn_context.h"
+#include "paddle/phi/backends/onednn/onednn_helper.h"
+#include "paddle/phi/backends/onednn/onednn_reuse.h"
+#include "paddle/phi/core/enforce.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void DeQuantKernel(const Context& dev_ctx,
+                   const DenseTensor& x,
+                   const float quantization_scale,
+                   const float quantization_shift,
+                   DenseTensor* out) {
+  const bool with_shift = quantization_shift != 0.0f;
+
+  PADDLE_ENFORCE(quantization_scale != 0.0f,
+                 phi::errors::InvalidArgument(
+                     "Dequantization scale must be different than 0.0f"));
+
+  PADDLE_ENFORCE(quantization_shift <= 255 && quantization_shift >= 0,
+                 phi::errors::InvalidArgument(
+                     "Dequantization shift must be lower or equal to ",
+                     "255 and greater or equal to 0, but got %f",
+                     quantization_shift));
+
+  auto x_tz = phi::vectorize<int64_t>(x.dims());
+  auto x_type = phi::funcs::ToOneDNNDataType(x.dtype());
+  auto out_type = phi::funcs::ToOneDNNDataType(out->dtype());
+
+  dnnl::primitive_attr attrs;
+  static constexpr int32_t mask = 0;  // same shift and scale for whole tensor
+
+  const float reorder_scale = 1. / quantization_scale;
+  attrs.set_output_scales(mask, {reorder_scale});
+
+  if (with_shift) {
+    attrs.set_zero_points(
+        DNNL_ARG_SRC, mask, {static_cast<int32_t>(quantization_shift)});
+  }
+
+  phi::funcs::ReorderOneDNNHandler reorder_handler(
+      x_tz, x.dtype(), x_type, out->dtype(), out_type, dev_ctx.GetEngine());
+
+  auto reorder_src_memory_p = reorder_handler.AcquireSrcMemory(
+      x.mem_desc(), phi::funcs::to_void_cast(x.data<T>()));
+  auto reorder_dst_memory_p =
+      reorder_handler.AcquireDstMemory(out, x.mem_desc(), dev_ctx.GetPlace());
+
+  auto reorder_p = reorder_handler.AcquireReorder(
+      reorder_dst_memory_p, reorder_src_memory_p, attrs);
+
+  auto& astream = phi::OneDNNContext::tls().get_stream();
+  reorder_p->execute(astream, *reorder_src_memory_p, *reorder_dst_memory_p);
+  astream.wait();
+
+  out->set_mem_desc(reorder_dst_memory_p->get_desc());
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(dequantize,
+                   OneDNN,
+                   ONEDNN,
+                   phi::DeQuantKernel,
+                   uint8_t,
+                   int8_t,
+                   phi::dtype::bfloat16) {
+  kernel->OutputAt(0).SetDataType(phi::DataType::FLOAT32);
+}
diff --git a/paddle/phi/ops/compat/dequantize_sig.cc b/paddle/phi/ops/compat/dequantize_sig.cc
new file mode 100644
index 0000000000000..18c76541033d8
--- /dev/null
+++ b/paddle/phi/ops/compat/dequantize_sig.cc
@@ -0,0 +1,26 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/core/compat/op_utils.h"
+
+namespace phi {
+
+KernelSignature DeQuantOpArgumentMapping(const ArgumentMappingContext& ctx) {
+  return KernelSignature(
+      "dequantize", {"Input"}, {"Scale", "Shift"}, {"Output"});
+}
+
+}  // namespace phi
+
+PD_REGISTER_ARG_MAPPING_FN(dequantize, phi::DeQuantOpArgumentMapping);

From 848deecde80134d5f3dc17b19400d07bbbfe691d Mon Sep 17 00:00:00 2001
From: Shaojie WANG <wsjmessi@163.com>
Date: Mon, 15 May 2023 06:44:02 -0700
Subject: [PATCH 338/405] [AMP]fix embedding model weight type mismatch error
 (#53770)

* fix embedding model weight type mismatch error

* Update fp16_utils.py

---------

Co-authored-by: Zhang Ting <zhangting_2017@163.com>
---
 python/paddle/static/amp/fp16_utils.py  |  22 +++-
 test/amp/test_amp_o2_embedding_model.py | 146 ++++++++++++++++++++++++
 2 files changed, 164 insertions(+), 4 deletions(-)
 create mode 100644 test/amp/test_amp_o2_embedding_model.py

diff --git a/python/paddle/static/amp/fp16_utils.py b/python/paddle/static/amp/fp16_utils.py
index 333d652f9f59d..0ba4db12a78fe 100644
--- a/python/paddle/static/amp/fp16_utils.py
+++ b/python/paddle/static/amp/fp16_utils.py
@@ -438,7 +438,7 @@ def set_param_dtype(program, dtype, amp_lists, use_fp16_guard, level):
             # input in transformer, so the weight is also in to_fp16_var_names.
             # TODO(zhangting2020): consider fix auto_parallel_fp16 and remove lookup_table
             # from black_list and unsupport_list.
-            if op in ['lookup_table', 'lookup_table_v2']:
+            if op.type in amp_lists.black_list:
                 continue
             if _need_keep_fp32(op, amp_lists.unsupported_list, use_fp16_guard):
                 for in_name in op.input_names:
@@ -461,8 +461,9 @@ def set_param_dtype(program, dtype, amp_lists, use_fp16_guard, level):
     return keep_fp32_var_names
 
 
-def op_need_keep_fp32(op, amp_lists, use_fp16_guard):
+def op_need_keep_fp32(op, amp_lists, use_fp16_guard, params_list):
     need_keep_fp32 = False
+    fp16_varname_list_in_fp32_op = set()
     if _need_keep_fp32(
         op,
         amp_lists.unsupported_list,
@@ -475,8 +476,14 @@ def op_need_keep_fp32(op, amp_lists, use_fp16_guard):
         need_keep_fp32 = True
     elif op.type in amp_lists.black_list:
         need_keep_fp32 = True
+        for in_name in op.input_names:
+            for params in params_list:
+                if op.input(in_name)[0] == params.name:
+                    fp16_varname_list_in_fp32_op = (
+                        fp16_varname_list_in_fp32_op.union(op.input(in_name))
+                    )
 
-    return need_keep_fp32
+    return need_keep_fp32, fp16_varname_list_in_fp32_op
 
 
 def get_promote_dtype(op, amp_dtype, block):
@@ -651,7 +658,14 @@ def need_process(op):
             if not need_process(op):
                 _logger.debug("---- The op does not need to be processed ----.")
                 continue
-            if op_need_keep_fp32(op, amp_lists, use_fp16_guard):
+            all_params = global_block.all_parameters()
+            op_keep_fp32, fp16_var_names_in_fp32_op = op_need_keep_fp32(
+                op, amp_lists, use_fp16_guard, all_params
+            )
+            to_fp16_var_names = to_fp16_var_names.union(
+                fp16_var_names_in_fp32_op
+            )
+            if op_keep_fp32:
                 keep_fp32_ops.add(op)
                 process_op_input_and_outputs(
                     op, block, global_block, core.VarDesc.VarType.FP32
diff --git a/test/amp/test_amp_o2_embedding_model.py b/test/amp/test_amp_o2_embedding_model.py
new file mode 100644
index 0000000000000..b1af4bde8d3d8
--- /dev/null
+++ b/test/amp/test_amp_o2_embedding_model.py
@@ -0,0 +1,146 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import random
+import unittest
+
+import numpy as np
+from amp_base_models import AmpTestBase, _build_optimizer
+
+import paddle
+from paddle import nn
+
+paddle.enable_static()
+
+_fixed_param = np.random.random(size=[64, 64]).astype("float32")
+
+
+class SimpleUnittedEmbeddingNet(nn.Layer):
+    def __init__(self):
+        super().__init__()
+        self.vocab_size = 64
+        self.hidden_size = 64
+        global _fixed_param
+
+        self.param_attr = paddle.ParamAttr(
+            initializer=paddle.nn.initializer.Assign(_fixed_param)
+        )
+        self.embedding = nn.Embedding(
+            self.vocab_size, self.hidden_size, weight_attr=self.param_attr
+        )
+        self.linear = nn.Linear(
+            in_features=self.hidden_size,
+            out_features=self.vocab_size,
+            weight_attr=self.param_attr,
+        )
+
+    def forward(self, x):
+        out = self.embedding(x)
+        scale = paddle.full(shape=[1], fill_value=2, dtype="int64")
+        out = paddle.multiply(out, scale.astype("float32"))
+        out = self.linear(out)
+        out = nn.functional.dropout(out, p=0.2)
+        return out
+
+
+def build_unitted_embedding_model(
+    use_amp,
+    amp_dtype="float16",
+    amp_level="O1",
+    use_promote=False,
+):
+    main_program = paddle.static.Program()
+    startup_program = paddle.static.Program()
+    with paddle.utils.unique_name.guard():
+        with paddle.static.program_guard(main_program, startup_program):
+            model = SimpleUnittedEmbeddingNet()
+            x = paddle.static.data(name='x', shape=[None, 32], dtype='int64')
+            out = model(x)
+            loss = paddle.mean(out)
+            if use_amp:
+                amp_lists = paddle.static.amp.AutoMixedPrecisionLists(
+                    custom_white_list=["elementwise_mul"],
+                    custom_black_list=["reduce_mean"],
+                    dtype=amp_dtype,
+                )
+            else:
+                amp_lists = None
+            optimizer = _build_optimizer(
+                use_amp,
+                amp_dtype,
+                amp_level,
+                amp_lists,
+                True,
+                use_promote=use_promote,
+            )
+            optimizer.minimize(loss)
+
+    feed_vars = [x]
+    fetch_vars = [loss]
+    return main_program, startup_program, optimizer, feed_vars, fetch_vars
+
+
+class TestUnittedEmbedding(AmpTestBase):
+    def _generate_feed_x(self):
+        seed = 0
+        paddle.seed(seed)
+        np.random.seed(seed)
+        random.seed(seed)
+
+        x = np.random.randint(1, 64, size=[1, 32]).astype("int64")
+        return x
+
+    def test_compare_o1_and_o2_master_grad(self):
+        def _run(place, exe, x_np, max_iters, level):
+            (
+                main_program,
+                startup_program,
+                optimizer,
+                feed_vars,
+                fetch_vars,
+            ) = build_unitted_embedding_model(
+                True,
+                "float16",
+                level,
+            )
+
+            seed = 0
+            paddle.seed(seed)
+            np.random.seed(seed)
+            random.seed(seed)
+
+            losses = self.run_program(
+                main_program,
+                startup_program,
+                optimizer,
+                feed_vars,
+                fetch_vars,
+                place,
+                exe,
+                x_np,
+                max_iters,
+                level,
+            )
+            return losses
+
+        max_iters = 5
+        x = self._generate_feed_x()
+        place = paddle.CUDAPlace(0)
+        exe = paddle.static.Executor(place)
+        losses_o2 = _run(place, exe, x, max_iters, 'O2')
+
+
+if __name__ == "__main__":
+    unittest.main()

From 2174e91c1ef1443813f51ae8058d4036d36e14d8 Mon Sep 17 00:00:00 2001
From: RuohengMa <120699764+RuohengMa@users.noreply.github.com>
Date: Tue, 16 May 2023 10:28:17 +0800
Subject: [PATCH 339/405] [PHI] fix duplicate instantiation of reduce_any when
 compiling with XPU_KP (#53806)

---
 paddle/phi/kernels/reduce_any_kernel.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/paddle/phi/kernels/reduce_any_kernel.cc b/paddle/phi/kernels/reduce_any_kernel.cc
index adb2f8168bc19..9d162f8e02033 100644
--- a/paddle/phi/kernels/reduce_any_kernel.cc
+++ b/paddle/phi/kernels/reduce_any_kernel.cc
@@ -37,7 +37,7 @@ PD_REGISTER_KERNEL(any, CPU, ALL_LAYOUT, phi::AnyKernel, bool) {}
 PD_REGISTER_KERNEL(any, GPU, ALL_LAYOUT, phi::AnyKernel, bool) {}
 #endif
 
-#if defined(PADDLE_WITH_XPU_KP)
+#if defined(PADDLE_WITH_XPU_KP) && !defined(PADDLE_WITH_XPU)
 PD_REGISTER_KERNEL(any, KPS, ALL_LAYOUT, phi::AnyKernel, bool) {}
 #endif
 

From 434343c657148529e1c12d4f75d841650933a30a Mon Sep 17 00:00:00 2001
From: andyj <87074272+andyjpaddle@users.noreply.github.com>
Date: Tue, 16 May 2023 10:59:42 +0800
Subject: [PATCH 340/405] fix pinv api for divide zero (#53815)

---
 .../tests/unittests/test_linalg_pinv_op.py    | 13 ++++++++
 python/paddle/tensor/linalg.py                | 30 +++----------------
 2 files changed, 17 insertions(+), 26 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/test_linalg_pinv_op.py b/python/paddle/fluid/tests/unittests/test_linalg_pinv_op.py
index dbd857d5a6cb8..8cf39061e7cb8 100644
--- a/python/paddle/fluid/tests/unittests/test_linalg_pinv_op.py
+++ b/python/paddle/fluid/tests/unittests/test_linalg_pinv_op.py
@@ -267,6 +267,19 @@ def init_config(self):
         self.hermitian = True
 
 
+class LinalgPinvTestCaseHermitian6(LinalgPinvTestCase):
+    def generate_input(self):
+        self._input_shape = (3, 5, 5)
+        np.random.seed(123)
+        x = np.ones(self._input_shape).astype(self.dtype)
+        self._input_data = x + 0.01
+
+    def init_config(self):
+        self.dtype = 'float64'
+        self.rcond = 1e-15
+        self.hermitian = True
+
+
 class LinalgPinvTestCaseHermitianFP32(LinalgPinvTestCase):
     def generate_input(self):
         self._input_shape = (3, 5, 5)
diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py
index 2235cf93cfb60..3aebc68dc61bf 100644
--- a/python/paddle/tensor/linalg.py
+++ b/python/paddle/tensor/linalg.py
@@ -26,9 +26,7 @@
 )
 from ..framework import LayerHelper, in_dygraph_mode
 from .creation import full
-from .logic import logical_not
 from .manipulation import cast
-from .math import add, multiply
 
 __all__ = []
 
@@ -2665,12 +2663,7 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None):
             y = float('inf')
             y = paddle.to_tensor(y, dtype=x.dtype)
 
-            condition = s > cutoff
-            cond_int = cast(condition, s.dtype)
-            cond_not_int = cast(logical_not(condition), s.dtype)
-            out1 = multiply(1 / s, cond_int)
-            out2 = multiply(1 / y, cond_not_int)
-            singular = add(out1, out2)
+            singular = paddle.where(s > cutoff, 1 / s, 1 / y)
             st = _C_ops.unsqueeze(singular, [-2])
 
             dims = list(range(len(vt.shape)))
@@ -2690,12 +2683,7 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None):
             y = float('inf')
             y = paddle.to_tensor(y, dtype=s.dtype)
 
-            condition = s_abs > cutoff
-            cond_int = cast(condition, s.dtype)
-            cond_not_int = cast(logical_not(condition), s.dtype)
-            out1 = multiply(1 / s, cond_int)
-            out2 = multiply(1 / y, cond_not_int)
-            singular = add(out1, out2)
+            singular = paddle.where(s_abs > cutoff, 1 / s, 1 / y)
             st = _C_ops.unsqueeze(singular, [-2])
 
             out_1 = u * st
@@ -2731,12 +2719,7 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None):
             y = float('inf')
             y = full(shape=[1], fill_value=y, dtype=dtype)
 
-            condition = s > cutoff
-            cond_int = cast(condition, dtype)
-            cond_not_int = cast(logical_not(condition), dtype)
-            out1 = multiply(1 / s, cond_int)
-            out2 = multiply(1 / y, cond_not_int)
-            singular = add(out1, out2)
+            singular = paddle.where(s > cutoff, 1 / s, 1 / y)
 
             st = helper.create_variable_for_type_inference(dtype=dtype)
             st_shape = helper.create_variable_for_type_inference(dtype=dtype)
@@ -2817,12 +2800,7 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None):
             y = float('inf')
             y = full(shape=[1], fill_value=y, dtype=s_type)
 
-            condition = s_abs > cutoff
-            cond_int = cast(condition, s_type)
-            cond_not_int = cast(logical_not(condition), s_type)
-            out1 = multiply(1 / s, cond_int)
-            out2 = multiply(1 / y, cond_not_int)
-            singular = add(out1, out2)
+            singular = paddle.where(s_abs > cutoff, 1 / s, 1 / y)
 
             st = helper.create_variable_for_type_inference(dtype=s_type)
             st_shape = helper.create_variable_for_type_inference(dtype=s_type)

From 926b886192c139b8fd2750cd9ad831e2d83f341a Mon Sep 17 00:00:00 2001
From: ronnywang <ronny1996@163.com>
Date: Tue, 16 May 2023 11:09:50 +0800
Subject: [PATCH 341/405] [CustomDevice] fix BatchNorm (#53820)

---
 python/paddle/nn/layer/norm.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/python/paddle/nn/layer/norm.py b/python/paddle/nn/layer/norm.py
index 9facd8e917273..eef98d96632fa 100644
--- a/python/paddle/nn/layer/norm.py
+++ b/python/paddle/nn/layer/norm.py
@@ -975,6 +975,29 @@ def __init__(
         )
         self._variance.stop_gradient = True
 
+        # TODO(qili93): temporary for ascned npu performance to be removed along with npu_identity op
+        if (
+            _global_flags()['FLAGS_npu_storage_format']
+            and 'npu' in get_all_custom_device_type()
+        ):
+            with no_grad():
+                weight_trans = _C_ops.npu_identity(
+                    self.weight, 3
+                )  # ACL_FORMAT_NC1HWC0 = 3
+                bias_trans = _C_ops.npu_identity(
+                    self.bias, 3
+                )  # ACL_FORMAT_NC1HWC0 = 3
+                mean_trans = _C_ops.npu_identity(
+                    self._mean, 3
+                )  # ACL_FORMAT_NC1HWC0 = 3
+                var_trans = _C_ops.npu_identity(
+                    self._variance, 3
+                )  # ACL_FORMAT_NC1HWC0 = 3
+                weight_trans._share_underline_tensor_to(self.weight)
+                bias_trans._share_underline_tensor_to(self.bias)
+                mean_trans._share_underline_tensor_to(self._mean)
+                var_trans._share_underline_tensor_to(self._variance)
+
         self._in_place = in_place
         self._data_layout = data_layout
         self._momentum = momentum

From 847c48a82dbbba8e1ddd43f16deb9c464d645de9 Mon Sep 17 00:00:00 2001
From: Mahmoud Ashraf <hassouna97.ma@gmail.com>
Date: Tue, 16 May 2023 06:13:57 +0300
Subject: [PATCH 342/405] fix simple typos (#53783)

* correct 1th to 1st

* correct 1th to 1st

* fix typo

* fix typos
---
 .../framework/ir/seq_concat_fc_fuse_pass.cc    |  4 ++--
 python/paddle/distributed/utils/moe_utils.py   | 18 +++++++++---------
 .../tests/unittests/test_sparse_utils_op.py    |  4 ++--
 tools/CrossStackProfiler/DCGMFileReader.py     |  2 +-
 4 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/paddle/fluid/framework/ir/seq_concat_fc_fuse_pass.cc b/paddle/fluid/framework/ir/seq_concat_fc_fuse_pass.cc
index 95dc32d20b094..070f6a51e3023 100644
--- a/paddle/fluid/framework/ir/seq_concat_fc_fuse_pass.cc
+++ b/paddle/fluid/framework/ir/seq_concat_fc_fuse_pass.cc
@@ -32,8 +32,8 @@ PDNode* BuildSeqExpandConcatPattern(PDPattern* pattern) {
 
   // The following variables will be treat as inputs:
   // concat mid input, 0th input for fused op
-  // sequence_expand input, 1th input for fused op
-  // sequence_expand input, 2th input for fused op
+  // sequence_expand input, 1st input for fused op
+  // sequence_expand input, 2nd input for fused op
 
   // The following variables will be treat as outputs:
   // concat output
diff --git a/python/paddle/distributed/utils/moe_utils.py b/python/paddle/distributed/utils/moe_utils.py
index ae18938941817..fa003c1a00d7c 100644
--- a/python/paddle/distributed/utils/moe_utils.py
+++ b/python/paddle/distributed/utils/moe_utils.py
@@ -35,19 +35,19 @@ def global_scatter(
 
     local_count[0] represents taking out 2 batches from x and sending 2 batches to the 0th expert of the 0th card;
 
-    local_count[1] represents taking out 0 batches from x and sending 0 batches to the 1th expert of the 0th card;
+    local_count[1] represents taking out 0 batches from x and sending 0 batches to the 1st expert of the 0th card;
 
-    local_count[2] represents taking out 2 batches from x and sending 2 batches to the 0th expert of the 1th card;
+    local_count[2] represents taking out 2 batches from x and sending 2 batches to the 0th expert of the 1st card;
 
-    local_count[3] represents taking out 0 batches from x and sending 0 batches to the 1th expert of the 1th card;
+    local_count[3] represents taking out 0 batches from x and sending 0 batches to the 1st expert of the 1st card;
 
     Therefore, the global_count[0] of the 0th card is equal to 2, which means that 2 batches of data are received from the 0th card to the 0th expert;
 
-    the global_count[1] of the 0th card is equal to 0, which means that 0 batches of data are received from the 0th card to the 1th expert;
+    the global_count[1] of the 0th card is equal to 0, which means that 0 batches of data are received from the 0th card to the 1st expert;
 
-    the global_count[0] of the 1th card is equal to 2, which means that 2 batches of data are received from the 0th card to the 0th expert;
+    the global_count[0] of the 1st card is equal to 2, which means that 2 batches of data are received from the 0th card to the 0th expert;
 
-    the global_count[1] of the 1th card is equal to 0, which means that 0 batches of data are received from the 0th card to the 1th expert.
+    the global_count[1] of the 1st card is equal to 0, which means that 0 batches of data are received from the 0th card to the 1st expert.
 
     .. image:: https://githubraw.cdn.bcebos.com/PaddlePaddle/docs/develop/docs/api/paddle/distributed/img/global_scatter_gather.png
         :width: 800
@@ -160,11 +160,11 @@ def global_gather(
 
     The global_count[0] of the 0th card represents sending 2 data to the 0th expert of the 0th card;
 
-    The global_count[1] of the 0th card represents sending 0 data to the 1th expert of the 0th card;
+    The global_count[1] of the 0th card represents sending 0 data to the 1st expert of the 0th card;
 
-    The global_count[0] of the 1th card represents sending 2 data to the 0th expert of the 0th card;
+    The global_count[0] of the 1st card represents sending 2 data to the 0th expert of the 0th card;
 
-    The global_count[1] of the 1th card represents sending 0 data to the 1th expert of the 0th card.
+    The global_count[1] of the 1st card represents sending 0 data to the 1st expert of the 0th card.
 
     .. image:: https://githubraw.cdn.bcebos.com/PaddlePaddle/docs/develop/docs/api/paddle/distributed/img/global_scatter_gather.png
         :width: 800
diff --git a/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py b/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py
index 3504af44e0009..f4d56d7e41ba4 100644
--- a/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py
+++ b/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py
@@ -342,12 +342,12 @@ def verify(dense_x):
         verify(dense_x)
 
         dense_x = paddle.randn(shape)
-        # set the 1th batch to zero
+        # set the 1st batch to zero
         dense_x[1] = 0
         verify(dense_x)
 
         dense_x = paddle.randn(shape)
-        # set the 2th batch to zero
+        # set the 2nd batch to zero
         dense_x[2] = 0
         verify(dense_x)
 
diff --git a/tools/CrossStackProfiler/DCGMFileReader.py b/tools/CrossStackProfiler/DCGMFileReader.py
index bb248e05868cc..f462ce5c9ad5e 100755
--- a/tools/CrossStackProfiler/DCGMFileReader.py
+++ b/tools/CrossStackProfiler/DCGMFileReader.py
@@ -136,7 +136,7 @@ def _parseSingleFile(self, fileName):
                 ) and not line.strip().startswith("# Entity"):
                     continue
 
-                # skip non-needed headers (only the header in 1th line was needed)
+                # skip non-needed headers (only the header in 1st line was needed)
                 if line.strip().startswith("# Entity"):
                     line = line.strip()[2:]
 

From 4f7dfd02e2568e839c4204fbc8cd03156f59f10a Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Tue, 16 May 2023 11:20:25 +0800
Subject: [PATCH 343/405] Fix typos, test=document_fix (#53803)

---
 paddle/phi/core/allocator.h     |  2 +-
 paddle/phi/core/dense_tensor.cc |  2 +-
 paddle/phi/core/enforce.h       |  2 +-
 paddle/phi/core/errors.h        |  2 +-
 paddle/phi/core/flags.cc        | 12 ++++++------
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/paddle/phi/core/allocator.h b/paddle/phi/core/allocator.h
index 849fc1548c7ec..1d89fd1b4aa88 100644
--- a/paddle/phi/core/allocator.h
+++ b/paddle/phi/core/allocator.h
@@ -70,7 +70,7 @@ class Allocation {
   //    the AlignedAllocator will always allocate memory as size + kAlignment.
   //    The raw pointer might not aligned, so an offset might be added to raw
   //    the pointer. The size of this allocation will be
-  //    `size + kAlignemnt - offset`.
+  //    `size + kAlignment - offset`.
   size_t size() const noexcept { return size_; }
 
   void* operator->() const noexcept { return ptr_; }
diff --git a/paddle/phi/core/dense_tensor.cc b/paddle/phi/core/dense_tensor.cc
index 3116093a7884d..2c8f36f6c34ae 100644
--- a/paddle/phi/core/dense_tensor.cc
+++ b/paddle/phi/core/dense_tensor.cc
@@ -106,7 +106,7 @@ void* DenseTensor::AllocateFrom(Allocator* allocator,
       phi::errors::InvalidArgument(
           "Required allocator shall not be nullptr, but received nullptr."));
   if (this->dtype() != dtype) {
-    VLOG(10) << "change data type in mutbale_data, target dtype - " << dtype;
+    VLOG(10) << "change data type in mutable_data, target dtype - " << dtype;
     meta_.dtype = dtype;
   }
 
diff --git a/paddle/phi/core/enforce.h b/paddle/phi/core/enforce.h
index 3f17b34b2bc60..6b98fd0488595 100644
--- a/paddle/phi/core/enforce.h
+++ b/paddle/phi/core/enforce.h
@@ -351,7 +351,7 @@ struct EnforceNotMet : public std::exception {
   // Complete error message
   // e.g. InvalidArgumentError: ***
   std::string err_str_;
-  // Simple errror message used when no C++ stack and python compile stack
+  // Simple error message used when no C++ stack and python compile stack
   // e.g. (InvalidArgument) ***
   std::string simple_err_str_;
 };
diff --git a/paddle/phi/core/errors.h b/paddle/phi/core/errors.h
index d1365de519625..b66cd5fd18f0a 100644
--- a/paddle/phi/core/errors.h
+++ b/paddle/phi/core/errors.h
@@ -66,7 +66,7 @@ enum ErrorCode {
   EXECUTION_TIMEOUT = 8,
 
   // Operation is not implemented or not supported/enabled in this service.
-  // Error type string: "UnimpelmentedError"
+  // Error type string: "UnimplementedError"
   UNIMPLEMENTED = 9,
 
   // The service is currently unavailable.  This is a most likely a
diff --git a/paddle/phi/core/flags.cc b/paddle/phi/core/flags.cc
index 59c2901d8f4f1..5d62c6733516f 100644
--- a/paddle/phi/core/flags.cc
+++ b/paddle/phi/core/flags.cc
@@ -1050,10 +1050,10 @@ DEFINE_int32(record_pool_max_size,
 DEFINE_int32(slotpool_thread_num, 1, "SlotRecordDataset slot pool thread num");
 DEFINE_bool(enable_slotpool_wait_release,
             false,
-            "enable slotrecord obejct wait release, default false");
+            "enable slotrecord object wait release, default false");
 DEFINE_bool(enable_slotrecord_reset_shrink,
             false,
-            "enable slotrecord obejct reset shrink memory, default false");
+            "enable slotrecord object reset shrink memory, default false");
 DEFINE_bool(enable_ins_parser_file,
             false,
             "enable parser ins file, default false");
@@ -1067,7 +1067,7 @@ PHI_DEFINE_EXPORTED_double(gpugraph_hbm_table_load_factor,
 PHI_DEFINE_EXPORTED_bool(
     gpugraph_enable_gpu_direct_access,
     false,
-    "enable direct access bwtween multi gpu cards, default false");
+    "enable direct access between multi gpu cards, default false");
 PHI_DEFINE_EXPORTED_bool(
     gpugraph_enable_segment_merge_grads,
     false,
@@ -1140,12 +1140,12 @@ PHI_DEFINE_EXPORTED_int32(search_cache_max_number,
                           "search_cache_max_number.");
 
 /**
- * Preformance related FLAG
+ * Performance related FLAG
  * Name: einsum_opt
  * Since Version: 2.3.0
  * Value Range: bool, default=false
  * Example:
- * Note: If True, EinsumOp will be optimimzed by innercache reuse, which
+ * Note: If True, EinsumOp will be optimized by innercache reuse, which
  * uses more gpu memory.
  */
 PHI_DEFINE_EXPORTED_bool(
@@ -1166,7 +1166,7 @@ PHI_DEFINE_EXPORTED_bool(
  */
 PHI_DEFINE_EXPORTED_string(jit_engine_type,
                            "Predictor",
-                           "Choose default funciton type in JitLayer.");
+                           "Choose default function type in JitLayer.");
 
 /**
  * Custom Device NPU related FLAG

From 79c84baff53e4804aa00f0e87fe5ea2354de83ec Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Tue, 16 May 2023 11:25:57 +0800
Subject: [PATCH 344/405] Fix typos in paddle/phi/core/generator.cc (#53802)

---
 paddle/phi/core/generator.cc | 4 ++--
 paddle/phi/core/generator.h  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/paddle/phi/core/generator.cc b/paddle/phi/core/generator.cc
index 64bff05654d73..4ed25af0814df 100644
--- a/paddle/phi/core/generator.cc
+++ b/paddle/phi/core/generator.cc
@@ -277,11 +277,11 @@ uint64_t Generator::Random64() {
 }
 
 std::pair<uint64_t, uint64_t> Generator::IncrementOffset(
-    uint64_t increament_offset) {
+    uint64_t increment_offset) {
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
   std::lock_guard<std::mutex> lock(this->mu_);
   uint64_t cur_offset = this->state_.thread_offset;
-  this->state_.thread_offset += increament_offset;
+  this->state_.thread_offset += increment_offset;
   return std::make_pair(this->state_.current_seed, cur_offset);
 #else
   PADDLE_THROW(phi::errors::PermissionDenied(
diff --git a/paddle/phi/core/generator.h b/paddle/phi/core/generator.h
index a1d985dc772e6..aeaa89f2abf5e 100644
--- a/paddle/phi/core/generator.h
+++ b/paddle/phi/core/generator.h
@@ -65,7 +65,7 @@ class Generator {
 
   uint64_t Random64();
 
-  std::pair<uint64_t, uint64_t> IncrementOffset(uint64_t increament_offset);
+  std::pair<uint64_t, uint64_t> IncrementOffset(uint64_t increment_offset);
 
   uint64_t get_device_id() { return this->state_.device; }
 

From 00c21abcd2d8f9bd642116f68309b0388c481084 Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Tue, 16 May 2023 11:37:47 +0800
Subject: [PATCH 345/405] [phi] move stft to phi - Step 1 (#53517)

* [phi]mv StftKernel to phi

* [phi] fix KernelSignature

* [phi]fix arr error

* [phi] Disable check_dygraph

* [phi]fix include

* [phi] rewrite mutable_data, add output register

* [phi] fix  Alloc

* [phi] fix Alloc again

* [phi] fix mutable_data

* [phi] fix onesided_out Resize
---
 paddle/fluid/operators/stft_op.cc             |  4 -
 paddle/fluid/operators/stft_op.cu             |  4 -
 paddle/fluid/operators/stft_op.h              | 73 --------------
 paddle/phi/kernels/cpu/stft_kernel.cc         | 28 ++++++
 paddle/phi/kernels/gpu/stft_kernel.cu         | 20 ++++
 paddle/phi/kernels/impl/stft_kernel_impl.h    | 99 +++++++++++++++++++
 paddle/phi/kernels/stft_kernel.h              | 27 +++++
 paddle/phi/ops/compat/stft_sig.cc             | 28 ++++++
 .../fluid/tests/unittests/test_stft_op.py     |  2 +-
 9 files changed, 203 insertions(+), 82 deletions(-)
 create mode 100644 paddle/phi/kernels/cpu/stft_kernel.cc
 create mode 100644 paddle/phi/kernels/gpu/stft_kernel.cu
 create mode 100644 paddle/phi/kernels/impl/stft_kernel_impl.h
 create mode 100644 paddle/phi/kernels/stft_kernel.h
 create mode 100644 paddle/phi/ops/compat/stft_sig.cc

diff --git a/paddle/fluid/operators/stft_op.cc b/paddle/fluid/operators/stft_op.cc
index 8c9507bc89ced..f85b856098dc5 100644
--- a/paddle/fluid/operators/stft_op.cc
+++ b/paddle/fluid/operators/stft_op.cc
@@ -162,10 +162,6 @@ REGISTER_OPERATOR(stft,
 
 REGISTER_OPERATOR(stft_grad, ops::StftGradOp);
 
-REGISTER_OP_CPU_KERNEL(stft,
-                       ops::StftKernel<phi::CPUContext, float>,
-                       ops::StftKernel<phi::CPUContext, double>);
-
 REGISTER_OP_CPU_KERNEL(stft_grad,
                        ops::StftGradKernel<phi::CPUContext, float>,
                        ops::StftGradKernel<phi::CPUContext, double>);
diff --git a/paddle/fluid/operators/stft_op.cu b/paddle/fluid/operators/stft_op.cu
index bd48112f0737e..98737cd08a39f 100644
--- a/paddle/fluid/operators/stft_op.cu
+++ b/paddle/fluid/operators/stft_op.cu
@@ -16,10 +16,6 @@
 
 namespace ops = paddle::operators;
 
-REGISTER_OP_CUDA_KERNEL(stft,
-                        ops::StftKernel<phi::GPUContext, float>,
-                        ops::StftKernel<phi::GPUContext, double>);
-
 REGISTER_OP_CUDA_KERNEL(stft_grad,
                         ops::StftGradKernel<phi::GPUContext, float>,
                         ops::StftGradKernel<phi::GPUContext, double>);
diff --git a/paddle/fluid/operators/stft_op.h b/paddle/fluid/operators/stft_op.h
index b0d6091ecd37b..92bb970809f7e 100644
--- a/paddle/fluid/operators/stft_op.h
+++ b/paddle/fluid/operators/stft_op.h
@@ -27,79 +27,6 @@
 namespace paddle {
 namespace operators {
 
-template <typename DeviceContext, typename T>
-class StftKernel : public framework::OpKernel<T> {
- public:
-  /*
-    Batch Signals (N, T) -> Frames (N, n_fft, num_frames) -> FFTR2C -> (N,
-    n_fft/2 + 1, num_frames) or (N, n_fft, num_frames)
-  */
-  void Compute(const framework::ExecutionContext& ctx) const override {
-    using C = paddle::platform::complex<T>;
-    const phi::DenseTensor* x = ctx.Input<phi::DenseTensor>("X");
-    const phi::DenseTensor* window = ctx.Input<phi::DenseTensor>("Window");
-    phi::DenseTensor* out = ctx.Output<phi::DenseTensor>("Out");
-    out->mutable_data<C>(ctx.GetPlace());
-
-    const size_t x_rank = x->dims().size();
-    const size_t out_rank = out->dims().size();
-
-    const int n_fft = ctx.Attr<int>("n_fft");
-    const int hop_length = ctx.Attr<int>("hop_length");
-    const bool normalized = ctx.Attr<bool>("normalized");
-    const bool onesided = ctx.Attr<bool>("onesided");
-
-    const int n_frames = out->dims()[out_rank - 1];
-    const int seq_length = x->dims()[x_rank - 1];
-
-    auto& dev_ctx = ctx.device_context<DeviceContext>();
-
-    std::vector<int64_t> axes = {1};
-
-    // Frame
-    phi::DenseTensor frames;
-    framework::DDim frames_dims(out->dims());
-    frames_dims.at(axes.back()) = n_fft;
-    frames.mutable_data<T>(frames_dims, ctx.GetPlace());
-    phi::funcs::FrameFunctor<DeviceContext, T>()(dev_ctx,
-                                                 x,
-                                                 &frames,
-                                                 seq_length,
-                                                 n_fft,
-                                                 n_frames,
-                                                 hop_length,
-                                                 /*is_grad*/ false);
-
-    // Window
-    phi::DenseTensor frames_w;
-    frames_w.mutable_data<T>(frames_dims, ctx.GetPlace());
-    ElementwiseComputeEx<MulFunctor<T>, DeviceContext, T>(
-        ctx, &frames, window, axes.back(), MulFunctor<T>(), &frames_w);
-
-    // FFTR2C
-    phi::funcs::FFTNormMode normalization;
-    if (normalized) {
-      normalization = phi::funcs::get_norm_from_string("ortho", true);
-    } else {
-      normalization = phi::funcs::get_norm_from_string("backward", true);
-    }
-    phi::funcs::FFTR2CFunctor<DeviceContext, T, C> fft_r2c_func;
-
-    if (onesided) {
-      fft_r2c_func(dev_ctx, frames_w, out, axes, normalization, true);
-    } else {
-      framework::DDim onesided_dims(out->dims());
-      const int64_t onesided_axis_size = out->dims().at(axes.back()) / 2 + 1;
-      onesided_dims.at(axes.back()) = onesided_axis_size;
-      phi::DenseTensor onesided_out;
-      onesided_out.mutable_data<C>(onesided_dims, ctx.GetPlace());
-      fft_r2c_func(dev_ctx, frames_w, &onesided_out, axes, normalization, true);
-      phi::funcs::FFTFillConj<DeviceContext, C>(
-          dev_ctx, &onesided_out, out, axes);
-    }
-  }
-};
-
 template <typename DeviceContext, typename T>
 class StftGradKernel : public framework::OpKernel<T> {
  public:
diff --git a/paddle/phi/kernels/cpu/stft_kernel.cc b/paddle/phi/kernels/cpu/stft_kernel.cc
new file mode 100644
index 0000000000000..2e790d9319571
--- /dev/null
+++ b/paddle/phi/kernels/cpu/stft_kernel.cc
@@ -0,0 +1,28 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/stft_kernel.h"
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/impl/stft_kernel_impl.h"
+
+PD_REGISTER_KERNEL(stft, CPU, ALL_LAYOUT, phi::StftKernel, float, double) {
+  if (kernel_key.dtype() == phi::DataType::FLOAT16 &&
+      kernel_key.dtype() == phi::DataType::FLOAT32 &&
+      kernel_key.dtype() == phi::DataType::FLOAT64) {
+    kernel->OutputAt(0).SetDataType(phi::DataType::COMPLEX64);
+  } else {
+    kernel->OutputAt(0).SetDataType(phi::DataType::COMPLEX128);
+  }
+}
diff --git a/paddle/phi/kernels/gpu/stft_kernel.cu b/paddle/phi/kernels/gpu/stft_kernel.cu
new file mode 100644
index 0000000000000..324883581610b
--- /dev/null
+++ b/paddle/phi/kernels/gpu/stft_kernel.cu
@@ -0,0 +1,20 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/stft_kernel.h"
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/impl/stft_kernel_impl.h"
+
+PD_REGISTER_KERNEL(stft, GPU, ALL_LAYOUT, phi::StftKernel, float, double) {}
diff --git a/paddle/phi/kernels/impl/stft_kernel_impl.h b/paddle/phi/kernels/impl/stft_kernel_impl.h
new file mode 100644
index 0000000000000..8d590de9d9767
--- /dev/null
+++ b/paddle/phi/kernels/impl/stft_kernel_impl.h
@@ -0,0 +1,99 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <vector>
+#include "paddle/phi/common/complex.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/kernels/cpu/elementwise.h"
+#include "paddle/phi/kernels/funcs/elementwise_base.h"
+#include "paddle/phi/kernels/funcs/elementwise_functor.h"
+#include "paddle/phi/kernels/funcs/fft.h"
+#include "paddle/phi/kernels/funcs/fft_fill_conj.h"
+#include "paddle/phi/kernels/funcs/frame_functor.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void StftKernel(const Context& ctx,
+                const DenseTensor& x,
+                const DenseTensor& window,
+                int n_fft,
+                int hop_length,
+                bool normalized,
+                bool onesided,
+                DenseTensor* out) {
+  using C = phi::dtype::complex<T>;
+
+  ctx.template Alloc<C>(out);
+
+  const size_t x_rank = x.dims().size();
+  const size_t out_rank = out->dims().size();
+
+  const int n_frames = out->dims()[out_rank - 1];
+  const int seq_length = x.dims()[x_rank - 1];
+
+  std::vector<int64_t> axes = {1};
+
+  // Frame
+  phi::DenseTensor frames;
+  phi::DDim frames_dims(out->dims());
+  frames_dims.at(axes.back()) = n_fft;
+  frames.Resize(frames_dims);
+  ctx.template Alloc<T>(&frames);
+  phi::funcs::FrameFunctor<Context, T>()(ctx,
+                                         &x,
+                                         &frames,
+                                         seq_length,
+                                         n_fft,
+                                         n_frames,
+                                         hop_length,
+                                         /*is_grad*/ false);
+
+  // Window
+  phi::DenseTensor frames_w;
+  frames_w.Resize(frames_dims);
+  ctx.template Alloc<T>(&frames_w);
+  phi::funcs::ElementwiseCompute<phi::funcs::MultiplyFunctor<T>, T, T>(
+      ctx,
+      frames,
+      window,
+      phi::funcs::MultiplyFunctor<T>(),
+      &frames_w,
+      axes.back());
+
+  // FFTR2C
+  phi::funcs::FFTNormMode normalization;
+  if (normalized) {
+    normalization = phi::funcs::get_norm_from_string("ortho", true);
+  } else {
+    normalization = phi::funcs::get_norm_from_string("backward", true);
+  }
+  phi::funcs::FFTR2CFunctor<Context, T, C> fft_r2c_func;
+
+  if (onesided) {
+    fft_r2c_func(ctx, frames_w, out, axes, normalization, true);
+  } else {
+    phi::DDim onesided_dims(out->dims());
+    const int64_t onesided_axis_size = out->dims().at(axes.back()) / 2 + 1;
+    onesided_dims.at(axes.back()) = onesided_axis_size;
+    phi::DenseTensor onesided_out;
+    onesided_out.Resize(onesided_dims);
+    ctx.template Alloc<T>(&onesided_out);
+    fft_r2c_func(ctx, frames_w, &onesided_out, axes, normalization, true);
+    phi::funcs::FFTFillConj<Context, C>(ctx, &onesided_out, out, axes);
+  }
+}
+}  // namespace phi
diff --git a/paddle/phi/kernels/stft_kernel.h b/paddle/phi/kernels/stft_kernel.h
new file mode 100644
index 0000000000000..3202d4e78cd8e
--- /dev/null
+++ b/paddle/phi/kernels/stft_kernel.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+template <typename T, typename Context>
+void StftKernel(const Context& ctx,
+                const DenseTensor& x,
+                const DenseTensor& window,
+                int n_fft,
+                int hop_length,
+                bool normalized,
+                bool onesided,
+                DenseTensor* out);
+
+}  // namespace phi
diff --git a/paddle/phi/ops/compat/stft_sig.cc b/paddle/phi/ops/compat/stft_sig.cc
new file mode 100644
index 0000000000000..c8c10881233a6
--- /dev/null
+++ b/paddle/phi/ops/compat/stft_sig.cc
@@ -0,0 +1,28 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/core/compat/op_utils.h"
+
+namespace phi {
+
+KernelSignature StftOpArgumentMapping(const ArgumentMappingContext& ctx) {
+  return KernelSignature("stft",
+                         {"X", "Window"},
+                         {"n_fft", "hop_length", "normalized", "onesided"},
+                         {"Out"});
+}
+
+}  // namespace phi
+
+PD_REGISTER_ARG_MAPPING_FN(stft, phi::StftOpArgumentMapping);
diff --git a/python/paddle/fluid/tests/unittests/test_stft_op.py b/python/paddle/fluid/tests/unittests/test_stft_op.py
index 44c01689d3a2f..8b672295d7cde 100644
--- a/python/paddle/fluid/tests/unittests/test_stft_op.py
+++ b/python/paddle/fluid/tests/unittests/test_stft_op.py
@@ -80,7 +80,7 @@ def initTestCase(self):
 
     def test_check_output(self):
         paddle.enable_static()
-        self.check_output()
+        self.check_output(check_dygraph=False)
         paddle.disable_static()
 
     def test_check_grad_normal(self):

From 2a94b81778b60eaff785bc444cb26d28e680437a Mon Sep 17 00:00:00 2001
From: Zhang Jun <ewalker@live.cn>
Date: Tue, 16 May 2023 12:57:22 +0800
Subject: [PATCH 346/405] [inference][trt]Remove unused code from teller.cc
 (#53758)

* remove unused code
---
 paddle/fluid/inference/tensorrt/op_teller.cc | 104 +------------------
 1 file changed, 1 insertion(+), 103 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc
index c8bc26c9200ee..4417a95cca0ef 100644
--- a/paddle/fluid/inference/tensorrt/op_teller.cc
+++ b/paddle/fluid/inference/tensorrt/op_teller.cc
@@ -416,12 +416,6 @@ struct SimpleOpTypeSetTeller : public Teller {
       auto x_var_name = desc.Input("X")[0];
       auto* x_var_desc = block->FindVar(x_var_name);
       const auto x_shape = x_var_desc->GetShape();
-      if (!with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
-        VLOG(3) << op_type
-                << " op does not support input's dim is 1 or 0 in tensorrt "
-                   "with static shape.";
-        return false;
-      }
 
       if (with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
         int axis = desc.HasAttr("axis")
@@ -1198,11 +1192,6 @@ struct SimpleOpTypeSetTeller : public Teller {
               dtype == framework::proto::VarType::FP16)) {
           return false;
         }
-        if (x_shape.size() == 1 || x_shape.size() == 0) {
-          VLOG(3) << "Scale op does not support 0 or 1-dimensional input in "
-                     "tensorrt";
-          return false;
-        }
       } else {
         // At present, only support float32 or float16 or int32 or int64 into
         // trt.
@@ -1527,13 +1516,7 @@ struct SimpleOpTypeSetTeller : public Teller {
         return false;
       }
       auto* x_var_desc = block->FindVar(desc.Input("X")[0]);
-      const auto x_shape = x_var_desc->GetShape();
-      if (!with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
-        VLOG(3) << op_type
-                << " op does not support input's dim is 1 or 0 in tensorrt "
-                   "static shape mode.";
-        return false;
-      }
+
       // the same as `elementwise_pow`.
       if (x_var_desc->GetDataType() ==
           paddle::framework::proto::VarType_Type::VarType_Type_INT32) {
@@ -1572,26 +1555,6 @@ struct SimpleOpTypeSetTeller : public Teller {
       }
     }
 
-    if (op_type == "sum") {
-      auto* block = desc.Block();
-      if (block == nullptr) {
-        VLOG(3) << "The block desc is nullptr, we can't continue to analyze. "
-                   "Developers need to check whether block_desc is passed in "
-                   "the pass.";
-        return false;
-      }
-      auto x_var_name = desc.Input("X")[0];
-      auto* x_var = block->FindVar(x_var_name);
-      const auto x_shape = x_var->GetShape();
-      if (!with_dynamic_shape && (x_shape.size() == 0 || x_shape.size() == 1)) {
-        VLOG(3) << op_type
-                << " op does not support input's dim is 0 or 1 in tensorrt "
-                   "with static shape.";
-        return false;
-      }
-      return true;
-    }
-
     if (op_type == "shape" && !with_dynamic_shape) {
       return false;
     }
@@ -1657,24 +1620,6 @@ struct SimpleOpTypeSetTeller : public Teller {
         if (PADDLE_GET_CONST(bool, desc.GetAttr("approximate"))) return false;
       }
 #endif
-
-      auto* block = desc.Block();
-      if (block == nullptr) {
-        VLOG(3) << "The block desc is nullptr, we can't continue to analyze. "
-                   "Developers need to check whether block_desc is passed in "
-                   "the pass.";
-        return false;
-      }
-
-      auto x_var_name = desc.Input("X")[0];
-      auto* x_var_desc = block->FindVar(x_var_name);
-      const auto x_shape = x_var_desc->GetShape();
-      if (!with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
-        VLOG(3) << op_type
-                << "gelu op does not support input's dim is 1 or 0 in tensorrt "
-                   "static shape mode.";
-        return false;
-      }
     }
 
     if (op_type == "layer_norm") {
@@ -1864,16 +1809,6 @@ struct SimpleOpTypeSetTeller : public Teller {
                    "static shape mode.";
         return false;
       }
-
-      auto x_var_name = desc.Input("X")[0];
-      auto* x_var = block->FindVar(x_var_name);
-      const auto x_shape = x_var->GetShape();
-      if (!with_dynamic_shape && (x_shape.size() == 1 || x_shape.size() == 0)) {
-        VLOG(3) << op_type
-                << " op does not support input's dim is 1 or 0 in tensorrt "
-                   "with static shape.";
-        return false;
-      }
     }
 
     if (op_type == "mish") {
@@ -1889,24 +1824,6 @@ struct SimpleOpTypeSetTeller : public Teller {
                 << desc.Output("Out").size() << ".";
         return false;
       }
-
-      auto* block = desc.Block();
-      if (block == nullptr) {
-        VLOG(3) << "The block desc is nullptr, we can't continue to analyze. "
-                   "Developers need to check whether block_desc is passed in "
-                   "the pass.";
-        return false;
-      }
-
-      auto x_var_name = desc.Input("X")[0];
-      auto* x_var_desc = block->FindVar(x_var_name);
-      const auto x_shape = x_var_desc->GetShape();
-      if ((!with_dynamic_shape && x_shape.size() == 1) || x_shape.size() == 0) {
-        VLOG(3) << op_type
-                << "mish op does not support input's dim is 1 in tensorrt "
-                   "static shape mode or 0.";
-        return false;
-      }
     }
 
     if (op_type == "roi_align") {
@@ -2194,25 +2111,6 @@ struct SimpleOpTypeSetTeller : public Teller {
       }
     }
 
-    if (op_type == "square") {
-      auto* block = desc.Block();
-      if (block == nullptr) {
-        VLOG(3) << "The block desc is nullptr, we can't continue to analyze. "
-                   "Developers need to check whether block_desc is passed in "
-                   "the pass.";
-        return false;
-      }
-      auto x_var_name = desc.Input("X")[0];
-      auto* x_var = block->FindVar(x_var_name);
-      const auto x_shape = x_var->GetShape();
-      if (!with_dynamic_shape && x_shape.size() == 0) {
-        VLOG(3) << op_type
-                << " op does not support input's dim is 0 in tensorrt "
-                   "with static shape.";
-        return false;
-      }
-    }
-
     if (op_type == "clip") {
       // Paddle-TRT does not support the input tensors: Min and Max
       auto clip_inputs = desc.Inputs();

From db407bf0531ea8f3deb183c110a6e79cc2389344 Mon Sep 17 00:00:00 2001
From: Yiqun Liu <Xreki@users.noreply.github.com>
Date: Tue, 16 May 2023 13:27:15 +0800
Subject: [PATCH 347/405] [AMP] Allow to switch whether to use promote strategy
 to choose kernel for O2 training. (#53742)

* Allow to switch whether to use promote strategy to choose kernel for O2 training.

* Fix comparing error and add unittest.
---
 paddle/fluid/eager/amp_utils.h              |  37 ++++--
 paddle/fluid/eager/api/utils/global_utils.h |   3 +
 paddle/fluid/imperative/tracer.cc           |   2 +
 paddle/fluid/imperative/tracer.h            |   8 ++
 paddle/fluid/pybind/imperative.cc           |   3 +
 python/paddle/amp/auto_cast.py              |  14 ++-
 test/amp/amp_base_models.py                 |  69 ++++++++++--
 test/amp/test_amp_api.py                    |  12 +-
 test/amp/test_amp_o2_embedding_model.py     |   1 +
 test/amp/test_amp_promote.py                | 118 +++++++++++++++++++-
 test/amp/test_model_cast_to_bf16.py         |   1 +
 11 files changed, 234 insertions(+), 34 deletions(-)

diff --git a/paddle/fluid/eager/amp_utils.h b/paddle/fluid/eager/amp_utils.h
index 4bfb2fdb4f64f..bfa58512eb238 100644
--- a/paddle/fluid/eager/amp_utils.h
+++ b/paddle/fluid/eager/amp_utils.h
@@ -120,18 +120,35 @@ inline phi::DataType GetAmpDestDtype(
       egr::Controller::Instance().GetCurrentTracer()->GetAmpPhiDtype();
   auto dst_type = amp_setting_dtype;
 
-  if (paddle::imperative::AmpOperators::Instance().GetMutableAllowOps()->count(
-          op_name)) {
-    dst_type = amp_setting_dtype;
-  } else if (paddle::imperative::AmpOperators::Instance()
-                 .GetMutableBlockOps()
-                 ->count(op_name)) {
-    dst_type = phi::DataType::FLOAT32;
-  } else {
-    if (amp_level == paddle::imperative::AmpLevel::OD) {
+  bool use_promote = true;
+  if (amp_level == paddle::imperative::AmpLevel::O2) {
+    use_promote =
+        egr::Controller::Instance().GetCurrentTracer()->GetUsePromote();
+  }
+
+  if (use_promote) {
+    if (paddle::imperative::AmpOperators::Instance()
+            .GetMutableAllowOps()
+            ->count(op_name)) {
+      dst_type = amp_setting_dtype;
+    } else if (paddle::imperative::AmpOperators::Instance()
+                   .GetMutableBlockOps()
+                   ->count(op_name)) {
       dst_type = phi::DataType::FLOAT32;
     } else {
-      dst_type = GetPromoteType(op_name, amp_tensors_vector, amp_setting_dtype);
+      if (amp_level == paddle::imperative::AmpLevel::OD) {
+        dst_type = phi::DataType::FLOAT32;
+      } else {
+        dst_type =
+            GetPromoteType(op_name, amp_tensors_vector, amp_setting_dtype);
+      }
+    }
+  } else {
+    // use_promote can be set to false only for O2 training.
+    if (paddle::imperative::AmpOperators::Instance()
+            .GetMutableBlockOps()
+            ->count(op_name)) {
+      dst_type = phi::DataType::FLOAT32;
     }
   }
 
diff --git a/paddle/fluid/eager/api/utils/global_utils.h b/paddle/fluid/eager/api/utils/global_utils.h
index ba79a63648dd7..75eb28a6af91d 100644
--- a/paddle/fluid/eager/api/utils/global_utils.h
+++ b/paddle/fluid/eager/api/utils/global_utils.h
@@ -58,6 +58,9 @@ class Controller {
     return tracer_->GetAmpLevel();
   }
 
+  void SetUsePromote(bool use_promote) { tracer_->SetUsePromote(use_promote); }
+  bool GetUsePromote() const { return tracer_->GetUsePromote(); }
+
   bool UseLayoutAutoTune() {
     bool use_autotune = false;
 #if defined(PADDLE_WITH_CUDA)
diff --git a/paddle/fluid/imperative/tracer.cc b/paddle/fluid/imperative/tracer.cc
index 2d4e6622c05b7..ccb58d320221c 100644
--- a/paddle/fluid/imperative/tracer.cc
+++ b/paddle/fluid/imperative/tracer.cc
@@ -44,6 +44,8 @@ thread_local bool Tracer::enable_program_desc_tracing_ = false;
 
 thread_local bool Tracer::has_grad_ = true;
 
+thread_local bool Tracer::use_promote_ = true;
+
 thread_local bool Tracer::use_layout_autotune_ = false;
 
 thread_local AmpLevel Tracer::amp_level_ = AmpLevel::O0;
diff --git a/paddle/fluid/imperative/tracer.h b/paddle/fluid/imperative/tracer.h
index 7355cec776e35..87e0edd036f06 100644
--- a/paddle/fluid/imperative/tracer.h
+++ b/paddle/fluid/imperative/tracer.h
@@ -156,6 +156,13 @@ class Tracer {
 
   void SetHasGrad(bool has_grad) { has_grad_ = has_grad; }
 
+  void SetUsePromote(bool use_promote) {
+    VLOG(4) << "set use_promote to " << use_promote;
+    use_promote_ = use_promote;
+  }
+
+  bool GetUsePromote() const { return use_promote_; }
+
   void SetAmpLevel(AmpLevel level) {
     VLOG(4) << "set amp_level to " << static_cast<unsigned int>(level);
     amp_level_ = level;
@@ -220,6 +227,7 @@ class Tracer {
   static thread_local bool enable_program_desc_tracing_;
   static thread_local bool use_layout_autotune_;
   static thread_local bool has_grad_;
+  static thread_local bool use_promote_;
   static thread_local AmpLevel amp_level_;
   static thread_local phi::DataType amp_dtype_;
 };
diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc
index 5bbd66fd09c98..67c8afbafbea2 100644
--- a/paddle/fluid/pybind/imperative.cc
+++ b/paddle/fluid/pybind/imperative.cc
@@ -2156,6 +2156,9 @@ void BindImperative(py::module *m_ptr) {
       .def_property("_enable_program_desc_tracing",
                     &imperative::Tracer::IsProgramDescTracingEnabled,
                     &imperative::Tracer::SetEnableProgramDescTracing)
+      .def_property("_use_promote",
+                    &imperative::Tracer::GetUsePromote,
+                    &imperative::Tracer::SetUsePromote)
       .def_property("_amp_level",
                     &imperative::Tracer::GetAmpLevel,
                     &imperative::Tracer::SetAmpLevel)
diff --git a/python/paddle/amp/auto_cast.py b/python/paddle/amp/auto_cast.py
index 2cb0af3530eab..c31b0bbef8e12 100644
--- a/python/paddle/amp/auto_cast.py
+++ b/python/paddle/amp/auto_cast.py
@@ -274,6 +274,7 @@ def amp_guard(
     custom_black_list=None,
     level='O1',
     dtype='float16',
+    use_promote=True,
 ):
     """
     Create a context which enables auto-mixed-precision(AMP) of operators executed in dynamic graph mode.
@@ -438,6 +439,11 @@ def master_grad_hook():
         original_amp_dtype = tracer._amp_dtype
         tracer._amp_dtype = amp_dtype
 
+        # switch promote
+        if amp_level == AMP_LEVEL.O2:
+            original_use_promote = tracer._use_promote
+            tracer._use_promote = use_promote
+
     # restore status
     try:
         yield
@@ -448,6 +454,8 @@ def master_grad_hook():
             tracer._set_amp_op_list(original_white_list, original_black_list)
             # set_flags(original_flags)
             tracer._amp_dtype = original_amp_dtype
+            if amp_level == AMP_LEVEL.O2:
+                tracer._use_promote = original_use_promote
 
 
 class StateDictHook:
@@ -641,6 +649,7 @@ def auto_cast(
     custom_black_list=None,
     level='O1',
     dtype='float16',
+    use_promote=True,
 ):
     """
     Create a context which enables auto-mixed-precision(AMP) of operators executed in dynamic graph mode.
@@ -663,6 +672,7 @@ def auto_cast(
              will be converted to float16/bfloat16, and that have any float32 input will be converted to float32. For the OD level, operators in
              default white list will compute in float16/bfloat16, and the others will compute in float32. Default is O1.
         dtype(str, optional): Whether to use 'float16' or 'bfloat16'. Default is 'float16'.
+        use_promote(bool, optional): Whether to promotes to fp32 when op has any float32 inputs. It is only supported when amp level is O2. Default is True.
 
     Examples:
 
@@ -696,7 +706,9 @@ def auto_cast(
             print(d.dtype) # paddle.float16
 
     """
-    return amp_guard(enable, custom_white_list, custom_black_list, level, dtype)
+    return amp_guard(
+        enable, custom_white_list, custom_black_list, level, dtype, use_promote
+    )
 
 
 def decorate(
diff --git a/test/amp/amp_base_models.py b/test/amp/amp_base_models.py
index 8b63b2391c020..aabf9e82f7b84 100644
--- a/test/amp/amp_base_models.py
+++ b/test/amp/amp_base_models.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
 import unittest
 
 import numpy as np
@@ -19,6 +20,7 @@
 import paddle
 from paddle import nn
 from paddle.fluid import core
+from paddle.fluid.framework import _non_static_mode
 
 _fixed_add_param = np.random.random(size=[16, 16]).astype("float32")
 
@@ -30,20 +32,27 @@ def _build_optimizer(
     amp_lists=None,
     use_grad_clip=False,
     use_promote=False,
+    model=None,
 ):
     if use_grad_clip:
         grad_clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0)
     else:
         grad_clip = None
+    if _non_static_mode():
+        assert model is not None
+        parameters = model.parameters()
+    else:
+        parameters = None
     optimizer = paddle.optimizer.AdamW(
         learning_rate=0.01,
+        parameters=parameters,
         grad_clip=grad_clip,
         beta1=0.78,
         beta2=0.836,
         epsilon=1e-4,
         weight_decay=0.01,
     )
-    if use_amp:
+    if not _non_static_mode() and use_amp:
         optimizer = paddle.static.amp.decorate(
             optimizer,
             amp_lists,
@@ -118,7 +127,7 @@ def __init__(self):
 
     def forward(self, x):
         out = self.conv(x)
-        out = nn.functional.relu(out)
+        out = nn.functional.relu(out.cast("float32"))
         out = out.flatten(start_axis=1, stop_axis=3)
         out = self.linear(out)
         out = nn.functional.softmax(out)
@@ -128,6 +137,22 @@ def forward(self, x):
 def build_conv_model(
     use_amp, amp_dtype="float16", amp_level="O1", use_promote=False
 ):
+    if _non_static_mode():
+        model = SimpleConvNet()
+        optimizer = _build_optimizer(use_amp=False, model=model)
+        if use_amp and amp_dtype == "float16":
+            scaler = paddle.amp.GradScaler()
+        else:
+            scaler = None
+        if use_amp and amp_level == "O2":
+            model, optimizer = paddle.amp.decorate(
+                models=model,
+                optimizers=optimizer,
+                level=amp_level,
+                dtype=amp_dtype,
+            )
+        return model, optimizer, scaler
+
     main_program = paddle.static.Program()
     startup_program = paddle.static.Program()
     with paddle.utils.unique_name.guard():
@@ -237,19 +262,36 @@ def setUp(self):
         self.amp_level = None
 
     def _check_op_calls(
-        self, op_stats_dict, expected_bf16_calls={}, expected_fp16_calls={}
+        self,
+        op_stats_dict,
+        expected_bf16_calls={},
+        expected_fp16_calls={},
+        debug_info=None,
     ):
-        for op_type, value in expected_bf16_calls.items():
+        def _extract_op_call(op_calls_str, pos):
+            return int(copy.copy(op_calls_str).split(",")[pos])
+
+        for op_type, expected_value in expected_bf16_calls.items():
+            # print(f"[BF16] op_type={op_type}, value={value}")
+            if isinstance(op_stats_dict[op_type], str):
+                actual_value = _extract_op_call(op_stats_dict[op_type], 1)
+            else:
+                actual_value = op_stats_dict[op_type].bf16_calls
             self.assertEqual(
-                op_stats_dict[op_type].bf16_calls,
-                value,
-                f"The number of bf16 calls of operator < {op_type} > is expected to be {value}, but recieved {op_stats_dict[op_type].bf16_calls}.",
+                actual_value,
+                expected_value,
+                f"[{debug_info}] The number of bf16 calls of operator < {op_type} > is expected to be {expected_value}, but recieved {actual_value}.",
             )
-        for op_type, value in expected_fp16_calls.items():
+        for op_type, expected_value in expected_fp16_calls.items():
+            # print(f"[FP16] op_type={op_type}, value={value}")
+            if isinstance(op_stats_dict[op_type], str):
+                actual_value = _extract_op_call(op_stats_dict[op_type], 0)
+            else:
+                actual_value = op_stats_dict[op_type].fp16_calls
             self.assertEqual(
-                op_stats_dict[op_type].fp16_calls,
-                value,
-                f"The number of fp16 calls of operator < {op_type} > is expected to be {value}, but recieved {op_stats_dict[op_type].fp16_calls}.",
+                actual_value,
+                expected_value,
+                f"[debug_info] The number of fp16 calls of operator < {op_type} > is expected to be {expected_value}, but recieved {actual_value}.",
             )
 
     def run_program(
@@ -263,6 +305,7 @@ def run_program(
         exe,
         x_np,
         max_iters,
+        dtype,
         level,
     ):
         losses = []
@@ -277,6 +320,8 @@ def run_program(
                     feed={feed_vars[0].name: x_np},
                     fetch_list=fetch_vars,
                 )
-                print(f"-- [BF16 {level}] iter={iter_id}, loss={results[0]}")
+                print(
+                    f"-- [AMP {dtype} {level}] iter={iter_id}, loss={results[0]}"
+                )
                 losses.append(results[0])
         return losses
diff --git a/test/amp/test_amp_api.py b/test/amp/test_amp_api.py
index 7d397c7043206..4120dcce93904 100644
--- a/test/amp/test_amp_api.py
+++ b/test/amp/test_amp_api.py
@@ -20,15 +20,17 @@
 
 
 class TestAutoCast(AmpTestBase):
-    def test_amp_OD_level(self):
-        conv = paddle.nn.Conv2D(
+    def setUp(self):
+        self._conv = paddle.nn.Conv2D(
             in_channels=1, out_channels=6, kernel_size=3, bias_attr=False
         )
-        linear = paddle.nn.Linear(in_features=4, out_features=4)
+        self._linear = paddle.nn.Linear(in_features=4, out_features=4)
+
+    def test_amp_OD_level(self):
         with paddle.amp.auto_cast(level='OD'):
-            out1 = conv(paddle.rand(shape=[1, 1, 6, 6], dtype='float32'))
+            out1 = self._conv(paddle.rand(shape=[1, 1, 6, 6], dtype='float32'))
             out2 = out1 + paddle.rand(shape=out1.shape, dtype='float16')
-            out3 = linear(out2)
+            out3 = self._linear(out2)
 
         self.assertEqual(out1.dtype, paddle.float16)
         self.assertEqual(out2.dtype, paddle.float32)
diff --git a/test/amp/test_amp_o2_embedding_model.py b/test/amp/test_amp_o2_embedding_model.py
index b1af4bde8d3d8..237ca1120d6d9 100644
--- a/test/amp/test_amp_o2_embedding_model.py
+++ b/test/amp/test_amp_o2_embedding_model.py
@@ -131,6 +131,7 @@ def _run(place, exe, x_np, max_iters, level):
                 exe,
                 x_np,
                 max_iters,
+                "float16",
                 level,
             )
             return losses
diff --git a/test/amp/test_amp_promote.py b/test/amp/test_amp_promote.py
index e75799b39c332..9f8395df4137f 100644
--- a/test/amp/test_amp_promote.py
+++ b/test/amp/test_amp_promote.py
@@ -20,13 +20,12 @@
 import paddle
 from paddle.static import amp
 
-paddle.enable_static()
 
-
-class TestAMPPromote(AmpTestBase):
+class TestStaticAmpPromoteStats(AmpTestBase):
     def check_promote_results(
-        self, use_amp, dtype, level, use_promote, expected_op_calls
+        self, use_amp, dtype, level, use_promote, expected_op_calls, debug_info
     ):
+        paddle.enable_static()
         (
             main_program,
             startup_program,
@@ -40,7 +39,9 @@ def check_promote_results(
         op_stats_list = amp.debugging._get_op_stats_list(main_program)
 
         self._check_op_calls(
-            op_stats_list[0], expected_fp16_calls=expected_op_calls
+            op_stats_list[0],
+            expected_fp16_calls=expected_op_calls,
+            debug_info=debug_info,
         )
 
         place = paddle.CUDAPlace(0)
@@ -58,8 +59,10 @@ def check_promote_results(
             exe,
             x_fp32,
             max_iters,
+            dtype,
             level,
         )
+        paddle.disable_static()
 
     def test_static_amp_o1(self):
         expected_fp16_calls = {
@@ -77,13 +80,14 @@ def test_static_amp_o1(self):
             'O1',
             use_promote=True,
             expected_op_calls=expected_fp16_calls,
+            debug_info="TestStaticAmpPromoteStats/test_static_amp_o1",
         )
 
     def test_static_amp_o2(self):
         expected_fp16_calls = {
             "conv2d": 1,
             "elementwise_add": 2,
-            "relu": 1,
+            "relu": 0,
             "matmul_v2": 1,
             "softmax": 1,
             "reduce_mean": 1,
@@ -95,7 +99,109 @@ def test_static_amp_o2(self):
             'O2',
             use_promote=True,
             expected_op_calls=expected_fp16_calls,
+            debug_info="TestStaticAmpPromoteStats/test_static_amp_o2",
+        )
+
+
+class TestEagerAmpPromoteStats(AmpTestBase):
+    def check_promote_results(
+        self, dtype, level, use_promote, expected_op_calls, debug_info
+    ):
+        model, optimizer, scaler = build_conv_model(
+            use_amp=True,
+            amp_dtype=dtype,
+            amp_level=level,
+            use_promote=use_promote,
+        )
+        model.train()
+
+        paddle.amp.debugging.enable_operator_stats_collection()
+        with paddle.amp.auto_cast(
+            enable=True, dtype=dtype, level=level, use_promote=use_promote
+        ):
+            x = paddle.rand(shape=[1, 1, 6, 6], dtype='float32')
+            out = model(x)
+            loss = paddle.mean(out)
+        scaled = scaler.scale(loss)
+        scaled.backward()
+        scaler.minimize(optimizer, scaled)
+        optimizer.clear_grad()
+        paddle.amp.debugging.disable_operator_stats_collection()
+        op_stats = paddle.fluid.core.get_low_precision_op_list()
+
+        self._check_op_calls(
+            op_stats,
+            expected_fp16_calls=expected_op_calls,
+            debug_info=debug_info,
+        )
+
+    def test_o2_promote_on(self):
+        expected_fp16_calls = {
+            "conv2d": 1,
+            "elementwise_add": 2,
+            "relu": 0,
+            "matmul_v2": 1,
+            "softmax": 1,
+            "reduce_mean": 1,
+            "adamw_": 4,
+        }
+        self.check_promote_results(
+            'float16',
+            'O2',
+            use_promote=True,
+            expected_op_calls=expected_fp16_calls,
+            debug_info="TestEagerAmpPromoteStats/test_o2_promote_on",
+        )
+
+    def test_o2_promote_off(self):
+        expected_fp16_calls = {
+            "conv2d": 1,
+            "elementwise_add": 2,
+            "relu": 1,
+            "matmul_v2": 1,
+            "softmax": 1,
+            "reduce_mean": 1,
+            "adamw_": 4,
+        }
+        self.check_promote_results(
+            'float16',
+            'O2',
+            use_promote=False,
+            expected_op_calls=expected_fp16_calls,
+            debug_info="TestEagerAmpPromoteStats/test_o2_promote_off",
+        )
+
+
+class TestEagerAmpPromoteSimple(AmpTestBase):
+    def setUp(self):
+        self._conv = paddle.nn.Conv2D(
+            in_channels=1, out_channels=6, kernel_size=3, bias_attr=False
         )
+        self._linear = paddle.nn.Linear(in_features=4, out_features=4)
+
+    def test_o2_use_promote_on(self):
+        with paddle.amp.auto_cast(level='O2'):
+            x = paddle.rand(shape=[1, 1, 6, 6], dtype='float32')
+            conv_out = self._conv(x)
+            y = paddle.rand(shape=conv_out.shape, dtype='float16')
+            add_out = conv_out + y
+            linear_out = self._linear(add_out)
+
+        self.assertEqual(conv_out.dtype, paddle.float16)
+        self.assertEqual(add_out.dtype, paddle.float16)
+        self.assertEqual(linear_out.dtype, paddle.float32)
+
+    def test_o2_use_promote_off(self):
+        with paddle.amp.auto_cast(level='O2', use_promote=False):
+            x = paddle.rand(shape=[1, 1, 6, 6], dtype='float32')
+            conv_out = self._conv(x)
+            y = paddle.rand(shape=conv_out.shape, dtype='float16')
+            add_out = conv_out + y
+            linear_out = self._linear(add_out)
+
+        self.assertEqual(conv_out.dtype, paddle.float16)
+        self.assertEqual(add_out.dtype, paddle.float16)
+        self.assertEqual(linear_out.dtype, paddle.float16)
 
 
 if __name__ == '__main__':
diff --git a/test/amp/test_model_cast_to_bf16.py b/test/amp/test_model_cast_to_bf16.py
index 7e4de2630d4a5..373a5ed06ace3 100644
--- a/test/amp/test_model_cast_to_bf16.py
+++ b/test/amp/test_model_cast_to_bf16.py
@@ -310,6 +310,7 @@ def _run(place, exe, x_np, max_iters, level):
                 exe,
                 x_np,
                 max_iters,
+                "bfloat16",
                 level,
             )
             return losses

From 51ecd933daed6c6b7a4a6918b12eaeb4f4790b13 Mon Sep 17 00:00:00 2001
From: Yuanle Liu <yuanlehome@163.com>
Date: Tue, 16 May 2023 13:57:08 +0800
Subject: [PATCH 348/405] [Inference] clean unused code/target for reduce
 inference so volume (PART I) (#53762)

* remove prelu land ookuip_table plugin, adjust .h include location

* clean code and adjust some .h

* update
---
 .../framework/ir/attention_lstm_fuse_pass.cc  |   6 +-
 paddle/fluid/framework/ir/graph_viz_pass.cc   |   2 +
 paddle/fluid/framework/ir/graph_viz_pass.h    |   5 -
 paddle/fluid/inference/analysis/analyzer.cc   |   1 -
 paddle/fluid/inference/analysis/analyzer.h    |   2 -
 paddle/fluid/inference/analysis/device.h      |  24 --
 paddle/fluid/inference/analysis/dot_tester.cc |  62 ----
 paddle/fluid/inference/analysis/flags.h       |  24 --
 .../ir_passes/tensorrt_subgraph_pass.cc       |   1 +
 .../ir_passes/tensorrt_subgraph_pass.h        |   1 -
 .../analysis/passes/ir_analysis_pass.cc       |   7 -
 .../analysis/passes/ir_analysis_pass.h        |   4 -
 .../analysis/passes/ir_graph_build_pass.cc    |   1 -
 .../ir_params_sync_among_devices_pass.cc      |   8 +-
 .../ir_params_sync_among_devices_pass.h       |   3 -
 .../analysis/passes/memory_optimize_pass.cc   |   2 +
 .../analysis/passes/memory_optimize_pass.h    |   3 -
 paddle/fluid/inference/api/CMakeLists.txt     |  15 -
 paddle/fluid/inference/api/README.md          |  17 -
 paddle/fluid/inference/api/high_level_api.md  |  59 ---
 .../fluid/inference/api/high_level_api_cn.md  |  84 -----
 paddle/fluid/inference/engine.h               |  62 ----
 .../inference/tensorrt/convert/CMakeLists.txt |   1 -
 .../convert/fused_lookup_tables_op.cc         | 115 ------
 .../inference/tensorrt/convert/prelu_op.cc    |   1 -
 paddle/fluid/inference/tensorrt/engine.h      |   1 -
 .../inference/tensorrt/plugin/CMakeLists.txt  |   8 +-
 .../inference/tensorrt/plugin/lookup_table.cu | 348 ------------------
 .../inference/tensorrt/plugin/lookup_table.h  | 126 -------
 .../tensorrt/plugin/prelu_op_plugin.cu        | 224 -----------
 .../tensorrt/plugin/prelu_op_plugin.h         | 217 -----------
 31 files changed, 11 insertions(+), 1423 deletions(-)
 delete mode 100644 paddle/fluid/inference/analysis/device.h
 delete mode 100644 paddle/fluid/inference/analysis/dot_tester.cc
 delete mode 100644 paddle/fluid/inference/analysis/flags.h
 mode change 100755 => 100644 paddle/fluid/inference/analysis/passes/memory_optimize_pass.cc
 delete mode 100644 paddle/fluid/inference/api/README.md
 delete mode 100644 paddle/fluid/inference/api/high_level_api.md
 delete mode 100644 paddle/fluid/inference/api/high_level_api_cn.md
 delete mode 100644 paddle/fluid/inference/engine.h
 delete mode 100644 paddle/fluid/inference/tensorrt/convert/fused_lookup_tables_op.cc
 delete mode 100644 paddle/fluid/inference/tensorrt/plugin/lookup_table.cu
 delete mode 100644 paddle/fluid/inference/tensorrt/plugin/lookup_table.h
 delete mode 100644 paddle/fluid/inference/tensorrt/plugin/prelu_op_plugin.cu
 delete mode 100644 paddle/fluid/inference/tensorrt/plugin/prelu_op_plugin.h

diff --git a/paddle/fluid/framework/ir/attention_lstm_fuse_pass.cc b/paddle/fluid/framework/ir/attention_lstm_fuse_pass.cc
index 8225840a69d7c..1aa12ded2611b 100644
--- a/paddle/fluid/framework/ir/attention_lstm_fuse_pass.cc
+++ b/paddle/fluid/framework/ir/attention_lstm_fuse_pass.cc
@@ -107,11 +107,7 @@ void AttentionLSTMFusePass::FindWhileOp(Graph* graph) const {
   gpd.mutable_pattern()->NewNode(
       [&](Node* n) { return fused_external_ops.count(n->id()); }, "while");
 
-  if (!graph->Has(kGraphvizMarkedNodeAttr)) {
-    graph->Set(kGraphvizMarkedNodeAttr, new GraphVizPass::marked_nodes_t);
-  }
-  auto& marked_nodes =
-      graph->Get<GraphVizPass::marked_nodes_t>(kGraphvizMarkedNodeAttr);
+  auto& marked_nodes = GetMarkedNodes(graph);
 
   auto handle = [&](const GraphPatternDetector::subgraph_t& subgraph,
                     Graph* g) {
diff --git a/paddle/fluid/framework/ir/graph_viz_pass.cc b/paddle/fluid/framework/ir/graph_viz_pass.cc
index eed4ab38f5be7..8403d9075e3bf 100644
--- a/paddle/fluid/framework/ir/graph_viz_pass.cc
+++ b/paddle/fluid/framework/ir/graph_viz_pass.cc
@@ -14,6 +14,8 @@ limitations under the License. */
 
 #include "paddle/fluid/framework/ir/graph_viz_pass.h"
 
+#include <fstream>
+#include <ostream>
 #include <string>
 
 #include "paddle/fluid/framework/ir/graph_helper.h"
diff --git a/paddle/fluid/framework/ir/graph_viz_pass.h b/paddle/fluid/framework/ir/graph_viz_pass.h
index 118c1bc6f3c78..0d3b32a7c1a78 100644
--- a/paddle/fluid/framework/ir/graph_viz_pass.h
+++ b/paddle/fluid/framework/ir/graph_viz_pass.h
@@ -14,12 +14,7 @@ limitations under the License. */
 
 #pragma once
 
-#include <fstream>
-#include <map>
-#include <memory>
-#include <string>
 #include <unordered_set>
-#include <vector>
 
 #include "paddle/fluid/framework/ir/graph.h"
 #include "paddle/fluid/framework/ir/pass.h"
diff --git a/paddle/fluid/inference/analysis/analyzer.cc b/paddle/fluid/inference/analysis/analyzer.cc
index 4aadb34d7b354..afa653dec9069 100644
--- a/paddle/fluid/inference/analysis/analyzer.cc
+++ b/paddle/fluid/inference/analysis/analyzer.cc
@@ -15,7 +15,6 @@
 #include "paddle/fluid/inference/analysis/analyzer.h"
 
 #include <string>
-#include <vector>
 
 #include "paddle/fluid/inference/analysis/passes/passes.h"
 #include "paddle/fluid/string/pretty_log.h"
diff --git a/paddle/fluid/inference/analysis/analyzer.h b/paddle/fluid/inference/analysis/analyzer.h
index 95a985158e678..3c05328366674 100644
--- a/paddle/fluid/inference/analysis/analyzer.h
+++ b/paddle/fluid/inference/analysis/analyzer.h
@@ -35,12 +35,10 @@ limitations under the License. */
  * phase in the inference service.
  */
 
-#include <string>
 #include <vector>
 
 #include "gflags/gflags.h"
 #include "paddle/fluid/inference/analysis/analysis_pass.h"
-#include "paddle/fluid/inference/analysis/flags.h"
 
 namespace paddle {
 namespace inference {
diff --git a/paddle/fluid/inference/analysis/device.h b/paddle/fluid/inference/analysis/device.h
deleted file mode 100644
index 585c9923291e5..0000000000000
--- a/paddle/fluid/inference/analysis/device.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-#pragma once
-
-namespace paddle {
-namespace inference {
-namespace analysis {
-
-enum class Device { CPU, GPU };
-
-}  // namespace analysis
-}  // namespace inference
-}  // namespace paddle
diff --git a/paddle/fluid/inference/analysis/dot_tester.cc b/paddle/fluid/inference/analysis/dot_tester.cc
deleted file mode 100644
index 0b669093a1f32..0000000000000
--- a/paddle/fluid/inference/analysis/dot_tester.cc
+++ /dev/null
@@ -1,62 +0,0 @@
-//   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <gtest/gtest.h>
-
-#include <memory>
-
-#include "paddle/fluid/inference/analysis/dot.h"
-
-namespace paddle {
-namespace inference {
-namespace analysis {
-
-class DotTester : public ::testing::Test {
- protected:
-  void SetUp() override {
-    std::vector<Dot::Attr> attrs({{"title", "hello"}});
-    dot.reset(new Dot(attrs));
-    dot->AddNode("a", {Dot::Attr{"shape", "box"}, Dot::Attr("color", "blue")});
-    dot->AddNode("b", {});
-    dot->AddNode("c", {});
-    dot->AddEdge("a", "b", {});
-    dot->AddEdge("b", "c", {});
-    dot->AddEdge("a", "c", {});
-  }
-
-  std::unique_ptr<Dot> dot;
-};
-
-TEST_F(DotTester, Build) {
-  auto codes = dot->Build();
-  // Output the DOT language code, the generated codes are too long to compare
-  // the string.
-  //
-  // The output is
-  //
-  // digraph G {
-  //   title="hello"
-  //   node_1
-  //   node_2
-  //   node_0[label="a" shape="box" color="blue"]
-  //   node_0->node_1
-  //   node_1->node_2
-  //   node_0->node_2
-  // } // end G
-  LOG(INFO) << '\n' << codes;
-}
-
-}  // namespace analysis
-}  // namespace inference
-}  // namespace paddle
diff --git a/paddle/fluid/inference/analysis/flags.h b/paddle/fluid/inference/analysis/flags.h
deleted file mode 100644
index ea0406128b53b..0000000000000
--- a/paddle/fluid/inference/analysis/flags.h
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#pragma once
-
-#include "gflags/gflags.h"
-
-// TODO(Superjomn) add a definition flag like PADDLE_WITH_TENSORRT and hide this
-// flag if not available.
-DECLARE_bool(IA_enable_tensorrt_subgraph_engine);
-DECLARE_string(IA_graphviz_log_root);
-DECLARE_string(IA_output_storage_path);
-DECLARE_bool(IA_enable_ir);
diff --git a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc
index e9733a07ec8b5..9d39b6e611827 100644
--- a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc
+++ b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc
@@ -28,6 +28,7 @@
 #include "paddle/fluid/framework/ir/subgraph_detector.h"
 #include "paddle/fluid/framework/op_version_registry.h"
 #include "paddle/fluid/inference/analysis/helper.h"
+#include "paddle/fluid/inference/analysis/ir_passes/subgraph_util.h"
 #include "paddle/fluid/inference/analysis/passes/convert_to_mixed_precision.h"
 #include "paddle/fluid/inference/api/helper.h"
 #include "paddle/fluid/inference/tensorrt/convert/op_converter.h"
diff --git a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.h b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.h
index a79c41f6a075f..b720f2c6b9f10 100644
--- a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.h
+++ b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.h
@@ -21,7 +21,6 @@
 
 #include "paddle/fluid/framework/ir/fuse_pass_base.h"
 #include "paddle/fluid/framework/ir/pass.h"
-#include "paddle/fluid/inference/analysis/ir_passes/subgraph_util.h"
 #include "paddle/fluid/inference/api/paddle_analysis_config.h"
 
 namespace paddle {
diff --git a/paddle/fluid/inference/analysis/passes/ir_analysis_pass.cc b/paddle/fluid/inference/analysis/passes/ir_analysis_pass.cc
index 2f9914ac54a79..f93606892b646 100644
--- a/paddle/fluid/inference/analysis/passes/ir_analysis_pass.cc
+++ b/paddle/fluid/inference/analysis/passes/ir_analysis_pass.cc
@@ -14,16 +14,9 @@
 
 #include "paddle/fluid/inference/analysis/passes/ir_analysis_pass.h"
 
-#include <memory>
-#include <utility>
-
 #include "paddle/fluid/framework/ir/fuse_pass_base.h"
 #include "paddle/fluid/inference/analysis/ir_pass_manager.h"
 
-#ifdef PADDLE_WITH_MKLDNN
-#include "paddle/fluid/framework/ir/mkldnn/mkldnn_pass_util.h"
-#endif
-
 namespace paddle {
 namespace inference {
 namespace analysis {
diff --git a/paddle/fluid/inference/analysis/passes/ir_analysis_pass.h b/paddle/fluid/inference/analysis/passes/ir_analysis_pass.h
index 98ee592245660..fca431b5d7779 100644
--- a/paddle/fluid/inference/analysis/passes/ir_analysis_pass.h
+++ b/paddle/fluid/inference/analysis/passes/ir_analysis_pass.h
@@ -33,10 +33,6 @@ class IrAnalysisPass : public AnalysisPass {
 
   void CollectFusionStatis(Argument* argument);
 
-  void ReadCalibrationInfo(
-      Argument* argument,
-      std::unordered_map<std::string, std::vector<float>>* var_quant_scales);
-
   std::string repr() const override;
 };
 
diff --git a/paddle/fluid/inference/analysis/passes/ir_graph_build_pass.cc b/paddle/fluid/inference/analysis/passes/ir_graph_build_pass.cc
index 3f13fd639aa55..60d94eae7a6e0 100644
--- a/paddle/fluid/inference/analysis/passes/ir_graph_build_pass.cc
+++ b/paddle/fluid/inference/analysis/passes/ir_graph_build_pass.cc
@@ -14,7 +14,6 @@
 
 #include "paddle/fluid/inference/analysis/passes/ir_graph_build_pass.h"
 
-#include <memory>
 #include <string>
 
 #include "paddle/fluid/framework/executor.h"
diff --git a/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc b/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc
index 54b7cd85ff2bb..65e149925e742 100644
--- a/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc
+++ b/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc
@@ -17,15 +17,15 @@
 #include <cstdlib>
 #include <string>
 #include <unordered_set>
+#include <vector>
 
-#include "paddle/fluid/framework/data_layout.h"
 #include "paddle/fluid/framework/framework.pb.h"
 #include "paddle/fluid/framework/ir/graph_helper.h"
-#include "paddle/fluid/framework/lod_tensor.h"
+#include "paddle/fluid/framework/scope.h"
 #include "paddle/fluid/framework/tensor_util.h"
-#include "paddle/fluid/platform/bfloat16.h"
 #include "paddle/fluid/platform/enforce.h"
-#include "paddle/phi/common/data_type.h"
+#include "paddle/fluid/platform/place.h"
+#include "paddle/phi/core/dense_tensor.h"
 
 DEFINE_bool(
     custom_model_save_cpu,
diff --git a/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.h b/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.h
index 9db17abc24d2a..ee29af1c13308 100644
--- a/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.h
+++ b/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.h
@@ -15,12 +15,9 @@
 #pragma once
 
 #include <string>
-#include <vector>
 
 #include "paddle/fluid/framework/ir/fuse_pass_base.h"
-#include "paddle/fluid/framework/scope.h"
 #include "paddle/fluid/inference/analysis/analysis_pass.h"
-#include "paddle/fluid/platform/place.h"
 
 namespace paddle {
 namespace inference {
diff --git a/paddle/fluid/inference/analysis/passes/memory_optimize_pass.cc b/paddle/fluid/inference/analysis/passes/memory_optimize_pass.cc
old mode 100755
new mode 100644
index 40a8c5ce66a2a..a9478cde66ded
--- a/paddle/fluid/inference/analysis/passes/memory_optimize_pass.cc
+++ b/paddle/fluid/inference/analysis/passes/memory_optimize_pass.cc
@@ -15,7 +15,9 @@
 #include "paddle/fluid/inference/analysis/passes/memory_optimize_pass.h"
 
 #include <string>
+#include <unordered_set>
 #include <utility>
+#include <vector>
 
 #include "glog/logging.h"
 #include "paddle/fluid/framework/ir/graph_helper.h"
diff --git a/paddle/fluid/inference/analysis/passes/memory_optimize_pass.h b/paddle/fluid/inference/analysis/passes/memory_optimize_pass.h
index 5dcd8b1059ebc..9ce804019744c 100644
--- a/paddle/fluid/inference/analysis/passes/memory_optimize_pass.h
+++ b/paddle/fluid/inference/analysis/passes/memory_optimize_pass.h
@@ -15,12 +15,9 @@
 #pragma once
 #include <string>
 #include <unordered_map>
-#include <unordered_set>
 #include <utility>
-#include <vector>
 
 #include "paddle/fluid/inference/analysis/analysis_pass.h"
-#include "paddle/phi/backends/dynload/port.h"
 
 namespace paddle {
 namespace framework {
diff --git a/paddle/fluid/inference/api/CMakeLists.txt b/paddle/fluid/inference/api/CMakeLists.txt
index f07e5406a002b..b681e56d3b9f9 100755
--- a/paddle/fluid/inference/api/CMakeLists.txt
+++ b/paddle/fluid/inference/api/CMakeLists.txt
@@ -1,18 +1,3 @@
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
 if(APPLE)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=pessimizing-move")
 endif()
diff --git a/paddle/fluid/inference/api/README.md b/paddle/fluid/inference/api/README.md
deleted file mode 100644
index 990b061c8f92b..0000000000000
--- a/paddle/fluid/inference/api/README.md
+++ /dev/null
@@ -1,17 +0,0 @@
-# Embed Paddle Inference in Your Application
-
-Paddle inference offers the APIs in `C` and `C++` languages.
-
-You can easily deploy a model trained by Paddle following the steps as below:
-
-1. Optimize the native model;
-2. Write some codes for deployment.
-
-## The APIs
-
-All the released APIs are located in the `paddle_inference_api.h` header file.
-The stable APIs are wrapped by `namespace paddle`, the unstable APIs are protected by `namespace paddle::contrib`.
-
-## Write some codes
-
-Read `paddle_inference_api.h` for more information.
diff --git a/paddle/fluid/inference/api/high_level_api.md b/paddle/fluid/inference/api/high_level_api.md
deleted file mode 100644
index ca22767a1b889..0000000000000
--- a/paddle/fluid/inference/api/high_level_api.md
+++ /dev/null
@@ -1,59 +0,0 @@
-# Inference High-level APIs
-This document describes the high-level inference APIs, one can use them to deploy a Paddle model for an application quickly.
-
-The APIs are described in `paddle_inference_api.h`, just one header file, and two libaries `libpaddle_inference.so` and `libpaddle_inference_io.so` are needed for a deployment.
-
-## PaddleTensor
-We provide the `PaddleTensor` data structure to give a general tensor interface.
-
-The definition is
-
-```c++
-struct PaddleTensor {
-  std::string name;  // variable name.
-  std::vector<int> shape;
-  PaddleBuf data;  // blob of data.
-  PaddleDType dtype;
-};
-```
-
-The data is stored in a continuous memory `PaddleBuf,` and a `PaddleDType` specifies tensor's data type.
-The `name` field is used to specify the name of an input variable,
-that is important when there are multiple inputs and need to distinguish which variable to set.
-
-## engine
-The inference APIs has two different underlying engines
-
-- the native engine
-- the tensorrt engine
-
-The native engine, which is consists of the native operators and framework, takes a native Paddle model
-as input, and supports any model that trained by Paddle.
-
-```c++
-enum class PaddleEngineKind {
-  kNative = 0,  // Use the native Fluid facility.
-  kAutoMixedTensorRT // Automatically mixing TensorRT with the Fluid ops.
-};
-```
-
-## PaddlePredictor and how to create one
-The main interface is `PaddlePredictor,` there are following methods
-
-- `bool Run(const std::vector<PaddleTensor>& inputs, std::vector<PaddleTensor>* output_data)`
-  - take inputs and output `output_data.`
-- `Clone` to clone a predictor from an existing one, with model parameter shared.
-
-There is a factory method to help create a predictor, and the user takes the ownership of this object.
-
-```c++
-template <typename ConfigT, PaddleEngineKind engine = PaddleEngineKind::kNative>
-std::unique_ptr<PaddlePredictor> CreatePaddlePredictor(const ConfigT& config);
-```
-
-By specifying the engine kind and config, one can get a specific implementation.
-
-## Reference
-
-- [paddle_inference_api.h](./paddle_inference_api.h)
-- [some demos](./demo_ci)
diff --git a/paddle/fluid/inference/api/high_level_api_cn.md b/paddle/fluid/inference/api/high_level_api_cn.md
deleted file mode 100644
index 6fb4a55f200eb..0000000000000
--- a/paddle/fluid/inference/api/high_level_api_cn.md
+++ /dev/null
@@ -1,84 +0,0 @@
-# Paddle 预测 API
-
-为了更简单方便的预测部署，Fluid 提供了一套高层 API 用来隐藏底层不同的优化实现。
-
-预测库包含:
-
-- 头文件 `paddle_inference_api.h` 定义了所有的接口
-- 库文件 `libpaddle_inference.so/.a(Linux/Mac)` `libpaddle_inference.lib/paddle_inference.dll(Windows)`
-
-下面是详细的一些 API 概念介绍
-
-## PaddleTensor
-
-PaddleTensor 定义了预测最基本的输入输出的数据格式，其定义是
-
-```c++
-struct PaddleTensor {
-  std::string name;  // variable name.
-  std::vector<int> shape;
-  PaddleBuf data;  // blob of data.
-  PaddleDType dtype;
-};
-```
-
-- `name` 用于指定输入数据对应的 模型中variable 的名字 （暂时没有用，但会在后续支持任意 target 时启用）
-- `shape` 表示一个 Tensor 的 shape
-- `data`  数据以连续内存的方式存储在`PaddleBuf` 中，`PaddleBuf` 可以接收外面的数据或者独立`malloc`内存，详细可以参考头文件中相关定义。
-- `dtype` 表示 Tensor 的数据类型
-
-## engine
-
-高层 API 底层有多种优化实现，我们称之为 engine，目前有两种 engine
-
-- 原生 engine，由 paddle 原生的 forward operator 组成，可以天然支持所有paddle 训练出的模型，
-- TensorRT mixed engine，用子图的方式支持了 [TensorRT](https://developer.nvidia.com/tensorrt) ，支持所有paddle 模型，并自动切割部分计算子图到 TensorRT 上加速（WIP）
-
-其实现为
-
-```c++
-enum class PaddleEngineKind {
-  kNative = 0,       // Use the native Fluid facility.
-  kAutoMixedTensorRT // Automatically mixing TensorRT with the Fluid ops.
-};
-```
-
-## 预测部署过程
-
-总体上分为以下步骤
-
-1. 用合适的配置创建 `PaddlePredictor`
-2. 创建输入用的 `PaddleTensor`，传入到 `PaddlePredictor` 中
-3. 获取输出的 `PaddleTensor` ，将结果取出
-
-下面完整演示一个简单的模型，部分细节代码隐去
-
-```c++
-#include "paddle_inference_api.h"
-
-// 创建一个 config，并修改相关设置
-paddle::NativeConfig config;
-config.model_dir = "xxx";
-config.use_gpu = false;
-// 创建一个原生的 PaddlePredictor
-auto predictor =
-      paddle::CreatePaddlePredictor<paddle::NativeConfig, paddle::PaddleEngineKind::kNative>(config);
-// 创建输入 tensor
-int64_t data[4] = {1, 2, 3, 4};
-paddle::PaddleTensor tensor{.name = "",
-                            .shape = std::vector<int>({4, 1}),
-                            .data = paddle::PaddleBuf(data, sizeof(data)),
-                            .dtype = paddle::PaddleDType::INT64};
-// 创建输出 tensor，输出 tensor 的内存可以复用
-std::vector<paddle::PaddleTensor> outputs;
-// 执行预测
-CHECK(predictor->Run(slots, &outputs));
-// 获取 outputs ...
-```
-
-编译时，联编 `libpaddle_inference.a/.so(Linux/Mac)` 或 `libpaddle_inference.lib/paddle_inference.dll(Windows)` 便可。
-
-## 详细代码参考
-
-- [inference demos](./demo_ci)
-- [复杂单线程/多线程例子](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/fluid/inference/api/test_api_impl.cc)
diff --git a/paddle/fluid/inference/engine.h b/paddle/fluid/inference/engine.h
deleted file mode 100644
index e29162cf5b23b..0000000000000
--- a/paddle/fluid/inference/engine.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#pragma once
-
-#include <string>
-
-namespace paddle {
-namespace inference {
-
-struct Buffer;
-enum class DeviceType { UNK = -1, CPU, GPU };
-
-/*
- * EngineBase is the base class of all inference engines. An inference engine
- * takes a paddle program as input, and outputs the result in fluid Tensor
- * format. It can be used to optimize performance of computation sub-blocks, for
- * example, break down the original block into sub-blocks and execute each
- * sub-blocks in different engines.
- *
- * For example:
- *   When inference, the resnet50 model can put most of the model into subgraph
- * and run it on a TensorRT engine.
- *
- * There are several engines such as TensorRT and other frameworks, so an
- * EngineBase is put forward to give an unified interface for all the
- * different engine implemention.
- */
-class EngineBase {
- public:
-  using DescType = ::paddle::framework::proto::BlockDesc;
-
-  // Build the model and do some preparation, for example, in TensorRT, run
-  // createInferBuilder, buildCudaEngine.
-  virtual void Build(const DescType& paddle_model) = 0;
-
-  // Execute the engine, that will run the inference network.
-  virtual void Execute(int batch_size) = 0;
-
-  virtual ~EngineBase() {}
-};  // class EngineBase
-
-struct Buffer {
-  void* buffer{nullptr};               // buffer should be allocated only once.
-  size_t max_size;                     // buffer allocated space.
-  size_t size;                         // data size.
-  DeviceType device{DeviceType::UNK};  // tells which device this buffer is on.
-};
-
-}  // namespace inference
-}  // namespace paddle
diff --git a/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt b/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt
index d09c344ca1643..a52d6b1c39df3 100755
--- a/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt
+++ b/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt
@@ -99,7 +99,6 @@ list(
   merge_layernorm_op.cc
   skip_merge_layernorm_op.cc
   generic_and_custom_plugin_creater.cc
-  fused_lookup_tables_op.cc
   lookup_table_op.cc
   elementwiseadd_transpose_op.cc
   skip_groupnorm_act_op.cc
diff --git a/paddle/fluid/inference/tensorrt/convert/fused_lookup_tables_op.cc b/paddle/fluid/inference/tensorrt/convert/fused_lookup_tables_op.cc
deleted file mode 100644
index aaeea2d725809..0000000000000
--- a/paddle/fluid/inference/tensorrt/convert/fused_lookup_tables_op.cc
+++ /dev/null
@@ -1,115 +0,0 @@
-/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include "paddle/fluid/inference/tensorrt/convert/op_converter.h"
-#include "paddle/fluid/inference/tensorrt/convert/utils.h"
-#include "paddle/fluid/inference/tensorrt/plugin/lookup_table.h"
-
-namespace paddle {
-namespace inference {
-namespace tensorrt {
-
-class FusedLookupTablesOpConverter : public OpConverter {
- public:
-  void operator()(const framework::proto::OpDesc& op,
-                  const framework::Scope& scope,
-                  bool test_mode) override {
-    if (!engine_->with_dynamic_shape()) {
-      PADDLE_THROW(
-          platform::errors::Fatal("lookup_table_op must with dynamic shape"));
-    }
-
-    framework::OpDesc op_desc(op, nullptr);
-    auto ids_name = op_desc.Input("Ids").front();
-    auto w_name = op_desc.Input("W").front();
-    auto output_name = op_desc.Output("Out").front();
-    bool enable_int8 = op_desc.HasAttr("enable_int8");
-    std::vector<nvinfer1::ITensor*> plugin_inputs;
-
-    auto ids_dims = engine_->GetITensor(ids_name)->getDimensions();
-    if (ids_dims.d[ids_dims.nbDims - 1] == 1) {
-      nvinfer1::Dims new_ids_dims;
-      new_ids_dims.nbDims = ids_dims.nbDims - 1;
-      for (int i = 0; i < ids_dims.nbDims - 1; i++) {
-        new_ids_dims.d[i] = 0;
-      }
-      auto* reshape_layer = TRT_ENGINE_ADD_LAYER(
-          engine_, Shuffle, *(engine_->GetITensor(ids_name)));
-      reshape_layer->setReshapeDimensions(new_ids_dims);
-      reshape_layer->setName(
-          ("lookup_table: Shuffle (Output: " + output_name + ")").c_str());
-      plugin_inputs.push_back(reshape_layer->getOutput(0));
-    } else {
-      plugin_inputs.push_back(engine_->GetITensor(ids_name));
-    }
-
-    TensorRTEngine::Weight weight;
-    auto* w_var = scope.FindVar(w_name);
-    auto* w_tensor = w_var->GetMutable<phi::DenseTensor>();
-    auto w_dims = w_tensor->dims();
-    weight = engine_->GetTrtWeight(w_name, *w_tensor);
-    auto weight_size = phi::product(w_dims);
-    int output_fp16;
-    if (engine_->precision() == AnalysisConfig::Precision::kFloat32) {
-      output_fp16 = 0;
-    } else {
-      output_fp16 = 1;
-    }
-
-    int32_t weight_width = static_cast<int32_t>(w_dims[1]);
-
-    std::vector<nvinfer1::PluginField> fields;
-    fields.emplace_back("lookup_table_weight",
-                        weight.get().values,
-                        GetPluginFieldType(weight.get().type),
-                        static_cast<int32_t>(weight_size));
-    fields.emplace_back("lookup_table_weight_width",
-                        &weight_width,
-                        nvinfer1::PluginFieldType::kINT32,
-                        1);
-    fields.emplace_back(
-        "output_fp16", &output_fp16, nvinfer1::PluginFieldType::kINT32, 1);
-    nvinfer1::PluginFieldCollection* plugin_ptr =
-        static_cast<nvinfer1::PluginFieldCollection*>(
-            malloc(sizeof(*plugin_ptr) +
-                   fields.size() * sizeof(nvinfer1::PluginField)));
-    plugin_ptr->nbFields = static_cast<int>(fields.size());
-    plugin_ptr->fields = fields.data();
-    auto creator =
-        GetPluginRegistry()->getPluginCreator("LookupTablePluginDynamic", "1");
-    auto plugin_obj =
-        creator->createPlugin("LookupTablePluginDynamic", plugin_ptr);
-
-    auto plugin_layer = engine_->network()->addPluginV2(
-        plugin_inputs.data(), plugin_inputs.size(), *plugin_obj);
-
-    plugin_layer->setName(
-        ("lookup_table: (Output: " + output_name + ")").c_str());
-    engine_->SetITensor(output_name, plugin_layer->getOutput(0));
-    free(plugin_ptr);
-    if (enable_int8) {
-      float out_scale =
-          PADDLE_GET_CONST(float, op_desc.GetAttr("out_threshold"));
-      engine_->SetTensorDynamicRange(plugin_layer->getOutput(0), out_scale);
-    }
-  }
-};
-
-}  // namespace tensorrt
-}  // namespace inference
-}  // namespace paddle
-
-// NOTE(liuyuanle): We will remove the implementation here later. Ref to
-// tensorrt/convert/lookup_table_op.cc.
-// REGISTER_TRT_OP_CONVERTER(lookup_table, FusedLookupTablesOpConverter);
diff --git a/paddle/fluid/inference/tensorrt/convert/prelu_op.cc b/paddle/fluid/inference/tensorrt/convert/prelu_op.cc
index 80a2ac46f44dc..2d2a804f394fd 100644
--- a/paddle/fluid/inference/tensorrt/convert/prelu_op.cc
+++ b/paddle/fluid/inference/tensorrt/convert/prelu_op.cc
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "paddle/fluid/inference/tensorrt/convert/op_converter.h"
-#include "paddle/fluid/inference/tensorrt/plugin/prelu_op_plugin.h"
 
 namespace paddle {
 namespace inference {
diff --git a/paddle/fluid/inference/tensorrt/engine.h b/paddle/fluid/inference/tensorrt/engine.h
index 3f4df60586573..a0d146c9b6dad 100644
--- a/paddle/fluid/inference/tensorrt/engine.h
+++ b/paddle/fluid/inference/tensorrt/engine.h
@@ -31,7 +31,6 @@ limitations under the License. */
 #include "paddle/fluid/framework/tensor.h"
 #include "paddle/fluid/framework/tensor_util.h"
 #include "paddle/fluid/inference/api/paddle_analysis_config.h"
-#include "paddle/fluid/inference/engine.h"
 #include "paddle/fluid/inference/tensorrt/helper.h"
 #include "paddle/fluid/inference/tensorrt/plugin/trt_plugin.h"
 #include "paddle/fluid/inference/tensorrt/trt_int8_calibrator.h"
diff --git a/paddle/fluid/inference/tensorrt/plugin/CMakeLists.txt b/paddle/fluid/inference/tensorrt/plugin/CMakeLists.txt
index e72ead0bbadb6..b1df5a733623e 100644
--- a/paddle/fluid/inference/tensorrt/plugin/CMakeLists.txt
+++ b/paddle/fluid/inference/tensorrt/plugin/CMakeLists.txt
@@ -4,7 +4,6 @@ list(
   trt_plugin.cc
   split_op_plugin.cu
   elementwise_op_plugin.cu
-  prelu_op_plugin.cu
   gelu_op_plugin.cu
   pool_op_plugin.cu
   swish_op_plugin.cu
@@ -40,7 +39,6 @@ list(
   preln_groupnorm_act_op_plugin.cu
   elementwiseadd_transpose_op_plugin.cu
   generic_plugin.cu
-  lookup_table.cu
   many_emb_layernorm_plugin.cu
   many_emb_layernorm_kernel.cu)
 
@@ -57,11 +55,7 @@ endif()
 nv_library(
   tensorrt_plugin
   SRCS ${TRT_FILES}
-  DEPS enforce
-       tensorrt_engine
-       prelu
-       tensor
-       bert_encoder_functor
+  DEPS enforce tensorrt_engine tensor bert_encoder_functor
        tensorrt_dynamic_shape_infermeta_factory
        tensorrt_plugin_arg_mapping_context)
 
diff --git a/paddle/fluid/inference/tensorrt/plugin/lookup_table.cu b/paddle/fluid/inference/tensorrt/plugin/lookup_table.cu
deleted file mode 100644
index 31d599bd2a67c..0000000000000
--- a/paddle/fluid/inference/tensorrt/plugin/lookup_table.cu
+++ /dev/null
@@ -1,348 +0,0 @@
-/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#pragma once
-#include "paddle/fluid/inference/tensorrt/plugin/lookup_table.h"
-
-namespace paddle {
-namespace inference {
-namespace tensorrt {
-namespace plugin {
-
-char const* PLUGINVERSION{"1"};
-char const* LOOKUPTABLEPLUGINNAME{"LookupTablePluginDynamic"};
-
-template <typename T, unsigned TPB>
-__global__ void lookup_table_kernel(int weight_height,
-                                    int32_t const* inputIds,
-                                    T const* wordEmb,
-                                    int32_t const wordSize,
-                                    T* output) {
-  // 1. lookup word and token of the block
-  // blockIdx.x = position in the sequence
-  // blockIdx.y = batch
-  // gridDim.x = S
-  // gridDim.y = B
-  __shared__ int wordId;
-  int32_t const seqPos = blockIdx.x + blockIdx.y * gridDim.x;
-  if (threadIdx.x == 0) {
-    wordId = inputIds[seqPos];
-  }
-  __syncthreads();
-
-  // 2. load word embeddings and add them toghether
-  // offset into embeddings is given by wordId * hidden_size
-  int32_t const woffset = wordId * weight_height;
-  // the output offset is given by b * (S*hidden_size) + s * hidden_size
-  int32_t const outOffset = seqPos * weight_height;
-  if (wordId >= 0 && wordId < wordSize) {
-    for (int it = threadIdx.x; it < weight_height; it += TPB) {
-      T const w(wordEmb[woffset + it]);
-      output[outOffset + it] = w;
-    }
-  } else {
-    printf(
-        "Error!!!!!!(LookupTablePlugin): ID cannot be lookup "
-        "table: ID < 0 or ID > max ");
-    return;
-  }
-}
-
-template <typename T>
-int lookup_table(cudaStream_t stream,
-                 int weight_height,
-                 int B,
-                 int S,
-                 int32_t const* inputIds,
-                 T const* wordEmb,
-                 int32_t const wordSize,
-                 T* output) {
-  constexpr int tpb = 256;
-  dim3 const grid(S, B, 1);
-  dim3 const block(tpb, 1, 1);
-  lookup_table_kernel<T, tpb><<<grid, block, 0, stream>>>(
-      weight_height, inputIds, wordEmb, wordSize, output);
-  return 0;
-}
-
-// Static class fields initialization
-nvinfer1::PluginFieldCollection LookupTablePluginDynamicCreator::mFC{};
-std::vector<nvinfer1::PluginField>
-    LookupTablePluginDynamicCreator::mPluginAttributes;
-
-LookupTablePluginDynamic::LookupTablePluginDynamic(
-    nvinfer1::DataType const type,
-    void* weight_dev,
-    int32_t weight_size,
-    int32_t width)
-    : mType(type),
-      mWeightDev(weight_dev),
-      mWeightSize(weight_size),
-      mWeightWidth(width) {}
-
-LookupTablePluginDynamic::LookupTablePluginDynamic(void const* data,
-                                                   size_t length) {
-  // Deserialize in the same order as serialization
-  deserialize_value(&data, &length, &mType);
-  deserialize_value(&data, &length, &mWeightSize);
-  deserialize_value(&data, &length, &mWeightWidth);
-  char const* d = static_cast<char const*>(data);
-  cudaMalloc(&mWeightDev, mWeightSize * getElementSize(mType));
-  cudaMemcpy(mWeightDev,
-             d,
-             mWeightSize * getElementSize(mType),
-             cudaMemcpyHostToDevice);
-}
-
-// IPluginV2DynamicExt Methods
-nvinfer1::IPluginV2DynamicExt* LookupTablePluginDynamic::clone()
-    const noexcept {
-  auto p = new LookupTablePluginDynamic(
-      mType, mWeightDev, mWeightSize, mWeightWidth);
-  p->setPluginNamespace(mNamespace.c_str());
-  return p;
-}
-
-nvinfer1::DimsExprs LookupTablePluginDynamic::getOutputDimensions(
-    int32_t outputIndex,
-    nvinfer1::DimsExprs const* inputs,
-    int32_t nbInputs,
-    nvinfer1::IExprBuilder& exprBuilder) noexcept {
-  nvinfer1::DimsExprs ret;
-  ret.nbDims = inputs[0].nbDims + 1;
-  for (int i = 0; i < inputs[0].nbDims; ++i) {
-    ret.d[i] = inputs[0].d[i];
-  }
-  ret.d[inputs[0].nbDims] = exprBuilder.constant(mWeightWidth);
-  return ret;
-}
-
-bool LookupTablePluginDynamic::supportsFormatCombination(
-    int32_t pos,
-    nvinfer1::PluginTensorDesc const* inOut,
-    int32_t nbInputs,
-    int32_t nbOutputs) noexcept {
-  nvinfer1::PluginTensorDesc const& desc = inOut[pos];
-  if (desc.format != nvinfer1::TensorFormat::kLINEAR) {
-    return false;
-  }
-  if (pos == 0) {
-    return desc.type == nvinfer1::DataType::kINT32;
-  }
-  if (pos == 1) {
-    if (mType == nvinfer1::DataType::kFLOAT) {
-      return desc.type == nvinfer1::DataType::kFLOAT;
-    } else {
-      return desc.type == nvinfer1::DataType::kHALF;
-    }
-  }
-}
-
-void LookupTablePluginDynamic::configurePlugin(
-    nvinfer1::DynamicPluginTensorDesc const* inputs,
-    int32_t nbInputs,
-    nvinfer1::DynamicPluginTensorDesc const* outputs,
-    int32_t nbOutputs) noexcept {}
-
-size_t LookupTablePluginDynamic::getWorkspaceSize(
-    nvinfer1::PluginTensorDesc const* inputs,
-    int32_t nbInputs,
-    nvinfer1::PluginTensorDesc const* outputs,
-    int32_t nbOutputs) const noexcept {
-  return 0;
-}
-
-int32_t LookupTablePluginDynamic::enqueue(
-    nvinfer1::PluginTensorDesc const* inputDesc,
-    nvinfer1::PluginTensorDesc const* outputDesc,
-    void const* const* inputs,
-    void* const* outputs,
-    void* workspace,
-    cudaStream_t stream) noexcept {
-  int32_t const batchSize = inputDesc->dims.d[0];
-  int32_t S;
-  if (inputDesc->dims.nbDims == 1) {
-    S = 1;
-  } else {
-    S = inputDesc->dims.d[1];
-  }
-  int32_t mWeightHeight = mWeightSize / mWeightWidth;
-  int32_t status = STATUS_FAILURE;
-  auto const inputIds = static_cast<int32_t const*>(inputs[0]);
-  if (mType == nvinfer1::DataType::kFLOAT) {
-    auto output = static_cast<float*>(outputs[0]);
-    auto const Weight = static_cast<const float*>(mWeightDev);
-    status = lookup_table<float>(stream,
-                                 static_cast<int32_t>(mWeightWidth),
-                                 batchSize,
-                                 S,
-                                 inputIds,
-                                 Weight,
-                                 mWeightHeight,
-                                 output);
-  } else if (mType == nvinfer1::DataType::kHALF) {
-    auto output = static_cast<half*>(outputs[0]);
-    auto const Weight = static_cast<const half*>(mWeightDev);
-    status = lookup_table<half>(stream,
-                                static_cast<int32_t>(mWeightWidth),
-                                batchSize,
-                                S,
-                                inputIds,
-                                Weight,
-                                mWeightHeight,
-                                output);
-  }
-  return status;
-}
-
-// IPluginV2Ext Methods
-nvinfer1::DataType LookupTablePluginDynamic::getOutputDataType(
-    int32_t index,
-    nvinfer1::DataType const* inputTypes,
-    int32_t nbInputs) const noexcept {
-  if (index == 0) {
-    assert(mType == nvinfer1::DataType::kHALF ||
-           mType == nvinfer1::DataType::kFLOAT);
-    return mType;
-  }
-}
-
-// IPluginV2 Methods
-char const* LookupTablePluginDynamic::getPluginType() const noexcept {
-  return LOOKUPTABLEPLUGINNAME;
-}
-
-char const* LookupTablePluginDynamic::getPluginVersion() const noexcept {
-  return PLUGINVERSION;
-}
-
-int32_t LookupTablePluginDynamic::getNbOutputs() const noexcept { return 1; }
-
-int32_t LookupTablePluginDynamic::initialize() noexcept { return 0; }
-
-void LookupTablePluginDynamic::terminate() noexcept { cudaFree(mWeightDev); }
-
-size_t LookupTablePluginDynamic::getSerializationSize() const noexcept {
-  size_t const wordSize = getElementSize(mType);
-  return sizeof(mType)              //
-         + sizeof(mWeightSize)      //
-         + sizeof(mWeightWidth)     //
-         + wordSize * mWeightSize;  //
-}
-
-void LookupTablePluginDynamic::serialize(void* buffer) const noexcept {
-  serialize_value(&buffer, mType);
-  serialize_value(&buffer, mWeightSize);
-  serialize_value(&buffer, mWeightWidth);
-  char* d = static_cast<char*>(buffer);
-  size_t const wordSize = getElementSize(mType);
-  serFromDev(&d, static_cast<char*>(mWeightDev), mWeightSize * wordSize);
-}
-
-void LookupTablePluginDynamic::destroy() noexcept {
-  // This gets called when the network containing plugin is destroyed
-  delete this;
-}
-
-void LookupTablePluginDynamic::setPluginNamespace(
-    char const* libNamespace) noexcept {
-  mNamespace = libNamespace;
-}
-
-char const* LookupTablePluginDynamic::getPluginNamespace() const noexcept {
-  return mNamespace.c_str();
-}
-
-LookupTablePluginDynamicCreator::LookupTablePluginDynamicCreator() {}
-
-char const* LookupTablePluginDynamicCreator::getPluginName() const noexcept {
-  return LOOKUPTABLEPLUGINNAME;
-}
-
-char const* LookupTablePluginDynamicCreator::getPluginVersion() const noexcept {
-  return PLUGINVERSION;
-}
-
-nvinfer1::PluginFieldCollection const*
-LookupTablePluginDynamicCreator::getFieldNames() noexcept {
-  return &mFC;
-}
-
-bool initializeFields(nvinfer1::PluginFieldCollection const* fc,
-                      nvinfer1::Weights* weight,
-                      int32_t& mWeightWidth) {  // NOLINT
-  bool output_fp16 = false;
-  for (int32_t i = 0; i < fc->nbFields; i++) {
-    std::string field_name(fc->fields[i].name);
-    if (field_name.compare("lookup_table_weight") == 0) {
-      weight->values = fc->fields[i].data;
-      weight->count = fc->fields[i].length;
-      weight->type = fieldTypeToDataType(fc->fields[i].type);
-    }
-    if (field_name.compare("lookup_table_weight_width") == 0) {
-      assert(fc->fields[i].type == nvinfer1::PluginFieldType::kINT32);
-      mWeightWidth = const_cast<int32_t*>(
-          static_cast<int32_t const*>(fc->fields[i].data))[0];  // NOLINT
-    }
-    if (field_name.compare("output_fp16") == 0) {
-      assert(fc->fields[i].type == nvinfer1::PluginFieldType::kINT32);
-      output_fp16 = static_cast<int32_t const*>(fc->fields[i].data)[0] != 0;
-    }
-  }
-  return output_fp16;
-}
-
-nvinfer1::IPluginV2* LookupTablePluginDynamicCreator::createPlugin(
-    char const* name, const nvinfer1::PluginFieldCollection* fc) noexcept {
-  nvinfer1::Weights weight;
-  int32_t mWeightWidth;
-  bool output_fp16 = initializeFields(fc, &weight, mWeightWidth);
-  nvinfer1::DataType type;
-  if (output_fp16) {
-    type = nvinfer1::DataType::kHALF;
-  } else {
-    type = nvinfer1::DataType::kFLOAT;
-  }
-  WeightsWithOwnership mWeight;
-  mWeight.convertAndCopy(weight, type);
-  void* cudaMem{nullptr};
-  cudaMalloc(&cudaMem, getWeightsSize(mWeight, type));
-  cudaMemcpy(cudaMem,
-             mWeight.values,
-             getWeightsSize(mWeight, type),
-             cudaMemcpyHostToDevice);
-  LookupTablePluginDynamic* p =
-      new LookupTablePluginDynamic(type, cudaMem, mWeight.count, mWeightWidth);
-  return p;
-}
-
-nvinfer1::IPluginV2* LookupTablePluginDynamicCreator::deserializePlugin(
-    char const* name, void const* serialData, size_t serialLength) noexcept {
-  return new LookupTablePluginDynamic(serialData, serialLength);
-}
-
-void LookupTablePluginDynamicCreator::setPluginNamespace(
-    char const* libNamespace) noexcept {
-  mNamespace = libNamespace;
-}
-
-char const* LookupTablePluginDynamicCreator::getPluginNamespace()
-    const noexcept {
-  return mNamespace.c_str();
-}
-
-}  // namespace plugin
-}  // namespace tensorrt
-}  // namespace inference
-}  // namespace paddle
diff --git a/paddle/fluid/inference/tensorrt/plugin/lookup_table.h b/paddle/fluid/inference/tensorrt/plugin/lookup_table.h
deleted file mode 100644
index 84f6282d4f239..0000000000000
--- a/paddle/fluid/inference/tensorrt/plugin/lookup_table.h
+++ /dev/null
@@ -1,126 +0,0 @@
-/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#pragma once
-#include <cuda.h>
-#include "NvInferPlugin.h"
-#include "NvInferRuntime.h"
-#include "paddle/fluid/inference/tensorrt/plugin/common/bertCommon.h"
-#include "paddle/fluid/inference/tensorrt/plugin/common/serialize.h"
-#include "paddle/fluid/inference/tensorrt/plugin/trt_plugin.h"
-#include "paddle/fluid/platform/enforce.h"
-
-namespace paddle {
-namespace inference {
-namespace tensorrt {
-namespace plugin {
-
-class LookupTablePluginDynamic : public nvinfer1::IPluginV2DynamicExt {
- public:
-  LookupTablePluginDynamic(nvinfer1::DataType const type,
-                           void* weight_dev,
-                           int32_t weight_size,
-                           int32_t width);
-
-  LookupTablePluginDynamic(void const* data, size_t length);
-
-  // It doesn't make sense to make EmbLayerNormVarSeqlenPlugin without
-  // arguments, so we delete default constructor.
-  LookupTablePluginDynamic() = delete;
-
-  // IPluginV2DynamicExt Methods
-  bool supportsFormatCombination(int32_t pos,
-                                 nvinfer1::PluginTensorDesc const* inOut,
-                                 int32_t nbInputs,
-                                 int32_t nbOutputs) noexcept override;
-  size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs,
-                          int32_t nbInputs,
-                          nvinfer1::PluginTensorDesc const* outputs,
-                          int32_t nbOutputs) const noexcept override;
-
-  // IPluginV2Ext Methods
-  nvinfer1::DataType getOutputDataType(
-      int32_t index,
-      nvinfer1::DataType const* inputTypes,
-      int32_t nbInputs) const noexcept override;
-
-  // IPluginV2 Methods
-  nvinfer1::IPluginV2DynamicExt* clone() const noexcept override;
-  nvinfer1::DimsExprs getOutputDimensions(
-      int32_t outputIndex,
-      const nvinfer1::DimsExprs* inputs,
-      int32_t nbInputs,
-      nvinfer1::IExprBuilder& exprBuilder) noexcept override;
-  void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in,
-                       int32_t nbInputs,
-                       nvinfer1::DynamicPluginTensorDesc const* out,
-                       int32_t nbOutputs) noexcept override;
-  char const* getPluginType() const noexcept override;
-  int32_t getNbOutputs() const noexcept override;
-  size_t getSerializationSize() const noexcept override;
-  void serialize(void* buffer) const noexcept override;
-  void destroy() noexcept override;
-  char const* getPluginNamespace() const noexcept override;
-  void setPluginNamespace(char const* pluginNamespace) noexcept override;
-  int32_t enqueue(nvinfer1::PluginTensorDesc const* inputDesc,
-                  nvinfer1::PluginTensorDesc const* outputDesc,
-                  void const* const* inputs,
-                  void* const* outputs,
-                  void* workspace,
-                  cudaStream_t stream) noexcept override;
-
-  int32_t initialize() noexcept override;
-  void terminate() noexcept override;
-  char const* getPluginVersion() const noexcept override;
-
- protected:
-  std::string mNamespace;
-  nvinfer1::DataType mType;
-  void* mWeightDev{nullptr};
-  int32_t mWeightSize;
-  int32_t mWeightWidth;
-};
-
-class LookupTablePluginDynamicCreator : public nvinfer1::IPluginCreator {
- public:
-  LookupTablePluginDynamicCreator();
-
-  char const* getPluginName() const noexcept override;
-
-  const nvinfer1::PluginFieldCollection* getFieldNames() noexcept override;
-
-  void setPluginNamespace(char const* pluginNamespace) noexcept override;
-
-  char const* getPluginNamespace() const noexcept override;
-
-  nvinfer1::IPluginV2* createPlugin(
-      char const* name,
-      const nvinfer1::PluginFieldCollection* fc) noexcept override;
-  char const* getPluginVersion() const noexcept override;
-  nvinfer1::IPluginV2* deserializePlugin(char const* name,
-                                         void const* serialData,
-                                         size_t serialLength) noexcept override;
-
- protected:
-  static nvinfer1::PluginFieldCollection mFC;
-  static std::vector<nvinfer1::PluginField> mPluginAttributes;
-  std::string mNamespace;
-};
-
-REGISTER_TRT_PLUGIN_V2(LookupTablePluginDynamicCreator);
-
-}  // namespace plugin
-}  // namespace tensorrt
-}  // namespace inference
-}  // namespace paddle
diff --git a/paddle/fluid/inference/tensorrt/plugin/prelu_op_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/prelu_op_plugin.cu
deleted file mode 100644
index 85d9b71d34b3c..0000000000000
--- a/paddle/fluid/inference/tensorrt/plugin/prelu_op_plugin.cu
+++ /dev/null
@@ -1,224 +0,0 @@
-// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <stdio.h>
-
-#include <cassert>
-#include <vector>
-
-#include "glog/logging.h"
-#include "paddle/fluid/inference/tensorrt/plugin/prelu_op_plugin.h"
-#include "paddle/fluid/operators/math/prelu.h"
-
-namespace paddle {
-namespace inference {
-namespace tensorrt {
-namespace plugin {
-
-int PReluPlugin::initialize() TRT_NOEXCEPT {
-  cudaMalloc(&p_gpu_weight_, sizeof(float) * weight_.size());
-  cudaMemcpy(p_gpu_weight_,
-             weight_.data(),
-             weight_.size() * sizeof(float),
-             cudaMemcpyHostToDevice);
-  return 0;
-}
-
-void PReluPlugin::terminate() TRT_NOEXCEPT {
-  if (p_gpu_weight_) {
-    cudaFree(p_gpu_weight_);
-    p_gpu_weight_ = nullptr;
-  }
-}
-
-nvinfer1::Dims PReluPlugin::getOutputDimensions(int index,
-                                                const nvinfer1::Dims *inputDims,
-                                                int nbInputs) TRT_NOEXCEPT {
-  assert(nbInputs == 1);
-  assert(index < this->getNbOutputs());
-  nvinfer1::Dims const &input_dims = inputDims[0];
-  nvinfer1::Dims output_dims = input_dims;
-  return output_dims;
-}
-
-int PReluPlugin::enqueue(int batch_size,
-                         const void *const *inputs,
-#if IS_TRT_VERSION_LT(8000)
-                         void **outputs,
-                         void *workspace,
-                         cudaStream_t stream) {
-#else
-                         void *const *outputs,
-                         void *workspace,
-                         cudaStream_t stream) TRT_NOEXCEPT {
-#endif
-  // input dims is CHW.
-  const auto &input_dims = this->getInputDims(0);
-  const float *input = reinterpret_cast<const float *>(inputs[0]);
-  // const float *alpha = reinterpret_cast<const float *>(alpha_.get().values);
-  const float *alpha = p_gpu_weight_;
-  float *const output = reinterpret_cast<float *const *>(outputs)[0];
-  int numel = 1;
-  for (int i = 0; i < input_dims.nbDims; i++) {
-    numel *= input_dims.d[i];
-  }
-
-  if (mode_ == "channel") {
-    bool channel_last = data_format_ == "NHWC";
-    operators::math::PreluChannelWiseDirectCUDAFunctor<float>
-        prelu_channel_wise;
-    prelu_channel_wise(stream,
-                       input,
-                       alpha,
-                       output,
-                       input_dims.d[0],
-                       input_dims.d[1],
-                       channel_last,
-                       numel);
-  } else if (mode_ == "element") {
-    operators::math::PreluElementWiseDirectCUDAFunctor<float>
-        prelu_element_wise;
-    prelu_element_wise(stream, input, alpha, output, input_dims.d[0], numel);
-  } else {
-    operators::math::PreluScalarDirectCUDAFunctor<float> prelu_scalar;
-    prelu_scalar(stream, input, alpha, output, numel);
-  }
-  return cudaGetLastError() != cudaSuccess;
-}
-
-#if IS_TRT_VERSION_GE(6000)
-
-void PReluPluginDynamic::terminate() TRT_NOEXCEPT {
-  if (p_gpu_weight_) {
-    cudaFree(p_gpu_weight_);
-  }
-}
-
-int PReluPluginDynamic::initialize() TRT_NOEXCEPT {
-  cudaMalloc(&p_gpu_weight_, sizeof(float) * weight_.size());
-  cudaMemcpy(p_gpu_weight_,
-             weight_.data(),
-             weight_.size() * sizeof(float),
-             cudaMemcpyHostToDevice);
-  return 0;
-}
-
-PReluPluginDynamic::PReluPluginDynamic(void const *serialData,
-                                       size_t serialLength) {
-  DeserializeValue(&serialData, &serialLength, &weight_);
-  const char *prelu_mode;
-  DeserializeValue(&serialData, &serialLength, &prelu_mode);
-  mode_ = std::string(prelu_mode);
-}
-
-size_t PReluPluginDynamic::getSerializationSize() const TRT_NOEXCEPT {
-  return SerializedSize(mode_.c_str()) + SerializedSize(weight_);
-}
-
-void PReluPluginDynamic::serialize(void *buffer) const TRT_NOEXCEPT {
-  SerializeValue(&buffer, weight_);
-  SerializeValue(&buffer, mode_.c_str());
-}
-
-nvinfer1::DimsExprs PReluPluginDynamic::getOutputDimensions(
-    int output_index,
-    const nvinfer1::DimsExprs *inputs,
-    int nb_inputs,
-    nvinfer1::IExprBuilder &expr_builder) TRT_NOEXCEPT {
-  return inputs[0];
-}
-
-bool PReluPluginDynamic::supportsFormatCombination(
-    int pos,
-    const nvinfer1::PluginTensorDesc *in_out,
-    int nb_inputs,
-    int nb_outputs) TRT_NOEXCEPT {
-  PADDLE_ENFORCE_NOT_NULL(
-      in_out,
-      platform::errors::InvalidArgument(
-          "The input of swish plugin shoule not be nullptr."));
-
-  PADDLE_ENFORCE_LT(
-      pos,
-      nb_inputs + nb_outputs,
-      platform::errors::InvalidArgument("The pos(%d) should be less than the "
-                                        "num(%d) of the input and the output.",
-                                        pos,
-                                        nb_inputs + nb_outputs));
-  (in_out && pos < (nb_inputs + nb_outputs));
-
-  return ((in_out[pos].type == nvinfer1::DataType::kFLOAT) &&
-          in_out[pos].format == nvinfer1::PluginFormat::kLINEAR);
-}
-
-nvinfer1::DataType PReluPluginDynamic::getOutputDataType(
-    int index,
-    const nvinfer1::DataType *input_types,
-    int nb_inputs) const TRT_NOEXCEPT {
-  PADDLE_ENFORCE_EQ(index,
-                    0,
-                    platform::errors::InvalidArgument(
-                        "The PRelu Plugin only has one input, so the "
-                        "index value should be 0, but get %d.",
-                        index));
-  PADDLE_ENFORCE_EQ((input_types[0] == nvinfer1::DataType::kFLOAT),
-                    true,
-                    platform::errors::InvalidArgument(
-                        "The input type should be half or float"));
-  return input_types[0];
-}
-
-int PReluPluginDynamic::enqueue(const nvinfer1::PluginTensorDesc *input_desc,
-                                const nvinfer1::PluginTensorDesc *output_desc,
-                                const void *const *inputs,
-                                void *const *outputs,
-                                void *workspace,
-                                cudaStream_t stream) TRT_NOEXCEPT {
-  auto input_dims = input_desc[0].dims;
-  const float *alpha = p_gpu_weight_;
-  const float *input = static_cast<const float *>(inputs[0]);
-  float *output = static_cast<float *>(outputs[0]);
-  int numel = 1;
-  for (int i = 0; i < input_dims.nbDims; i++) {
-    numel *= input_dims.d[i];
-  }
-
-  if (mode_ == "channel") {
-    bool channel_last = data_format_ == "NHWC";
-    operators::math::PreluChannelWiseDirectCUDAFunctor<float>
-        prelu_channel_wise;
-    prelu_channel_wise(stream,
-                       input,
-                       alpha,
-                       output,
-                       input_dims.d[0],
-                       input_dims.d[1],
-                       channel_last,
-                       numel);
-  } else if (mode_ == "element") {
-    operators::math::PreluElementWiseDirectCUDAFunctor<float>
-        prelu_element_wise;
-    prelu_element_wise(stream, input, alpha, output, input_dims.d[0], numel);
-  } else {
-    operators::math::PreluScalarDirectCUDAFunctor<float> prelu_scalar;
-    prelu_scalar(stream, input, alpha, output, numel);
-  }
-  return cudaGetLastError() != cudaSuccess;
-}
-#endif
-
-}  // namespace plugin
-}  // namespace tensorrt
-}  // namespace inference
-}  // namespace paddle
diff --git a/paddle/fluid/inference/tensorrt/plugin/prelu_op_plugin.h b/paddle/fluid/inference/tensorrt/plugin/prelu_op_plugin.h
deleted file mode 100644
index 921f0091f333b..0000000000000
--- a/paddle/fluid/inference/tensorrt/plugin/prelu_op_plugin.h
+++ /dev/null
@@ -1,217 +0,0 @@
-// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#pragma once
-
-#include <algorithm>
-#include <string>
-#include <vector>
-
-#include "paddle/fluid/framework/tensor.h"
-#include "paddle/fluid/framework/tensor_util.h"
-#include "paddle/fluid/inference/tensorrt/engine.h"
-#include "paddle/fluid/inference/tensorrt/plugin/trt_plugin.h"
-
-namespace paddle {
-namespace inference {
-namespace tensorrt {
-namespace plugin {
-
-class PReluPlugin : public PluginTensorRT {
-  std::vector<float> weight_;
-  float* p_gpu_weight_;
-  std::string mode_;
-  std::string data_format_;
-
- public:
-  size_t getSerializationSize() const TRT_NOEXCEPT override {
-    return getBaseSerializationSize() + SerializedSize(mode_.c_str()) +
-           SerializedSize(data_format_.c_str()) + SerializedSize(weight_);
-  }
-
-  // TRT will call this func when we need to serialize the configuration of
-  // tensorrt.
-  // It should not be called by users.
-  void serialize(void* buffer) const TRT_NOEXCEPT override {
-    serializeBase(buffer);
-    SerializeValue(&buffer, weight_);
-    SerializeValue(&buffer, mode_.c_str());
-    SerializeValue(&buffer, data_format_.c_str());
-  }
-
-  PReluPlugin(const float* weight,
-              const int weight_num,
-              std::string const& mode,
-              std::string const& data_format)
-      : mode_(mode), data_format_(data_format) {
-    weight_.resize(weight_num);
-    std::copy(weight, weight + weight_num, weight_.data());
-  }
-
-  // It was used for tensorrt deserialization.
-  // It should not be called by users.
-  PReluPlugin(void const* serialData, size_t serialLength) {
-    deserializeBase(serialData, serialLength);
-    DeserializeValue(&serialData, &serialLength, &weight_);
-    const char* prelu_mode;
-    DeserializeValue(&serialData, &serialLength, &prelu_mode);
-    mode_ = std::string(prelu_mode);
-    const char* prelu_data_format;
-    DeserializeValue(&serialData, &serialLength, &prelu_data_format);
-    data_format_ = std::string(prelu_data_format);
-  }
-  ~PReluPlugin() {}
-  int initialize() TRT_NOEXCEPT override;
-  void terminate() TRT_NOEXCEPT override;
-
-  PReluPlugin* clone() const TRT_NOEXCEPT override {
-    auto* ptr =
-        new PReluPlugin(weight_.data(), weight_.size(), mode_, data_format_);
-    ptr->p_gpu_weight_ = p_gpu_weight_;
-    return ptr;
-  }
-
-  const char* getPluginType() const TRT_NOEXCEPT override {
-    return "prelu_plugin";
-  }
-  int getNbOutputs() const TRT_NOEXCEPT override { return 1; }
-  nvinfer1::Dims getOutputDimensions(int index,
-                                     const nvinfer1::Dims* inputs,
-                                     int nbInputDims) TRT_NOEXCEPT override;
-#if IS_TRT_VERSION_LT(8000)
-  int enqueue(int batchSize,
-              const void* const* inputs,
-              void** outputs,
-#else
-  int enqueue(int batchSize,
-              const void* const* inputs,
-              void* const* outputs,
-#endif
-              void* workspace,
-              cudaStream_t stream) TRT_NOEXCEPT override;
-};
-
-class PReluPluginCreator : public TensorRTPluginCreator {
- public:
-  const char* getPluginName() const TRT_NOEXCEPT override {
-    return "prelu_plugin";
-  }
-
-  const char* getPluginVersion() const TRT_NOEXCEPT override { return "1"; }
-
-  nvinfer1::IPluginV2* deserializePlugin(const char* name,
-                                         const void* serial_data,
-                                         size_t serial_length)
-      TRT_NOEXCEPT override {
-    return new PReluPlugin(serial_data, serial_length);
-  }
-};
-REGISTER_TRT_PLUGIN_V2(PReluPluginCreator);
-
-#if IS_TRT_VERSION_GE(6000)
-class PReluPluginDynamic : public DynamicPluginTensorRT {
- public:
-  PReluPluginDynamic(const float* weight,
-                     const int weight_num,
-                     std::string const& mode,
-                     std::string const& data_format)
-      : mode_(mode), data_format_(data_format) {
-    weight_.resize(weight_num);
-    std::copy(weight, weight + weight_num, weight_.data());
-  }
-
-  PReluPluginDynamic(void const* serialData, size_t serialLength);
-  ~PReluPluginDynamic() {}
-  nvinfer1::IPluginV2DynamicExt* clone() const TRT_NOEXCEPT override {
-    auto ptr = new PReluPluginDynamic(
-        weight_.data(), weight_.size(), mode_, data_format_);
-    ptr->p_gpu_weight_ = p_gpu_weight_;
-    return ptr;
-  }
-
-  const char* getPluginType() const TRT_NOEXCEPT override {
-    return "prelu_plugin_dynamic";
-  }
-  int getNbOutputs() const TRT_NOEXCEPT override { return 1; }
-  int initialize() TRT_NOEXCEPT override;
-  void terminate() TRT_NOEXCEPT override;
-
-  size_t getSerializationSize() const TRT_NOEXCEPT override;
-  void serialize(void* buffer) const TRT_NOEXCEPT override;
-
-  nvinfer1::DimsExprs getOutputDimensions(int output_index,
-                                          const nvinfer1::DimsExprs* inputs,
-                                          int nb_inputs,
-                                          nvinfer1::IExprBuilder& expr_builder)
-      TRT_NOEXCEPT override;
-
-  bool supportsFormatCombination(int pos,
-                                 const nvinfer1::PluginTensorDesc* inOut,
-                                 int nbInputs,
-                                 int nbOutputs) TRT_NOEXCEPT override;
-
-  void configurePlugin(const nvinfer1::DynamicPluginTensorDesc* in,
-                       int nbInputs,
-                       const nvinfer1::DynamicPluginTensorDesc* out,
-                       int nbOutputs) TRT_NOEXCEPT override {}
-
-  size_t getWorkspaceSize(const nvinfer1::PluginTensorDesc* inputs,
-                          int nbInputs,
-                          const nvinfer1::PluginTensorDesc* outputs,
-                          int nbOutputs) const TRT_NOEXCEPT override {
-    return 0;
-  }
-
-  int enqueue(const nvinfer1::PluginTensorDesc* inputDesc,
-              const nvinfer1::PluginTensorDesc* outputDesc,
-              const void* const* inputs,
-              void* const* outputs,
-              void* workspace,
-              cudaStream_t stream) TRT_NOEXCEPT override;
-  nvinfer1::DataType getOutputDataType(int index,
-                                       const nvinfer1::DataType* inputTypes,
-                                       int nbInputs) const
-      TRT_NOEXCEPT override;
-
-  void destroy() TRT_NOEXCEPT override { delete this; }
-
- private:
-  std::vector<float> weight_;
-  float* p_gpu_weight_;
-  std::string mode_;
-  std::string data_format_;
-};
-#endif
-
-class PReluPluginDynamicCreator : public TensorRTPluginCreator {
- public:
-  const char* getPluginName() const TRT_NOEXCEPT override {
-    return "prelu_plugin_dynamic";
-  }
-
-  const char* getPluginVersion() const TRT_NOEXCEPT override { return "1"; }
-
-  nvinfer1::IPluginV2* deserializePlugin(const char* name,
-                                         const void* serial_data,
-                                         size_t serial_length)
-      TRT_NOEXCEPT override {
-    return new PReluPluginDynamic(serial_data, serial_length);
-  }
-};
-REGISTER_TRT_PLUGIN_V2(PReluPluginDynamicCreator);
-
-}  // namespace plugin
-}  // namespace tensorrt
-}  // namespace inference
-}  // namespace paddle

From 98100fd22141b41c07696efb0da6a988975ef8b4 Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Tue, 16 May 2023 14:06:05 +0800
Subject: [PATCH 349/405] Add fill_constant_batch_size_like tests (#53736)

---
 .../test_fill_constant_batch_size_like.py          | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like.py b/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like.py
index d97fe6465dbba..bd077c87984ac 100644
--- a/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like.py
+++ b/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like.py
@@ -38,11 +38,12 @@ def fill_constant_batch_size_like(
     )
 
 
-class TestFillConstatnBatchSizeLike1(OpTest):
+class TestFillConstantBatchSizeLike1(OpTest):
     # test basic
     def setUp(self):
         self.op_type = "fill_constant_batch_size_like"
         self.python_api = fill_constant_batch_size_like
+        self.init_dtype()
         self.init_data()
 
         input = np.zeros(self.shape)
@@ -59,9 +60,11 @@ def setUp(self):
             'force_cpu': self.force_cpu,
         }
 
+    def init_dtype(self):
+        self.dtype = np.float32
+
     def init_data(self):
         self.shape = [10, 10]
-        self.dtype = np.float32
         self.value = 100
         self.input_dim_idx = 0
         self.output_dim_idx = 0
@@ -71,11 +74,16 @@ def test_check_output(self):
         self.check_output()
 
 
+class TestFillConstantBatchSizeLikeFP16Op(TestFillConstantBatchSizeLike1):
+    def init_dtype(self):
+        self.dtype = np.float16
+
+
 @unittest.skipIf(
     not core.is_compiled_with_cuda() or not core.supports_bfloat16(),
     "core is not compiled with CUDA or place do not support bfloat16",
 )
-class TestFillConstatnBatchSizeLikeBf16(OpTest):
+class TestFillConstantBatchSizeLikeBF16Op(OpTest):
     # test bf16
     def setUp(self):
         self.op_type = "fill_constant_batch_size_like"

From b133317572417ac1f9c6c82cd2cb9154b91356e1 Mon Sep 17 00:00:00 2001
From: Meteor Liu <smith_135@163.com>
Date: Tue, 16 May 2023 14:09:05 +0800
Subject: [PATCH 350/405] [dygraph]remove legacy code : _in_eager_mode_ and
 _in_eager_without_dygraph_check() (#53761)

* remove _in_eager_mode_

* remove _in_eager_mode_
---
 python/paddle/autograd/backward_mode.py       |  12 +-
 python/paddle/fluid/data_feeder.py            |   1 -
 python/paddle/fluid/dygraph/base.py           |  51 +--
 python/paddle/fluid/dygraph/math_op_patch.py  | 293 +-----------------
 .../fluid/dygraph/tensor_patch_methods.py     |  92 ++----
 python/paddle/fluid/framework.py              |  10 +-
 python/paddle/fluid/layer_helper_base.py      |   1 -
 python/paddle/fluid/reader.py                 |  10 +-
 .../tests/unittests/test_run_program_op.py    |  12 +-
 python/paddle/optimizer/optimizer.py          |  12 +-
 python/paddle/tensor/creation.py              |   8 +-
 test/legacy_test/test_model.py                |   2 +-
 12 files changed, 56 insertions(+), 448 deletions(-)

diff --git a/python/paddle/autograd/backward_mode.py b/python/paddle/autograd/backward_mode.py
index 767a59f70cab6..5b61a91a15035 100644
--- a/python/paddle/autograd/backward_mode.py
+++ b/python/paddle/autograd/backward_mode.py
@@ -107,10 +107,7 @@ def check_tensors(in_out_list, name):
                     each_tensor, (paddle.Tensor, core.eager.Tensor)
                 ), "The argument 'grad_tensors' of paddle.autograd.backward is invalid, it can be 'None', 'paddle.Tensor' or 'list[None/paddle.Tensor]'."
     else:
-        if framework.global_var._in_eager_mode_:
-            grad_tensors = []
-        else:
-            grad_tensors = [None] * len(tensors)
+        grad_tensors = []
 
     if len(grad_tensors) > 0:
         assert len(tensors) == len(
@@ -119,9 +116,4 @@ def check_tensors(in_out_list, name):
 
     assert isinstance(retain_graph, bool), "retain_graph must be True or False"
 
-    if framework.global_var._in_eager_mode_:
-        core.eager.run_backward(tensors, grad_tensors, retain_graph)
-    else:
-        core.dygraph_run_backward(
-            tensors, grad_tensors, retain_graph, framework._dygraph_tracer()
-        )
+    core.eager.run_backward(tensors, grad_tensors, retain_graph)
diff --git a/python/paddle/fluid/data_feeder.py b/python/paddle/fluid/data_feeder.py
index 908fedf7c4b9c..ef711fb4fa1c0 100644
--- a/python/paddle/fluid/data_feeder.py
+++ b/python/paddle/fluid/data_feeder.py
@@ -24,7 +24,6 @@
     default_main_program,
     _current_expected_place,
     _non_static_mode,
-    _in_eager_without_dygraph_check,
 )
 from .framework import _cpu_num, _cuda_ids
 
diff --git a/python/paddle/fluid/dygraph/base.py b/python/paddle/fluid/dygraph/base.py
index e76f0f3fa3161..3430d78d6d7f9 100644
--- a/python/paddle/fluid/dygraph/base.py
+++ b/python/paddle/fluid/dygraph/base.py
@@ -26,10 +26,7 @@
 import logging
 from ..data_feeder import convert_dtype
 import warnings
-from ..framework import (
-    _get_paddle_place,
-    _in_eager_without_dygraph_check,
-)
+from ..framework import _get_paddle_place
 import paddle
 import warnings
 
@@ -796,14 +793,9 @@ def check_in_out(in_out_list, name):
                 var, core.eager.Tensor
             ), "no_grad_vars can only contains Tensor"
     else:
-        if _in_eager_without_dygraph_check():
-            raise AssertionError(
-                "no_grad_vars must be None, Tensor or list/tuple/set of Tensors"
-            )
-        else:
-            raise AssertionError(
-                "no_grad_vars must be None, Variable or list/tuple/set of Variables"
-            )
+        raise AssertionError(
+            "no_grad_vars must be None, Tensor or list/tuple/set of Tensors"
+        )
 
     assert isinstance(create_graph, bool), "create_graph must be True or False"
 
@@ -819,31 +811,16 @@ def check_in_out(in_out_list, name):
     assert isinstance(only_inputs, bool), "only_inputs must be True or False"
     assert only_inputs, "only_inputs=False is not supported yet"
 
-    if _in_eager_without_dygraph_check():
-        return core.eager.run_partial_grad(
-            outputs,
-            inputs,
-            grad_outputs,
-            retain_graph,
-            create_graph,
-            only_inputs,
-            allow_unused,
-            no_grad_vars,
-        )
-    else:
-        place = core.Place()
-        place.set_place(framework._current_expected_place())
-        return core.dygraph_partial_grad(
-            inputs,
-            outputs,
-            grad_outputs,
-            no_grad_vars,
-            place,
-            create_graph,
-            retain_graph,
-            allow_unused,
-            only_inputs,
-        )
+    return core.eager.run_partial_grad(
+        outputs,
+        inputs,
+        grad_outputs,
+        retain_graph,
+        create_graph,
+        only_inputs,
+        allow_unused,
+        no_grad_vars,
+    )
 
 
 @framework.dygraph_only
diff --git a/python/paddle/fluid/dygraph/math_op_patch.py b/python/paddle/fluid/dygraph/math_op_patch.py
index 7b20e626b8e95..5d8fa6493ee72 100644
--- a/python/paddle/fluid/dygraph/math_op_patch.py
+++ b/python/paddle/fluid/dygraph/math_op_patch.py
@@ -71,31 +71,6 @@ def monkey_patch_math_tensor():
     The difference is, in dygraph mode, use auto-generated op functions for better performance.
     """
 
-    @no_grad
-    def create_tensor(value, dtype, shape):
-        if framework.global_var._in_eager_mode_:
-            out = _C_ops.full(
-                shape, value, dtype, framework._current_expected_place()
-            )
-        else:
-            out = framework_create_tensor(dtype=dtype)
-            out = _legacy_C_ops.fill_constant(
-                out,
-                'dtype',
-                dtype,
-                'shape',
-                shape,
-                'value',
-                value,
-                'force_cpu',
-                False,
-            )
-        out.stop_gradient = True
-        return out
-
-    def create_scalar(value, dtype):
-        return create_tensor(value, dtype, shape=[])
-
     def astype(self, dtype):
         """
 
@@ -191,253 +166,6 @@ def _T_(var):
         out = _C_ops.transpose(var, perm)
         return out
 
-    def _scalar_add_(var, value):
-        return _scalar_elementwise_op_(var, 1.0, value)
-
-    def _scalar_sub_(var, value):
-        return _scalar_elementwise_op_(var, 1.0, -value)
-
-    def _scalar_rsub_(var, value):
-        return _scalar_elementwise_op_(var, -1.0, value)
-
-    def _scalar_mul_(var, value):
-        return _scalar_elementwise_op_(var, value, 0.0)
-
-    def _scalar_div_(var, value):
-        return _scalar_elementwise_op_(var, 1.0 / value, 0.0)
-
-    # for binary operator such as elementwise, compare
-    def _binary_creator_(
-        method_name,
-        op_type,
-        reverse=False,
-        scalar_method=None,
-        call_final_api=False,
-    ):
-        def __impl__(self, other_var):
-            # 1. scalar exists cases
-            # we need combine the tensor.dtype and scalar.dtype, cast correct object
-            if isinstance(other_var, float):
-                # in all cases(+, -, *, /, **, //, %), we need cast tensor.dtype to float
-                if self.dtype in _supported_int_dtype_:
-                    self = astype(self, 'float32')
-                # here use `scale` replace `elementwise` to get better performance
-                # but only +, -, *, / can use this method
-                if scalar_method is not None:
-                    return scalar_method(self, other_var)
-            elif isinstance(other_var, int):
-                # in all cases(+, -, *, /, **, //, %), we can cast it to float
-                # because the output tensor.dtype depend on the type of input tensor
-                other_var = float(other_var)
-                # division is a special case
-                # NOTE(chenweihang): because we cast tensor to float32 instead float64,
-                # the division result can only guarantee the numerical accuracy of 6 digits
-                # after the decimal point. The result of numpy calculation is of float64 type,
-                # so the calculation result here and the calculation result of numpy are
-                # different after 6 decimal point. If necessary, we can also use float64 here.
-                # torch's behavior here is consistent with ours
-                if (
-                    op_type == "divide" or op_type == "elementwise_div"
-                ) and self.dtype in _supported_int_dtype_:
-                    self = astype(self, 'float32')
-                # here use `scale` replace `elementwise` to get better performance
-                # but only +, -, *, / can use this method
-                if scalar_method is not None:
-                    return scalar_method(self, other_var)
-            else:
-                # do nothing
-                pass
-
-            # 2. create Tensor for scalar
-            lhs_dtype = self.dtype
-            other_var_should_be = core.eager.Tensor
-            if not isinstance(other_var, other_var_should_be):
-                if isinstance(other_var, complex):
-                    import paddle
-
-                    other_var = paddle.to_tensor(other_var, dtype='complex64')
-                else:
-                    if reverse:
-                        other_var = create_tensor(
-                            other_var, dtype=lhs_dtype, shape=self.shape
-                        )
-                    else:
-                        # add fill_op
-                        other_var = create_scalar(
-                            value=other_var, dtype=lhs_dtype
-                        )
-
-            # 3. promote types or unify right var type to left var
-            rhs_dtype = other_var.dtype
-            if lhs_dtype != rhs_dtype:
-                if method_name in _supported_promote_complex_types_ and (
-                    lhs_dtype in _complex_dtypes or rhs_dtype in _complex_dtypes
-                ):
-                    # only when lhs_dtype or rhs_dtype is complex type,
-                    # the dtype will promote, in other cases, directly
-                    # use lhs_dtype, this is consistent will original rule
-                    promote_dtype = core._promote_types_if_complex_exists(
-                        lhs_dtype, rhs_dtype
-                    )
-                    self = (
-                        self
-                        if lhs_dtype == promote_dtype
-                        else astype(self, promote_dtype)
-                    )
-                    other_var = (
-                        other_var
-                        if rhs_dtype == promote_dtype
-                        else astype(other_var, promote_dtype)
-                    )
-                else:
-                    warnings.warn(
-                        'The dtype of left and right variables are not the same, left dtype is {}, but right dtype is {}, the right dtype will convert to {}'.format(
-                            lhs_dtype, rhs_dtype, lhs_dtype
-                        )
-                    )
-                    other_var = astype(other_var, lhs_dtype)
-
-            if reverse:
-                tmp = self
-                self = other_var
-                other_var = tmp
-
-            if (
-                op_type == "divide" or op_type == "elementwise_div"
-            ) and self.dtype in _supported_int_dtype_:
-                self = astype(self, 'float32')
-                other_var = astype(other_var, 'float32')
-
-            # 4. calculation
-            axis = -1
-            if in_dygraph_mode():
-                math_op = getattr(_C_ops, op_type)
-            else:
-                math_op = getattr(_legacy_C_ops, op_type)
-            if call_final_api:
-                if op_type == "matmul":
-                    return math_op(self, other_var, False, False)
-                if op_type == "pow":
-                    if isinstance(other_var, core.eager.Tensor):
-                        return _C_ops.elementwise_pow(self, other_var)
-                    else:
-                        return _C_ops.elementwise_pow(self, other_var)
-                return math_op(self, other_var, -1)
-            return math_op(self, other_var, 'axis', axis)
-
-        if call_final_api:
-            comment = ""
-        else:
-            comment = OpProtoHolder.instance().get_op_proto(op_type).comment
-
-        __impl__.__doc__ = """
-        {0}
-        Args:
-            other_var(Tensor|float|int): right hand Tensor
-
-        Returns:
-            Tensor
-        """.format(
-            comment
-        )
-        __impl__.__name__ = method_name
-        return __impl__
-
-    tensor_methods = [
-        ('__neg__', _neg_),
-        ('__float__', _float_),
-        ('__long__', _long_),
-        ('__int__', _int_),
-        ('__len__', _len_),
-        ('__index__', _index_),
-        ('astype', astype),
-        ('dim', lambda x: len(x.shape)),
-        ('ndimension', lambda x: len(x.shape)),
-        ('ndim', _ndim_),
-        ('size', _size_),
-        ('T', _T_),
-        (
-            '__add__',
-            _binary_creator_('__add__', 'elementwise_add', False, _scalar_add_),
-        ),
-        #  a+b == b+a. Do not need to reverse explicitly
-        (
-            '__radd__',
-            _binary_creator_(
-                '__radd__', 'elementwise_add', False, _scalar_add_
-            ),
-        ),
-        (
-            '__sub__',
-            _binary_creator_('__sub__', 'elementwise_sub', False, _scalar_sub_),
-        ),
-        (
-            '__rsub__',
-            _binary_creator_(
-                '__rsub__', 'elementwise_sub', True, _scalar_rsub_
-            ),
-        ),
-        (
-            '__mul__',
-            _binary_creator_('__mul__', 'elementwise_mul', False, _scalar_mul_),
-        ),
-        ## a*b == b*a. Do not need to reverse explicitly
-        (
-            '__rmul__',
-            _binary_creator_(
-                '__rmul__', 'elementwise_mul', False, _scalar_mul_
-            ),
-        ),
-        (
-            '__div__',
-            _binary_creator_('__div__', 'elementwise_div', False, _scalar_div_),
-        ),
-        (
-            '__truediv__',
-            _binary_creator_(
-                '__truediv__', 'elementwise_div', False, _scalar_div_
-            ),
-        ),
-        (
-            '__rdiv__',
-            _binary_creator_('__rdiv__', 'elementwise_div', True, None),
-        ),
-        (
-            '__rtruediv__',
-            _binary_creator_('rtruediv__', 'elementwise_div', True, None),
-        ),
-        (
-            '__pow__',
-            _binary_creator_('__pow__', 'elementwise_pow', False, None),
-        ),
-        (
-            '__rpow__',
-            _binary_creator_('__rpow__', 'elementwise_pow', True, None),
-        ),
-        (
-            '__floordiv__',
-            _binary_creator_(
-                '__floordiv__', 'elementwise_floordiv', False, None
-            ),
-        ),
-        (
-            '__mod__',
-            _binary_creator_('__mod__', 'elementwise_mod', False, None),
-        ),
-        (
-            '__matmul__',
-            _binary_creator_('__matmul__', "matmul_v2", False, None),
-        ),
-        ## for logical compare
-        ('__eq__', _binary_creator_('__eq__', 'equal', False, None)),
-        ('__ne__', _binary_creator_('__ne__', 'not_equal', False, None)),
-        ('__lt__', _binary_creator_('__lt__', 'less_than', False, None)),
-        ('__le__', _binary_creator_('__le__', 'less_equal', False, None)),
-        ('__gt__', _binary_creator_('__gt__', 'greater_than', False, None)),
-        ('__ge__', _binary_creator_('__ge__', 'greater_equal', False, None)),
-        ('__array_ufunc__', None),
-    ]
-
     eager_methods = [
         ('__neg__', _neg_),
         ('__float__', _float_),
@@ -486,22 +214,15 @@ def __impl__(self, other_var):
     local_tensor = core.eager.Tensor
 
     if not local_already_patch:
-        if framework.global_var._in_eager_mode_:
-            for method_name in eager_cpp_level_patch:
-                method_impl = getattr(local_tensor, method_name, None)
-                if method_impl:
-                    setattr(local_tensor, method_name, method_impl)
-
-            for method in eager_methods:
-                method_name = method[0]
-                method_impl = method[1]
+        for method_name in eager_cpp_level_patch:
+            method_impl = getattr(local_tensor, method_name, None)
+            if method_impl:
                 setattr(local_tensor, method_name, method_impl)
 
-        else:
-            for method in tensor_methods:
-                method_name = method[0]
-                method_impl = method[1]
-                setattr(local_tensor, method_name, method_impl)
+        for method in eager_methods:
+            method_name = method[0]
+            method_impl = method[1]
+            setattr(local_tensor, method_name, method_impl)
     else:
         import paddle.tensor
 
diff --git a/python/paddle/fluid/dygraph/tensor_patch_methods.py b/python/paddle/fluid/dygraph/tensor_patch_methods.py
index 80d2b5e973cfe..a081c6ca294ee 100644
--- a/python/paddle/fluid/dygraph/tensor_patch_methods.py
+++ b/python/paddle/fluid/dygraph/tensor_patch_methods.py
@@ -54,11 +54,7 @@ class TensorHookRemoveHelper:
     """
 
     def __init__(self, tensor, hook_id):
-        self._tensor = (
-            tensor
-            if framework.global_var._in_eager_mode_
-            else weakref.ref(tensor)
-        )
+        self._tensor = tensor
         self._hook_id = hook_id
 
     def remove(self):
@@ -68,11 +64,7 @@ def remove(self):
         Returns:
             bool: Return True if removed successfully
         """
-        tensor = (
-            self._tensor
-            if framework.global_var._in_eager_mode_
-            else self._tensor()
-        )
+        tensor = self._tensor
         if tensor is not None:
             res = tensor._remove_grad_hook(self._hook_id)
             if res is True:
@@ -285,37 +277,26 @@ def backward(self, grad_tensor=None, retain_graph=False):
                 )
                 record_event.begin()
             if grad_tensor is not None:
-                if framework.global_var._in_eager_mode_:
-                    assert isinstance(
-                        grad_tensor, core.eager.Tensor
-                    ), "The type of grad_tensor must be paddle.Tensor"
-                else:
-                    assert isinstance(
-                        grad_tensor, paddle.Tensor
-                    ), "The type of grad_tensor must be paddle.Tensor"
+                assert isinstance(
+                    grad_tensor, core.eager.Tensor
+                ), "The type of grad_tensor must be paddle.Tensor"
+
                 assert (
                     grad_tensor.shape == self.shape
                 ), "Tensor shape not match, Tensor of grad_tensor [ {} ] with shape {} mismatch Tensor [ {} ] with shape {}".format(
                     grad_tensor.name, grad_tensor.shape, self.name, self.shape
                 )
 
-            if framework.global_var._in_eager_mode_:
-                if grad_tensor is None:
-                    grad_tensor = []
-                else:
-                    grad_tensor = [grad_tensor]
+            if grad_tensor is None:
+                grad_tensor = []
+            else:
+                grad_tensor = [grad_tensor]
             if _grad_scalar:
                 # When using amp with Fleet DistributedStrategy, we do loss scaling implicitly.
                 self = _grad_scalar.scale(self)
-            if framework.global_var._in_eager_mode_:
-                core.eager.run_backward([self], grad_tensor, retain_graph)
-            else:
-                core.dygraph_run_backward(
-                    [self],
-                    [grad_tensor],
-                    retain_graph,
-                    framework._dygraph_tracer(),
-                )
+
+            core.eager.run_backward([self], grad_tensor, retain_graph)
+
             if in_profiler_mode():
                 record_event.end()
         else:
@@ -352,31 +333,11 @@ def gradient(self):
                 # [500.]
 
         """
-        if framework.global_var._in_eager_mode_:
-            if self.grad is None:
-                return None
-            if self.grad.is_selected_rows():
-                return (np.array(self.grad), np.array(self.grad.rows()))
-            return np.array(self.grad)
-        else:
-            if self._grad_ivar() is None:
-                return None
-
-            new_ivar = self._grad_ivar()
-            # TODO(qili93): temporary for ascned npu performance to be removed along with npu_identity op
-            if (
-                _global_flags()['FLAGS_npu_storage_format']
-                and 'npu' in get_all_custom_device_type()
-            ):
-                new_ivar = paddle.incubate._npu_identity(x=new_ivar, format=-1)
-            new_ivar = new_ivar._copy_to(core.CPUPlace(), True)
-            if self._grad_ivar().type == core.VarDesc.VarType.SELECTED_ROWS:
-                return (
-                    np.array(new_ivar.value().get_selected_rows().get_tensor()),
-                    np.array(new_ivar.value().get_selected_rows().rows()),
-                )
-            else:
-                return np.array(new_ivar.value().get_tensor())
+        if self.grad is None:
+            return None
+        if self.grad.is_selected_rows():
+            return (np.array(self.grad), np.array(self.grad.rows()))
+        return np.array(self.grad)
 
     @framework.dygraph_only
     def register_hook(self, hook):
@@ -705,13 +666,8 @@ def __nonzero__(self):
         assert (
             numel == 1
         ), "When Variable is used as the condition of if/while , Variable can only contain one element."
-        if framework.global_var._in_eager_mode_:
-            assert self._is_initialized(), "tensor not initialized"
-            return bool(np.array(self) > 0)
-        else:
-            tensor = self.value().get_tensor()
-            assert tensor._is_initialized(), "tensor not initialized"
-            return bool(np.array(tensor) > 0)
+        assert self._is_initialized(), "tensor not initialized"
+        return bool(np.array(self) > 0)
 
     def __bool__(self):
         return self.__nonzero__()
@@ -830,11 +786,7 @@ def is_combine_index(item):
             return _setitem_impl_(self, item, value)
 
         else:
-            if framework.global_var._in_eager_mode_:
-                return self.__setitem_eager_tensor__(item, value)
-            else:
-                # Call c++ func __setitem_varbase__ to speedup.
-                return self.__setitem_varbase__(item, value)
+            return self.__setitem_eager_tensor__(item, value)
 
     @framework.dygraph_only
     def _set_grad_ivar(self, value):
@@ -1000,7 +952,7 @@ def to_sparse_coo(self, sparse_dim):
     def __hash__(self):
         return hash(id(self))
 
-    if framework.global_var._in_eager_mode_ and not hasattr(core, "eager"):
+    if not hasattr(core, "eager"):
         return
 
     for method_name, method in (
diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py
index 88354f8226f1d..e440cf4c9e623 100644
--- a/python/paddle/fluid/framework.py
+++ b/python/paddle/fluid/framework.py
@@ -83,7 +83,6 @@ def __init__(self):
         self._in_declarative_mode_ = False
         self._functional_dygraph_context_manager = None
         self._dygraph_tracer_ = _dygraph_tracer_
-        self._in_eager_mode_ = True
 
     def __str__(self):
         strings = []
@@ -95,7 +94,6 @@ def __str__(self):
             + str(self._functional_dygraph_context_manager)
         )
         strings.append("_dygraph_tracer_:" + str(self._dygraph_tracer_))
-        strings.append("_in_eager_mode_:" + str(self._in_eager_mode_))
         return "\n".join(strings)
 
     def __setattr__(self, name, val):
@@ -180,10 +178,6 @@ def __setattr__(self, name, val):
 # to make sure in most case, we find new dygraph mode first with only one if statement.
 
 
-def _in_eager_without_dygraph_check():
-    return global_var._in_eager_mode_
-
-
 # FIXME(dev): We haven't fully verified eager mode on XPU et.al but
 # only GPU/CPU. Remove this after we improve this feature.
 _is_first_import_ = True
@@ -216,9 +210,7 @@ def in_dygraph_mode():
             print(paddle.in_dynamic_mode())  # True, Now we are in dynamic mode
 
     """
-    return (
-        global_var._dygraph_tracer_ is not None
-    ) and global_var._in_eager_mode_
+    return global_var._dygraph_tracer_ is not None
 
 
 def _non_static_mode():
diff --git a/python/paddle/fluid/layer_helper_base.py b/python/paddle/fluid/layer_helper_base.py
index e2723e45f3c83..4a9b71c0cdf8e 100644
--- a/python/paddle/fluid/layer_helper_base.py
+++ b/python/paddle/fluid/layer_helper_base.py
@@ -22,7 +22,6 @@
     default_startup_program,
     _non_static_mode,
     _current_expected_place,
-    _in_eager_without_dygraph_check,
 )
 from . import unique_name
 from .param_attr import ParamAttr, WeightNormParamAttr
diff --git a/python/paddle/fluid/reader.py b/python/paddle/fluid/reader.py
index ed294700b1621..6dfd487ef490e 100644
--- a/python/paddle/fluid/reader.py
+++ b/python/paddle/fluid/reader.py
@@ -29,7 +29,6 @@
     _non_static_mode,
     cpu_places,
     _current_expected_place,
-    _in_eager_without_dygraph_check,
 )
 from .executor import global_scope
 from .data_feeder import DataFeeder, BatchedTensorProvider
@@ -663,12 +662,9 @@ def __iter__(self):
 
     def __next__(self):
         try:
-            if _in_eager_without_dygraph_check():
-                return core.eager.read_next_tensor_list(
-                    self._reader.read_next_list()[0]
-                )
-            else:
-                return self._reader.read_next_var_list()
+            return core.eager.read_next_tensor_list(
+                self._reader.read_next_list()[0]
+            )
         except StopIteration:
             self._reset()
             raise
diff --git a/python/paddle/fluid/tests/unittests/test_run_program_op.py b/python/paddle/fluid/tests/unittests/test_run_program_op.py
index 0f381b5bdda5e..50429e2bced02 100644
--- a/python/paddle/fluid/tests/unittests/test_run_program_op.py
+++ b/python/paddle/fluid/tests/unittests/test_run_program_op.py
@@ -21,7 +21,6 @@
 from paddle import _legacy_C_ops, fluid
 from paddle.fluid import core, framework
 from paddle.fluid.dygraph.base import switch_to_static_graph
-from paddle.fluid.framework import global_var
 
 paddle.enable_static()
 
@@ -207,16 +206,7 @@ def create_var_base(is_input, name):
         for name in self.output_names['Out']:
             outputs['Out'].append(create_var_base(False, name))
 
-        if global_var._in_eager_mode_:
-            outputs['OutScope'] = [core.Scope()]
-        else:
-            outputs['OutScope'] = framework._create_tensor(
-                type=core.VarDesc.VarType.STEP_SCOPES,
-                name="program_out_scope",
-                persistable=True,
-            )
-            inner_scope = core.Scope()
-            outputs['OutScope'].value().set_scope(inner_scope)
+        outputs['OutScope'] = [core.Scope()]
 
         outputs['DOut'] = [create_var_base(False, "Fake_var")]
         return outputs
diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py
index ce271a7169b8e..cc014e7cf1a8f 100644
--- a/python/paddle/optimizer/optimizer.py
+++ b/python/paddle/optimizer/optimizer.py
@@ -24,7 +24,6 @@
 from paddle.fluid.framework import (
     Variable,
     _current_expected_place,
-    _in_eager_without_dygraph_check,
     default_main_program,
     device_guard,
     in_dygraph_mode,
@@ -736,9 +735,7 @@ def _add_accumulator(
             name=var_name,
             persistable=True,
             dtype=dtype or param.dtype,
-            type=core.VarDesc.VarType.LOD_TENSOR
-            if framework._in_eager_without_dygraph_check()
-            else (param.type if type is None else type),
+            type=core.VarDesc.VarType.LOD_TENSOR,
             shape=shape,
             belong_to_optimizer=True,
         )
@@ -1380,11 +1377,8 @@ def clear_grad(self, set_to_zero=True):
                     if not p.stop_gradient:
                         param_list.append(p)
 
-        if _in_eager_without_dygraph_check():
-            for p in param_list:
-                p.clear_gradient(set_to_zero)
-        else:
-            core.clear_gradients(param_list, set_to_zero)
+        for p in param_list:
+            p.clear_gradient(set_to_zero)
 
     @imperative_base.no_grad()
     def minimize(
diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py
index 6fe3f97cd656a..59ce9d6b3677c 100644
--- a/python/paddle/tensor/creation.py
+++ b/python/paddle/tensor/creation.py
@@ -30,11 +30,7 @@
     convert_dtype,
     convert_float_to_uint16,
 )
-from ..fluid.framework import (
-    Variable,
-    _in_eager_without_dygraph_check,
-    device_guard,
-)
+from ..fluid.framework import Variable, device_guard
 from ..fluid.param_attr import ParamAttr
 from ..framework import (
     LayerHelper,
@@ -625,7 +621,7 @@ def _handle_np_dtype(ndarray, dtype):
     if dtype:
         data = _handle_np_dtype(data, dtype)
 
-    if _in_eager_without_dygraph_check() and isinstance(data, np.ndarray):
+    if isinstance(data, np.ndarray):
         return core.eager.Tensor(
             value=data,
             place=place,
diff --git a/test/legacy_test/test_model.py b/test/legacy_test/test_model.py
index c5d62761f7f68..b3cecca16c094 100644
--- a/test/legacy_test/test_model.py
+++ b/test/legacy_test/test_model.py
@@ -776,7 +776,7 @@ def test_summary_error(self):
         paddle.summary(nlp_net, (1, 1, 2))
 
     def test_static_flops(self):
-        if paddle.fluid.framework._in_eager_without_dygraph_check():
+        if True:
             return
         paddle.disable_static()
         net = models.__dict__['mobilenet_v2'](pretrained=False)

From 481511a6b4230ba92438afc965bcd6c4d2dd4f59 Mon Sep 17 00:00:00 2001
From: cyberslack_lee <luhputu0815@gmail.com>
Date: Tue, 16 May 2023 14:09:44 +0800
Subject: [PATCH 351/405] =?UTF-8?q?=E3=80=90Hackathon4=20No.61=E3=80=91rem?=
 =?UTF-8?q?ainder=20=E7=AE=97=E5=AD=90FP16/BF16=E5=8D=95=E6=B5=8B=E5=AE=8C?=
 =?UTF-8?q?=E5=96=84=20(#52920)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../phi/kernels/funcs/elementwise_functor.h   | 14 +++
 paddle/phi/kernels/kps/elementwise_kernel.cu  |  3 +-
 .../legacy/kps/elementwise_raw_kernel.cu      |  3 +-
 .../unittests/test_elementwise_mod_op.py      | 91 ++++++++++++++++++-
 4 files changed, 106 insertions(+), 5 deletions(-)

diff --git a/paddle/phi/kernels/funcs/elementwise_functor.h b/paddle/phi/kernels/funcs/elementwise_functor.h
index c51417730850c..b7dbf252103c6 100644
--- a/paddle/phi/kernels/funcs/elementwise_functor.h
+++ b/paddle/phi/kernels/funcs/elementwise_functor.h
@@ -563,6 +563,20 @@ struct RemainderFunctor<dtype::float16> {
   }
 };
 
+template <>
+struct RemainderFunctor<dtype::bfloat16> {
+  inline HOSTDEVICE dtype::bfloat16 operator()(const dtype::bfloat16 a,
+                                               const dtype::bfloat16 b) const {
+    float b_float = static_cast<float>(b);
+    float res = fmod(static_cast<float>(a), b_float);
+
+    // Accoding to #PR26732: in dividen % divsor
+    // remainder shall have the same sign as divsor.
+    if ((res != 0.0f) && ((res < 0.0f) != (b_float < 0.0f))) res += b_float;
+    return static_cast<dtype::bfloat16>(res);
+  }
+};
+
 template <typename T, typename Enable = void>
 struct InverseRemainderFunctor {
   inline HOSTDEVICE T operator()(const T a, const T b) const {
diff --git a/paddle/phi/kernels/kps/elementwise_kernel.cu b/paddle/phi/kernels/kps/elementwise_kernel.cu
index e1eb5f8528839..c6e693ddd3b90 100644
--- a/paddle/phi/kernels/kps/elementwise_kernel.cu
+++ b/paddle/phi/kernels/kps/elementwise_kernel.cu
@@ -117,7 +117,8 @@ PD_REGISTER_KERNEL(remainder,
                    double,
                    int,
                    int64_t,
-                   phi::dtype::float16) {}
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
 PD_REGISTER_KERNEL(
     floor_divide, KPS, ALL_LAYOUT, phi::FloorDivideKernel, int, int64_t) {}
 PD_REGISTER_KERNEL(elementwise_pow,
diff --git a/paddle/phi/kernels/legacy/kps/elementwise_raw_kernel.cu b/paddle/phi/kernels/legacy/kps/elementwise_raw_kernel.cu
index 8fab237f0a5ee..23ea1264c63f4 100644
--- a/paddle/phi/kernels/legacy/kps/elementwise_raw_kernel.cu
+++ b/paddle/phi/kernels/legacy/kps/elementwise_raw_kernel.cu
@@ -157,7 +157,8 @@ PD_REGISTER_KERNEL(remainder_raw,
                    double,
                    int,
                    float16,
-                   int64_t) {}
+                   int64_t,
+                   bfloat16) {}
 PD_REGISTER_KERNEL(floor_divide_raw,
                    KPS,
                    ALL_LAYOUT,
diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py
index 55ebc5d28cff9..8f34328241fef 100644
--- a/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py
+++ b/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py
@@ -16,10 +16,15 @@
 import unittest
 
 import numpy as np
-from eager_op_test import OpTest
+from eager_op_test import (
+    OpTest,
+    convert_float_to_uint16,
+    convert_uint16_to_float,
+)
 
 import paddle
 from paddle import fluid
+from paddle.fluid import core
 
 
 class TestElementwiseModOp(OpTest):
@@ -106,14 +111,17 @@ def test_check_output(self):
             self.check_output()
 
 
-class TestElementwiseModOpFp16(TestElementwiseModOp):
+@unittest.skipIf(
+    not core.is_compiled_with_cuda(), "core is not compiled with CUDA"
+)
+class TestElementwiseModFP16Op(TestElementwiseModOp):
     def init_dtype(self):
         self.dtype = np.float16
 
     def init_input_output(self):
         self.x = np.random.uniform(-1000, 1000, [10, 10]).astype(self.dtype)
         self.y = np.random.uniform(-100, 100, [10, 10]).astype(self.dtype)
-        self.out = np.mod(self.x, self.y)
+        self.out = np.fmod(self.y + np.fmod(self.x, self.y), self.y)
 
     def test_check_output(self):
         if self.attrs['axis'] == -1:
@@ -122,6 +130,83 @@ def test_check_output(self):
             self.check_output()
 
 
+class TestElementwiseModFP16Op_ZeroDim1(TestElementwiseModFP16Op):
+    def init_input_output(self):
+        self.x = np.random.uniform(0, 10000, []).astype(np.float16)
+        self.y = np.random.uniform(0, 1000, []).astype(np.float16)
+        self.out = np.fmod(self.y + np.fmod(self.x, self.y), self.y)
+
+
+class TestElementwiseModFP16Op_ZeroDim2(TestElementwiseModFP16Op):
+    def init_input_output(self):
+        self.x = np.random.uniform(0, 10000, [10, 10]).astype(np.float16)
+        self.y = np.random.uniform(0, 1000, []).astype(np.float16)
+        self.out = np.fmod(self.y + np.fmod(self.x, self.y), self.y)
+
+
+class TestElementwiseModFP16Op_ZeroDim3(TestElementwiseModFP16Op):
+    def init_input_output(self):
+        self.x = np.random.uniform(0, 10000, []).astype(np.float16)
+        self.y = np.random.uniform(0, 1000, [10, 10]).astype(np.float16)
+        self.out = np.fmod(self.y + np.fmod(self.x, self.y), self.y)
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support the bfloat16",
+)
+class TestElementwiseModBF16Op(OpTest):
+    def init_kernel_type(self):
+        self.use_mkldnn = False
+
+    def init_input_output(self):
+        self.x = np.random.uniform(0, 10000, [10, 10]).astype(np.float32)
+        self.x = convert_uint16_to_float(convert_float_to_uint16(self.x))
+        self.y = np.random.uniform(0, 1000, [10, 10]).astype(np.float32)
+        self.y = convert_uint16_to_float(convert_float_to_uint16(self.y))
+        self.out = np.fmod(self.y + np.fmod(self.x, self.y), self.y)
+
+    def setUp(self):
+        self.op_type = "elementwise_mod"
+        self.python_api = paddle.remainder
+        self.public_python_api = paddle.remainder
+        self.axis = -1
+        self.init_dtype()
+        self.init_input_output()
+        self.init_kernel_type()
+        self.init_axis()
+        self.inputs = {
+            'X': convert_float_to_uint16(
+                OpTest.np_dtype_to_fluid_dtype(self.x)
+            ),
+            'Y': convert_float_to_uint16(
+                OpTest.np_dtype_to_fluid_dtype(self.y)
+            ),
+        }
+        self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn}
+        self.outputs = {'Out': convert_float_to_uint16(self.out)}
+
+    def test_check_output(self):
+        place = core.CUDAPlace(0)
+        self.check_output_with_place(place)
+
+    def init_dtype(self):
+        self.dtype = np.uint16
+
+    def init_axis(self):
+        pass
+
+
+class TestElementwiseModBF16Op_ZeroDim1(TestElementwiseModBF16Op):
+    def init_input(self):
+        self.x = np.random.uniform(0, 10000, []).astype("float32")
+        self.x = convert_uint16_to_float(convert_float_to_uint16(self.x))
+        self.y = np.random.uniform(0, 1000, []).astype("float32")
+        self.y = convert_uint16_to_float(convert_float_to_uint16(self.y))
+        self.out = np.fmod(self.y + np.fmod(self.x, self.y), self.y)
+
+
 class TestElementwiseModOpDouble(TestElementwiseModOpFloat):
     def init_dtype(self):
         self.dtype = np.float64

From ad45b368c53a006546020847818fd7b5572323a8 Mon Sep 17 00:00:00 2001
From: Ikko Eltociear Ashimine <eltociear@gmail.com>
Date: Tue, 16 May 2023 15:09:54 +0900
Subject: [PATCH 352/405] Add Japanese README (#53726)

* Add Japanese README

* Update README_ja.md
---
 README.md    |  2 +-
 README_cn.md |  2 +-
 README_ja.md | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 98 insertions(+), 2 deletions(-)
 create mode 100644 README_ja.md

diff --git a/README.md b/README.md
index 78396610c0793..84650c3058285 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@
 
 --------------------------------------------------------------------------------
 
-English | [简体中文](./README_cn.md)
+English | [简体中文](./README_cn.md) | [日本語](./README_ja.md)
 
 [![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](https://paddlepaddle.org.cn/documentation/docs/en/guides/index_en.html)
 [![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](https://paddlepaddle.org.cn/documentation/docs/zh/guides/index_cn.html)
diff --git a/README_cn.md b/README_cn.md
index 7c7002839dda7..abf327e919ece 100644
--- a/README_cn.md
+++ b/README_cn.md
@@ -5,7 +5,7 @@
 
 --------------------------------------------------------------------------------
 
-[English](./README.md) | 简体中文
+[English](./README.md) | 简体中文 | [日本語](./README_ja.md)
 
 [![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](https://paddlepaddle.org.cn/documentation/docs/en/guides/index_en.html)
 [![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](https://paddlepaddle.org.cn/documentation/docs/zh/guides/index_cn.html)
diff --git a/README_ja.md b/README_ja.md
new file mode 100644
index 0000000000000..c5725fa82ee5f
--- /dev/null
+++ b/README_ja.md
@@ -0,0 +1,96 @@
+<p align="center">
+<img align="center" src="doc/imgs/logo.png", width=1600>
+<p>
+
+--------------------------------------------------------------------------------
+
+[English](./README.md) | [简体中文](./README_cn.md) | 日本語
+
+[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](https://paddlepaddle.org.cn/documentation/docs/en/guides/index_en.html)
+[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](https://paddlepaddle.org.cn/documentation/docs/zh/guides/index_cn.html)
+[![Release](https://img.shields.io/github/release/PaddlePaddle/Paddle.svg)](https://github.com/PaddlePaddle/Paddle/releases)
+[![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](LICENSE)
+[![Twitter](https://img.shields.io/badge/Twitter-1ca0f1.svg?logo=twitter&logoColor=white)](https://twitter.com/PaddlePaddle_)
+
+PaddlePaddle GitHub へようこそ。
+
+PaddlePaddle は中国初の独立系 R&D ディープラーニングプラットフォームとして、2016年からプロのコミュニティに正式にオープンソース化されました。コアとなる深層学習フレームワーク、基本モデルライブラリ、エンドツーエンドの開発キット、ツール＆コンポーネント、さらにサービスプラットフォームを網羅する、高度な技術と豊富な機能を備えた産業プラットフォームです。
+PaddlePaddle は、工業化に対するコミットメントを持つ工業的実践から生まれたものです。製造業、農業、企業サービスなど幅広い分野で採用され、535万人以上の開発者、20万以上の企業、67万以上のモデルを生み出しています。それにより PaddlePaddle は、ますます多くのパートナーの AI 商用化を支援しています。
+
+
+## インストール
+
+### PaddlePaddle の最新リリース: [v2.4](https://github.com/PaddlePaddle/Paddle/tree/release/2.4)
+
+私たちのビジョンは、PaddlePaddle を通じて、誰もが深層学習を行えるようにすることです。
+PaddlePaddle の最新機能を追跡するために、私たちの[リリースのお知らせ](https://github.com/PaddlePaddle/Paddle/releases)を参照してください。
+### 最新の安定版リリースのインストール:
+```
+# CPU
+pip install paddlepaddle
+# GPU
+pip install paddlepaddle-gpu
+
+```
+インストール方法については、[クイックインストール](https://www.paddlepaddle.org.cn/install/quick)をご覧ください
+
+この度、開発者の皆様が Tesla V100 のオンライン計算資源を無償で取得できるようになりました。AI Studio でプログラムを作成した場合、1日あたり8時間のオンライン学習が可能です。[スタートはこちら](https://aistudio.baidu.com/aistudio/index)。
+
+## 四大技術
+
+- **ディープニューラルネットワークの産業用開発のためのアジャイルフレームワーク**
+
+    PaddlePaddle ディープラーニングフレームワークは、ニューラルネットワークをアーキテクトするプログラマブルスキームを活用することで、技術的負担を軽減しながら開発を容易にする。宣言型プログラミングと命令型プログラミングの両方をサポートし、開発の柔軟性と高い実行性能を両立しています。 ニューラル・アーキテクチャは、アルゴリズムによって自動的に設計され、人間の専門家が設計したものよりも優れた性能を発揮する可能性があります。
+
+
+-  **ディープニューラルネットワークの超大規模学習をサポート**
+
+    PaddlePaddle は、超大規模なディープニューラルネットワークのトレーニングでブレークスルーを起こしました。数百のノードに分散したデータソースを用いて、1000億の特徴量と数兆のパラメータを持つディープネットワークのトレーニングをサポートする、世界初の大規模オープンソース・トレーニング・プラットフォームを立ち上げたのです。PaddlePaddle は、超大規模ディープラーニングモデルのオンラインディープラーニングの課題を克服し、さらに1兆以上のパラメータでリアルタイムにモデル更新を実現しました。
+     [詳しくはこちら](https://github.com/PaddlePaddle/Fleet)
+
+
+- **総合的な展開環境に対応した高性能推論エンジン**
+
+   PaddlePaddle は、サードパーティのオープンソースフレームワークで学習されたモデルとの互換性があるだけでなく、様々な生産シナリオに対応した完全な推論エンジン、システム、スイートを提供しています。当社の推論エンジン、システム、スイートには、[Paddle Inference](https://paddle-inference.readthedocs.io/en/master/guides/introduction/index_intro.html) があります： [Paddle Serving](https://github.com/PaddlePaddle/Serving): 高性能なサーバーおよびクラウド推論用のネイティブ推論ライブラリ： [Paddle Serving](https://github.com/PaddlePaddle/Paddle-Lite): 分散型やパイプライン型プロダクションに適したサービス指向フレームワーク; [Paddle Lite](https://github.com/PaddlePaddle/Paddle-Lite)： モバイルや IoT 環境向けの超軽量推論エンジン; [Paddle.js](https://www.paddlepaddle.org.cn/paddle/paddlejs)： ブラウザやミニアプリのためのフロントエンド推論エンジンです。さらに、各シナリオの主要なハードウェアに最適化することで、Paddle の推論エンジンは他の主流フレームワークのほとんどを凌駕しています。
+
+
+- **オープンソースリポジトリによる業界指向のモデルやライブラリ**
+
+     PaddlePaddle は、業界で長い間実践され、磨かれてきた100以上の主流モデルを含み、維持しています。これらのモデルの中には、主要な国際コンペティションで主要な賞を受賞したものもあります。一方、PaddlePaddle は、産業用アプリケーションの迅速な開発を促進するために、200以上のプレトレーニングモデル（そのうちのいくつかはソースコード付き）をさらに整備しています。
+     [詳しくはこちら](https://github.com/PaddlePaddle/models)
+
+
+## ドキュメント
+
+[英語](https://www.paddlepaddle.org.cn/documentation/docs/en/guides/index_en.html)と
+[中国語](https://www.paddlepaddle.org.cn/documentation/docs/zh/guide/index_cn.html)のドキュメントを提供しています。
+
+- [ガイド](https://www.paddlepaddle.org.cn/documentation/docs/en/guides/index_en.html)
+
+  PaddlePaddle でディープラーニングの基本を実装する方法から始めてみてはいかがでしょうか。
+
+- [プラクティス](https://www.paddlepaddle.org.cn/documentation/docs/zh/tutorial/index_cn.html)
+
+  Paddle を使ってモデルを構築し、ディープラーニングタスクをより効率的に実行しましょう。
+
+- [API リファレンス](https://www.paddlepaddle.org.cn/documentation/docs/en/api/index_en.html)
+
+   新しい API により、より短時間のプログラムが可能となりました。
+
+- [コントリビュート方法](https://www.paddlepaddle.org.cn/documentation/docs/en/guides/08_contribution/index_en.html)
+
+   皆様のご投稿に感謝いたします！
+
+## コミュニケーション
+
+- [Github Issues](https://github.com/PaddlePaddle/Paddle/issues): バグレポート、機能リクエスト、インストールに関する問題、使用方法に関する問題など。
+- QQディスカッショングループ: 441226485 (PaddlePaddle)です。
+- [フォーラム](https://aistudio.baidu.com/paddle/forum): 実装や研究などについて話し合います。
+
+## コース
+
+- [Server Deployments](https://aistudio.baidu.com/aistudio/course/introduce/19084): ローカルサービスやリモートサービスを利用した高性能なサーバー展開を紹介するコースです。
+- [Edge Deployments](https://aistudio.baidu.com/aistudio/course/introduce/22690): モバイル、IoT から Web、アプレットまで、エッジの展開を紹介するコース。
+
+## Copyright とライセンス
+PaddlePaddle は [Apache-2.0 license](LICENSE) の下で提供されています。

From 7b81092b498274255ad92e0ac88e6029aa506758 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?=
 <83450930+Liyulingyue@users.noreply.github.com>
Date: Tue, 16 May 2023 14:17:17 +0800
Subject: [PATCH 353/405] [static op generation] InstanceNorm (#53340)

* mv InstanceNorm

* modify op_version.yaml

* modify add Operator:: in get_expected_kernel_func.cc

* rm gradexpectedkernel

* add extra

* add float epsilon=1e-5
---
 .../generator/get_expected_kernel_func.cc     |  30 +++
 .../generator/get_expected_kernel_func.h      |   4 +
 paddle/fluid/operators/instance_norm_op.cc    | 238 ------------------
 paddle/fluid/operators/instance_norm_op.h     | 116 ---------
 paddle/phi/api/yaml/backward.yaml             |  24 ++
 paddle/phi/api/yaml/legacy_backward.yaml      |  24 --
 paddle/phi/api/yaml/legacy_ops.yaml           |  12 -
 paddle/phi/api/yaml/op_compat.yaml            |   4 +
 paddle/phi/api/yaml/op_version.yaml           |  11 +
 paddle/phi/api/yaml/ops.yaml                  |  12 +
 paddle/phi/ops/compat/instance_norm_sig.cc    |  56 -----
 11 files changed, 85 insertions(+), 446 deletions(-)
 delete mode 100644 paddle/fluid/operators/instance_norm_op.cc
 delete mode 100644 paddle/fluid/operators/instance_norm_op.h
 delete mode 100644 paddle/phi/ops/compat/instance_norm_sig.cc

diff --git a/paddle/fluid/operators/generator/get_expected_kernel_func.cc b/paddle/fluid/operators/generator/get_expected_kernel_func.cc
index 5c654d942c245..49697a48a176b 100644
--- a/paddle/fluid/operators/generator/get_expected_kernel_func.cc
+++ b/paddle/fluid/operators/generator/get_expected_kernel_func.cc
@@ -181,5 +181,35 @@ phi::KernelKey GetUniqueExpectedKernelType(
   }
 }
 
+phi::KernelKey GetInstanceNormExpectedKernelType(
+    const framework::ExecutionContext& ctx,
+    const framework::OperatorWithKernel* op_ptr) {
+  auto input_data_type =
+      op_ptr->OperatorWithKernel::IndicateVarDataType(ctx, "X");
+  // By default, the type of the scale, bias, mean,
+  // and var tensors should both be float. (For float or float16 input tensor)
+  // or double (For double input tensor).
+  auto in_param_type = framework::proto::VarType::FP32;
+  if (input_data_type == framework::proto::VarType::FP64) {
+    in_param_type = framework::proto::VarType::FP64;
+  }
+  if (ctx.HasInput("Scale")) {
+    PADDLE_ENFORCE_EQ(in_param_type,
+                      framework::TransToProtoVarType(
+                          ctx.Input<phi::DenseTensor>("Scale")->dtype()),
+                      platform::errors::InvalidArgument(
+                          "Scale input should be of float type"));
+  }
+  if (ctx.HasInput("Bias")) {
+    PADDLE_ENFORCE_EQ(in_param_type,
+                      framework::TransToProtoVarType(
+                          ctx.Input<phi::DenseTensor>("Bias")->dtype()),
+                      platform::errors::InvalidArgument(
+                          "Bias input should be of float type"));
+  }
+
+  return phi::KernelKey(input_data_type, ctx.GetPlace());
+}
+
 }  // namespace operators
 }  // namespace paddle
diff --git a/paddle/fluid/operators/generator/get_expected_kernel_func.h b/paddle/fluid/operators/generator/get_expected_kernel_func.h
index a6fe0fc4b225a..4ef889099846a 100644
--- a/paddle/fluid/operators/generator/get_expected_kernel_func.h
+++ b/paddle/fluid/operators/generator/get_expected_kernel_func.h
@@ -48,6 +48,10 @@ phi::KernelKey GetUniqueExpectedKernelType(
     const framework::ExecutionContext& ctx,
     const framework::OperatorWithKernel* op_ptr);
 
+phi::KernelKey GetInstanceNormExpectedKernelType(
+    const framework::ExecutionContext& ctx,
+    const framework::OperatorWithKernel* op_ptr);
+
 phi::KernelKey GetYoloLossExpectedKernelType(
     const framework::ExecutionContext& ctx,
     const framework::OperatorWithKernel* op_ptr);
diff --git a/paddle/fluid/operators/instance_norm_op.cc b/paddle/fluid/operators/instance_norm_op.cc
deleted file mode 100644
index 8d76b46968b3e..0000000000000
--- a/paddle/fluid/operators/instance_norm_op.cc
+++ /dev/null
@@ -1,238 +0,0 @@
-/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include "paddle/fluid/operators/instance_norm_op.h"
-
-#include <memory>
-#include <string>
-#include <unordered_map>
-
-#include "paddle/fluid/framework/data_layout.h"
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/fluid/framework/op_version_registry.h"
-#include "paddle/fluid/prim/api/composite_backward/composite_backward_api.h"
-#include "paddle/fluid/prim/utils/static/composite_grad_desc_maker.h"
-#include "paddle/fluid/prim/utils/static/desc_tensor.h"
-#include "paddle/phi/core/infermeta_utils.h"
-#include "paddle/phi/infermeta/backward.h"
-#include "paddle/phi/infermeta/ternary.h"
-#include "paddle/phi/kernels/funcs/math_function.h"
-
-namespace paddle {
-namespace operators {
-
-phi::KernelKey InstanceNormOp::GetExpectedKernelType(
-    const framework::ExecutionContext &ctx) const {
-  auto input_data_type = OperatorWithKernel::IndicateVarDataType(ctx, "X");
-  // By default, the type of the scale, bias, mean,
-  // and var tensors should both be float. (For float or float16 input tensor)
-  // or double (For double input tensor).
-  auto in_param_type = framework::proto::VarType::FP32;
-  if (input_data_type == framework::proto::VarType::FP64) {
-    in_param_type = framework::proto::VarType::FP64;
-  }
-  if (ctx.HasInput("Scale")) {
-    PADDLE_ENFORCE_EQ(in_param_type,
-                      framework::TransToProtoVarType(
-                          ctx.Input<phi::DenseTensor>("Scale")->dtype()),
-                      platform::errors::InvalidArgument(
-                          "Scale input should be of float type"));
-  }
-  if (ctx.HasInput("Bias")) {
-    PADDLE_ENFORCE_EQ(in_param_type,
-                      framework::TransToProtoVarType(
-                          ctx.Input<phi::DenseTensor>("Bias")->dtype()),
-                      platform::errors::InvalidArgument(
-                          "Bias input should be of float type"));
-  }
-
-  return phi::KernelKey(input_data_type, ctx.GetPlace());
-}
-
-void InstanceNormOpMaker::Make() {
-  AddAttr<float>("epsilon", "")
-      .SetDefault(1e-5)
-      .AddCustomChecker([](const float &epsilon) {
-        PADDLE_ENFORCE_EQ(epsilon >= 0.0f && epsilon <= 0.001f,
-                          true,
-                          platform::errors::InvalidArgument(
-                              "'epsilon' should be between 0.0 and 0.001."));
-      });
-  AddInput("X", "The input tensor");
-  AddInput("Scale",
-           "Scale is a 1-dimensional tensor of size C "
-           "that is applied to the output")
-      .AsDispensable();
-  AddInput("Bias",
-           "Bias is a 1-dimensional tensor of size C "
-           "that is applied to the output")
-      .AsDispensable();
-  AddOutput("Y", "result after normalization");
-  AddOutput("SavedMean",
-            "Mean of the current mini batch, "
-            "will apply to output when training")
-      .AsIntermediate()
-      .AsExtra();
-  AddOutput("SavedVariance",
-            "Variance of the current mini batch, "
-            "will apply to output when training")
-      .AsIntermediate()
-      .AsExtra();
-  AddComment(R"DOC(
-Instance Normalization.
-
-Instance Norm has been implemented as disscussed in the paper:
-https://arxiv.org/pdf/1607.08022.pdf
-Can be used as a normalizer function for conv2d and fully_connected operations.
-The required data format for this layer is as following:
-NCHW `[batch, in_channels, in_height, in_width]`
-
-)DOC");
-}
-
-phi::KernelKey InstanceNormGradOp::GetExpectedKernelType(
-    const framework::ExecutionContext &ctx) const {
-  const auto *var = ctx.InputVar(framework::GradVarName("Y"));
-  if (var == nullptr) {
-    PADDLE_THROW(
-        platform::errors::NotFound("cannot find gradient variable of Y"));
-  }
-  const phi::DenseTensor *t = nullptr;
-  if (var->IsType<phi::DenseTensor>()) {
-    t = &var->Get<phi::DenseTensor>();
-  } else if (var->IsType<phi::DenseTensor>()) {
-    t = &var->Get<phi::DenseTensor>();
-  }
-  if (t == nullptr) {
-    PADDLE_THROW(
-        platform::errors::InvalidArgument("gradient variable of Y is empty"));
-  }
-  return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(ctx, "X"),
-                        ctx.GetPlace());
-}
-
-phi::KernelKey InstanceNormDoubleGradOp::GetExpectedKernelType(
-    const framework::ExecutionContext &ctx) const {
-  const auto *var = ctx.InputVar("DY");
-  if (var == nullptr) {
-    PADDLE_THROW(
-        platform::errors::NotFound("cannot find gradient variable of Y"));
-  }
-  const phi::DenseTensor *t = nullptr;
-  if (var->IsType<phi::DenseTensor>()) {
-    t = &var->Get<phi::DenseTensor>();
-  } else if (var->IsType<phi::DenseTensor>()) {
-    t = &var->Get<phi::DenseTensor>();
-  }
-  if (t == nullptr) {
-    PADDLE_THROW(
-        platform::errors::InvalidArgument("gradient variable of Y is empty"));
-  }
-  return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(ctx, "X"),
-                        ctx.GetPlace());
-}
-
-class InstanceNormCompositeGradOpMaker : public prim::CompositeGradOpMakerBase {
-  using prim::CompositeGradOpMakerBase::CompositeGradOpMakerBase;
-
- public:
-  void Apply() override {
-    // inputs and outputs of batch_norm
-    paddle::Tensor x = this->GetSingleForwardInput("X");
-    paddle::Tensor scale = this->GetSingleForwardInput("Scale");
-    paddle::Tensor saved_mean = this->GetSingleForwardOutput("SavedMean");
-    paddle::Tensor saved_variance =
-        this->GetSingleForwardOutput("SavedVariance");
-
-    paddle::Tensor y_grad = this->GetSingleOutputGrad("Y");
-    paddle::Tensor x_grad = this->GetSingleInputGrad("X");
-    paddle::Tensor scale_grad = this->GetSingleInputGrad("Scale");
-    paddle::Tensor bias_grad = this->GetSingleInputGrad("Bias");
-
-    auto x_grad_ptr = this->GetOutputPtr(&x_grad);
-    std::string x_grad_name = this->GetOutputName(x_grad);
-    auto scale_grad_ptr = this->GetOutputPtr(&scale_grad);
-    std::string scale_grad_name = this->GetOutputName(scale_grad);
-    auto bias_grad_ptr = this->GetOutputPtr(&bias_grad);
-    std::string bias_grad_name = this->GetOutputName(bias_grad);
-
-    auto epsilon = this->Attr<float>("epsilon");
-
-    VLOG(3) << "Runing instance_norm composite func";
-    prim::instance_norm_grad<prim::DescTensor>(x,
-                                               scale,
-                                               saved_mean,
-                                               saved_variance,
-                                               y_grad,
-                                               epsilon,
-                                               x_grad_ptr,
-                                               scale_grad_ptr,
-                                               bias_grad_ptr);
-    this->RecoverOutputName(x_grad, x_grad_name);
-    this->RecoverOutputName(scale_grad, scale_grad_name);
-    this->RecoverOutputName(bias_grad, bias_grad_name);
-  }
-};
-
-DECLARE_INPLACE_OP_INFERER(InstanceNormDoubleGradOpInplaceInferer,
-                           {"DY", "DDY"});
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-DECLARE_INFER_SHAPE_FUNCTOR(instance_norm,
-                            InstanceNormInferShapeFunctor,
-                            PD_INFER_META(phi::InstanceNormInferMeta));
-DECLARE_INFER_SHAPE_FUNCTOR(instance_norm_grad,
-                            InstanceNormGradInferShapeFunctor,
-                            PD_INFER_META(phi::InstanceNormGradInferMeta));
-DECLARE_INFER_SHAPE_FUNCTOR(
-    instance_norm_grad_grad,
-    InstanceNormDoubleGradInferShapeFunctor,
-    PD_INFER_META(phi::InstanceNormDoubleGradInferMeta));
-REGISTER_OPERATOR(instance_norm,
-                  ops::InstanceNormOp,
-                  ops::InstanceNormOpMaker,
-                  ops::InstanceNormOpInferVarType,
-                  ops::InstanceNormGradMaker<paddle::framework::OpDesc>,
-                  ops::InstanceNormGradMaker<paddle::imperative::OpBase>,
-                  InstanceNormInferShapeFunctor,
-                  ops::InstanceNormCompositeGradOpMaker);
-REGISTER_OPERATOR(instance_norm_grad,
-                  ops::InstanceNormGradOp,
-                  ops::InstanceNormDoubleGradMaker<paddle::framework::OpDesc>,
-                  ops::InstanceNormDoubleGradMaker<paddle::imperative::OpBase>,
-                  InstanceNormGradInferShapeFunctor);
-REGISTER_OPERATOR(instance_norm_grad_grad,
-                  ops::InstanceNormDoubleGradOp,
-                  ops::InstanceNormDoubleGradOpInplaceInferer,
-                  InstanceNormDoubleGradInferShapeFunctor);
-
-REGISTER_OP_VERSION(instance_norm)
-    .AddCheckpoint(
-        R"ROC(
-      Change dispensable of attribute from False to True in instance_norm.
-    )ROC",
-        paddle::framework::compatible::OpVersionDesc()
-            .ModifyAttr(
-                "Bias",
-                "The arg 'dispensable' of Input 'Bias' is changed: from "
-                "'False' to 'True'.",
-                true)
-            .ModifyAttr(
-                "Scale",
-                "The arg 'dispensable' of Input 'Scale' is changed: from "
-                "'False' to 'True'.",
-                true));
diff --git a/paddle/fluid/operators/instance_norm_op.h b/paddle/fluid/operators/instance_norm_op.h
deleted file mode 100644
index 9a885e47e40a0..0000000000000
--- a/paddle/fluid/operators/instance_norm_op.h
+++ /dev/null
@@ -1,116 +0,0 @@
-/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#pragma once
-#include <memory>
-#include <string>
-#include <unordered_map>
-
-#include "paddle/fluid/framework/op_registry.h"
-
-namespace paddle {
-namespace operators {
-
-using DataLayout = phi::DataLayout;
-
-class InstanceNormOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext &ctx) const override;
-};
-
-class InstanceNormGradOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext &ctx) const override;
-};
-
-class InstanceNormDoubleGradOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext &ctx) const override;
-};
-
-class InstanceNormOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override;
-};
-
-template <typename T>
-class InstanceNormGradMaker : public framework::SingleGradOpMaker<T> {
- public:
-  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
-
- protected:
-  void Apply(GradOpPtr<T> op) const override {
-    op->SetType("instance_norm_grad");
-    op->SetInput("X", this->Input("X"));
-    op->SetInput(framework::GradVarName("Y"), this->OutputGrad("Y"));
-
-    op->SetInput("Scale", this->Input("Scale"));
-    op->SetInput("SavedMean", this->Output("SavedMean"));
-    op->SetInput("SavedVariance", this->Output("SavedVariance"));
-
-    op->SetAttrMap(this->Attrs());
-    op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
-    op->SetOutput(framework::GradVarName("Scale"), this->InputGrad("Scale"));
-    op->SetOutput(framework::GradVarName("Bias"), this->InputGrad("Bias"));
-  }
-};
-
-template <typename T>
-class InstanceNormDoubleGradMaker : public framework::SingleGradOpMaker<T> {
- public:
-  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
-
- protected:
-  void Apply(GradOpPtr<T> op) const override {
-    op->SetType("instance_norm_grad_grad");
-    op->SetInput("X", this->Input("X"));
-    op->SetInput("Scale", this->Input("Scale"));
-    op->SetInput("SavedMean", this->Input("SavedMean"));
-    op->SetInput("SavedVariance", this->Input("SavedVariance"));
-    op->SetInput("DDX", this->OutputGrad(framework::GradVarName("X")));
-    op->SetInput("DDScale", this->OutputGrad(framework::GradVarName("Scale")));
-    op->SetInput("DDBias", this->OutputGrad(framework::GradVarName("Bias")));
-    op->SetInput("DY", this->Input(framework::GradVarName("Y")));
-
-    op->SetAttrMap(this->Attrs());
-    op->SetOutput("DX", this->InputGrad("X"));
-    op->SetOutput("DScale", this->InputGrad("Scale"));
-    op->SetOutput("DDY", this->InputGrad(framework::GradVarName("Y")));
-  }
-};
-
-class InstanceNormOpInferVarType
-    : public framework::PassInDtypeAndVarTypeToOutput {
- protected:
-  std::unordered_map<std::string, std::string> &GetInputOutputWithSameType()
-      const override {
-    static std::unordered_map<std::string, std::string> m{{"X", "Y"}};
-    return m;
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index 92ce63e470976..a75fe0e596a26 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -851,6 +851,30 @@
     data_type : out_grad
   no_need_buffer : x
 
+- backward_op : instance_norm_double_grad
+  forward : instance_norm_grad(Tensor x, Tensor fwd_scale, Tensor saved_mean, Tensor saved_variance, Tensor grad_y, float epsilon) -> Tensor(grad_x), Tensor(grad_scale), Tensor(grad_bias)
+  args : (Tensor x, Tensor fwd_scale, Tensor saved_mean, Tensor saved_variance, Tensor grad_y, Tensor grad_x_grad, Tensor grad_scale_grad, Tensor grad_bias_grad, float epsilon)
+  output : Tensor(x_grad), Tensor(fwd_scale_grad), Tensor(grad_y_grad)
+  infer_meta :
+    func : InstanceNormDoubleGradInferMeta
+  kernel :
+    func : instance_norm_double_grad
+    data_type : x
+  optional : fwd_scale, grad_x_grad, grad_scale_grad, grad_bias_grad
+
+- backward_op : instance_norm_grad
+  forward : instance_norm(Tensor x, Tensor scale, Tensor bias, float epsilon) -> Tensor(y), Tensor(saved_mean), Tensor(saved_variance)
+  args : (Tensor x, Tensor scale, Tensor saved_mean, Tensor saved_variance, Tensor y_grad, float epsilon=1e-5)
+  output : Tensor(x_grad), Tensor(scale_grad), Tensor(bias_grad)
+  infer_meta :
+    func : InstanceNormGradInferMeta
+  kernel :
+    func : instance_norm_grad
+    data_type : x
+  optional : scale
+  backward : instance_norm_double_grad
+  composite: instance_norm_grad(x, scale, saved_mean, saved_variance, y_grad, epsilon, x_grad, scale_grad, bias_grad)
+
 - backward_op : inverse_grad
   forward : inverse(Tensor x) -> Tensor(out)
   args : (Tensor out, Tensor out_grad)
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index 37a3626861c08..4c5106f963b05 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -476,30 +476,6 @@
   kernel :
     func : hsigmoid_loss_grad
 
-- backward_op : instance_norm_double_grad
-  forward : instance_norm_grad(Tensor x, Tensor fwd_scale, Tensor saved_mean, Tensor saved_variance, Tensor grad_y, float epsilon) -> Tensor(grad_x), Tensor(grad_scale), Tensor(grad_bias)
-  args : (Tensor x, Tensor fwd_scale, Tensor saved_mean, Tensor saved_variance, Tensor grad_y, Tensor grad_x_grad, Tensor grad_scale_grad, Tensor grad_bias_grad, float epsilon)
-  output : Tensor(x_grad), Tensor(fwd_scale_grad), Tensor(grad_y_grad)
-  infer_meta :
-    func : InstanceNormDoubleGradInferMeta
-  kernel :
-    func : instance_norm_double_grad
-    data_type : x
-  optional : fwd_scale, grad_x_grad, grad_scale_grad, grad_bias_grad
-
-- backward_op : instance_norm_grad
-  forward : instance_norm(Tensor x, Tensor scale, Tensor bias, float epsilon) -> Tensor(y), Tensor(saved_mean), Tensor(saved_variance)
-  args : (Tensor x, Tensor scale, Tensor saved_mean, Tensor saved_variance, Tensor y_grad, float epsilon)
-  output : Tensor(x_grad), Tensor(scale_grad), Tensor(bias_grad)
-  infer_meta :
-    func : InstanceNormGradInferMeta
-  kernel :
-    func : instance_norm_grad
-    data_type : x
-  optional : scale
-  backward : instance_norm_double_grad
-  composite: instance_norm_grad(x, scale, saved_mean, saved_variance, y_grad, epsilon, x_grad, scale_grad, bias_grad)
-
 - backward_op : layer_norm_grad
   forward : layer_norm (Tensor x, Tensor scale, Tensor bias, float epsilon, int begin_norm_axis) -> Tensor(out), Tensor(mean), Tensor(variance)
   args : (Tensor x,  Tensor scale, Tensor bias, Tensor mean, Tensor variance, Tensor out_grad, float epsilon, int begin_norm_axis)
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index cf85f1b8bfe32..408fe94987cff 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -597,18 +597,6 @@
     func : increment
   inplace : (x -> out)
 
-- op : instance_norm
-  args : (Tensor x, Tensor scale, Tensor bias, float epsilon)
-  output : Tensor(y), Tensor(saved_mean), Tensor(saved_variance)
-  infer_meta :
-    func : InstanceNormInferMeta
-  kernel :
-    func : instance_norm
-    data_type : x
-  optional : scale, bias
-  intermediate : saved_mean, saved_variance
-  backward : instance_norm_grad
-
 - op : layer_norm
   args : (Tensor x, Tensor scale, Tensor bias, float epsilon, int begin_norm_axis)
   output : Tensor(out), Tensor(mean), Tensor(variance)
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index 5ad73f4e1045f..ab8274ac60181 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -1171,6 +1171,10 @@
     y : Y
     saved_mean : SavedMean
     saved_variance : SavedVariance
+  extra:
+    outputs: [ saved_mean, saved_variance ]
+  get_expected_kernel_type:
+    instance_norm: GetInstanceNormExpectedKernelType
 
 - op : inverse
   inputs :
diff --git a/paddle/phi/api/yaml/op_version.yaml b/paddle/phi/api/yaml/op_version.yaml
index 9d9cb6187b218..5d6904ee58927 100644
--- a/paddle/phi/api/yaml/op_version.yaml
+++ b/paddle/phi/api/yaml/op_version.yaml
@@ -167,6 +167,17 @@
           comment : In order to specify interpolation mode
           default : std::string("bilinear")
 
+- op : instance_norm
+  version :
+    - checkpoint : Change dispensable of attribute from False to True in instance_norm.
+      action :
+        - modify_attr : Bias
+          comment : "The arg 'dispensable' of Input 'Bias' is changed: from 'False' to 'True'."
+          default : "true"
+        - modify_attr : Scale
+          comment : "The arg 'dispensable' of Input 'Scale' is changed: from 'False' to 'True'."
+          default : "true"
+
 - op : lamb
   version :
     - checkpoint : Upgrade lamb, add two new outputs [Beta1PowOut] and [Beta2PowOut].
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index b8f99bbe106b0..7fc3acc5614d2 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -989,6 +989,18 @@
     data_type : x
   backward : index_select_grad
 
+- op : instance_norm
+  args : (Tensor x, Tensor scale, Tensor bias, float epsilon=1e-5)
+  output : Tensor(y), Tensor(saved_mean), Tensor(saved_variance)
+  infer_meta :
+    func : InstanceNormInferMeta
+  kernel :
+    func : instance_norm
+    data_type : x
+  optional : scale, bias
+  intermediate : saved_mean, saved_variance
+  backward : instance_norm_grad
+
 - op : inverse
   args : (Tensor x)
   output : Tensor(out)
diff --git a/paddle/phi/ops/compat/instance_norm_sig.cc b/paddle/phi/ops/compat/instance_norm_sig.cc
deleted file mode 100644
index 6ccf120979887..0000000000000
--- a/paddle/phi/ops/compat/instance_norm_sig.cc
+++ /dev/null
@@ -1,56 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/core/compat/op_utils.h"
-
-namespace phi {
-
-KernelSignature InstanceNormOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature("instance_norm",
-                         {"X", "Scale", "Bias"},
-                         {"epsilon"},
-                         {"Y", "SavedMean", "SavedVariance"});
-}
-
-KernelSignature InstanceNormGradOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature("instance_norm_grad",
-                         {"X", "Scale", "SavedMean", "SavedVariance", "Y@GRAD"},
-                         {"epsilon"},
-                         {"X@GRAD", "Scale@GRAD", "Bias@GRAD"});
-}
-KernelSignature InstanceNormDoubleGradOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature("instance_norm_double_grad",
-                         {"X",
-                          "Scale",
-                          "SavedMean",
-                          "SavedVariance",
-                          "DY",
-                          "DDX",
-                          "DDScale",
-                          "DDBias"},
-                         {"epsilon"},
-                         {"DX", "DScale", "DDY"});
-}
-}  // namespace phi
-
-PD_REGISTER_BASE_KERNEL_NAME(instance_norm_grad_grad,
-                             instance_norm_double_grad);
-PD_REGISTER_ARG_MAPPING_FN(instance_norm, phi::InstanceNormOpArgumentMapping);
-PD_REGISTER_ARG_MAPPING_FN(instance_norm_grad,
-                           phi::InstanceNormGradOpArgumentMapping);
-PD_REGISTER_ARG_MAPPING_FN(instance_norm_grad_grad,
-                           phi::InstanceNormDoubleGradOpArgumentMapping);

From b86bbe85ab897c447091acda9dbf250aed8bb1dc Mon Sep 17 00:00:00 2001
From: cyberslack_lee <luhputu0815@gmail.com>
Date: Tue, 16 May 2023 14:18:59 +0800
Subject: [PATCH 354/405] support auto generation V2 abs (#53341)

---
 paddle/fluid/operators/abs_op.cc              | 184 ------------------
 paddle/phi/api/yaml/backward.yaml             |  26 +++
 paddle/phi/api/yaml/legacy_backward.yaml      |  22 ---
 paddle/phi/api/yaml/legacy_ops.yaml           |   9 -
 paddle/phi/api/yaml/ops.yaml                  |  10 +
 paddle/phi/ops/compat/abs_sig.cc              |  37 ----
 .../paddle/tensor/layer_function_generator.py |  28 ++-
 7 files changed, 58 insertions(+), 258 deletions(-)
 delete mode 100644 paddle/fluid/operators/abs_op.cc
 delete mode 100644 paddle/phi/ops/compat/abs_sig.cc

diff --git a/paddle/fluid/operators/abs_op.cc b/paddle/fluid/operators/abs_op.cc
deleted file mode 100644
index adc9f20b8c93d..0000000000000
--- a/paddle/fluid/operators/abs_op.cc
+++ /dev/null
@@ -1,184 +0,0 @@
-// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <memory>
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/prim/api/composite_backward/composite_backward_api.h"
-#include "paddle/fluid/prim/utils/static/composite_grad_desc_maker.h"
-#include "paddle/fluid/prim/utils/static/desc_tensor.h"
-#include "paddle/phi/core/infermeta_utils.h"
-#include "paddle/phi/infermeta/unary.h"
-
-namespace paddle {
-namespace operators {
-
-class AbsOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    auto input_data_type =
-        framework::OperatorWithKernel::IndicateVarDataType(ctx, "X");
-    return phi::KernelKey(input_data_type, ctx.GetPlace());
-  }
-};
-
-class AbsOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput("X", "(Tensor), The input tensor of abs op.");
-    AddOutput("Out", "(Tensor), The output tensor of abs op.");
-    AddComment(R"DOC(
-Abs Operator.
-
-This operator is used to perform elementwise abs for input $X$.
-$$out = |x|$$
-
-)DOC");
-  }
-};
-
-class AbsGradOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-  void InferShape(framework::InferShapeContext* ctx) const override {
-    OP_INOUT_CHECK(ctx->HasInput(framework::GradVarName("Out")),
-                   "Input",
-                   "Out@Grad",
-                   "AbsGrad");
-    OP_INOUT_CHECK(ctx->HasOutput(framework::GradVarName("X")),
-                   "Output",
-                   "X@Grad",
-                   "AbsGrad");
-
-    auto dout_dims = ctx->GetInputDim(framework::GradVarName("Out"));
-    ctx->SetOutputDim(framework::GradVarName("X"), dout_dims);
-  }
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    auto input_data_type =
-        framework::OperatorWithKernel::IndicateVarDataType(ctx, "X");
-    return phi::KernelKey(input_data_type, ctx.GetPlace());
-  }
-};
-
-template <typename T>
-class AbsGradMaker : public framework::SingleGradOpMaker<T> {
- public:
-  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
-
-  void Apply(GradOpPtr<T> retv) const override {
-    retv->SetType("abs_grad");
-    retv->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out"));
-    retv->SetInput("X", this->Input("X"));
-    retv->SetAttrMap(this->Attrs());
-    retv->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
-  }
-};
-
-class AbsCompositeGradOpMaker : public prim::CompositeGradOpMakerBase {
-  using prim::CompositeGradOpMakerBase::CompositeGradOpMakerBase;
-
- public:
-  void Apply() override {
-    paddle::Tensor input = this->GetSingleForwardInput("X");
-    paddle::Tensor out_grad = this->GetSingleOutputGrad("Out");
-    paddle::Tensor input_grad = this->GetSingleInputGrad("X");
-
-    auto dx_ptr = this->GetOutputPtr(&input_grad);
-    std::string dx_name = this->GetOutputName(input_grad);
-
-    VLOG(6) << "Running abs_grad composite func";
-    prim::abs_grad<prim::DescTensor>(input, out_grad, dx_ptr);
-    this->RecoverOutputName(input_grad, dx_name);
-  }
-};
-
-// AbsGrad: dx=dy if x >=0 else -dy
-// AbsDoubleGrad: ddy = ddx if x >=0 else -ddx
-template <typename T>
-class AbsDoubleGradMaker : public framework::SingleGradOpMaker<T> {
- public:
-  using ::paddle::framework::SingleGradOpMaker<T>::SingleGradOpMaker;
-
- protected:
-  void Apply(GradOpPtr<T> op) const override {
-    op->SetType("abs_double_grad");
-    // input1: x
-    op->SetInput("X", this->Input("X"));
-    // input2: ddx
-    op->SetInput("DDX", this->OutputGrad(framework::GradVarName("X")));
-    op->SetAttrMap(this->Attrs());
-    // output: ddy
-    op->SetOutput("DDOut", this->InputGrad(framework::GradVarName("Out")));
-  }
-};
-
-class AbsDoubleGradOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
-  void InferShape(framework::InferShapeContext* ctx) const override {
-    if (ctx->HasOutput("DDOut")) {
-      ctx->ShareDim("X", "DDOut");
-      ctx->ShareLoD("X", "DDOut");
-    }
-  }
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    auto dtype = OperatorWithKernel::IndicateVarDataType(ctx, "DDX");
-    return phi::KernelKey(dtype, ctx.GetPlace());
-  }
-
-  phi::KernelKey GetKernelTypeForVar(
-      const std::string& var_name,
-      const phi::DenseTensor& tensor,
-      const phi::KernelKey& expected_kernel_type) const override {
-    return phi::KernelKey(tensor.place(), tensor.layout(), tensor.dtype());
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-DECLARE_INFER_SHAPE_FUNCTOR(abs,
-                            AbsInferShapeFunctor,
-                            PD_INFER_META(phi::RealAndImagInferMeta));
-
-namespace ops = paddle::operators;
-
-REGISTER_OPERATOR(abs,
-                  ops::AbsOp,
-                  ops::AbsOpMaker,
-                  ops::AbsCompositeGradOpMaker,
-                  ops::AbsGradMaker<paddle::framework::OpDesc>,
-                  ops::AbsGradMaker<paddle::imperative::OpBase>,
-                  AbsInferShapeFunctor);
-
-REGISTER_OPERATOR(abs_grad,
-                  ops::AbsGradOp,
-                  ops::AbsDoubleGradMaker<paddle::framework::OpDesc>,
-                  ops::AbsDoubleGradMaker<paddle::imperative::OpBase>);
-
-REGISTER_OPERATOR(abs_double_grad, ops::AbsDoubleGradOp);
diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index a75fe0e596a26..18738c1af8beb 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -1,6 +1,32 @@
 # This file is designed for backward C++ operators associated with
 # the operator in ops.yaml.
 
+- backward_op : abs_double_grad
+  forward : abs_grad (Tensor x, Tensor grad_out) -> Tensor(grad_x)
+  args : (Tensor x, Tensor grad_x_grad)
+  output : Tensor(grad_out_grad)
+  infer_meta :
+    func : UnchangedInferMeta
+    param : [x]
+  data_transform :
+    support_trans_dtype : x, grad_x_grad
+  kernel :
+    func : abs_double_grad
+    data_type : grad_x_grad
+
+- backward_op : abs_grad
+  forward : abs (Tensor x) -> Tensor(out)
+  args : (Tensor x, Tensor out_grad)
+  output : Tensor(x_grad)
+  infer_meta :
+    func : UnchangedInferMeta
+    param : [x]
+  kernel :
+    func : abs_grad
+    data_type : x
+  composite : abs_grad(x, out_grad, x_grad)
+  backward : abs_double_grad
+
 - backward_op : acos_grad
   forward : acos (Tensor x) -> Tensor(out)
   args : (Tensor x, Tensor out_grad)
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index 4c5106f963b05..87ed42986d45c 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -1,25 +1,3 @@
-- backward_op : abs_double_grad
-  forward : abs_grad (Tensor x, Tensor grad_out) -> Tensor(grad_x)
-  args : (Tensor x, Tensor grad_x_grad)
-  output : Tensor(grad_out_grad)
-  infer_meta :
-    func : UnchangedInferMeta
-    param : [x]
-  kernel :
-    func : abs_double_grad
-
-- backward_op : abs_grad
-  forward : abs (Tensor x) -> Tensor(out)
-  args : (Tensor x, Tensor out_grad)
-  output : Tensor(x_grad)
-  infer_meta :
-    func : UnchangedInferMeta
-    param : [x]
-  kernel :
-    func : abs_grad
-  composite : abs_grad(x, out_grad, x_grad)
-  backward : abs_double_grad
-
 - backward_op : add_double_grad
   forward : add_grad (Tensor x, Tensor y, Tensor grad_out, int axis = -1) -> Tensor(grad_x), Tensor(grad_y)
   args : (Tensor y, Tensor grad_out, Tensor grad_x_grad, Tensor grad_y_grad, int axis = -1)
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index 408fe94987cff..7c113a65dcb43 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -1,15 +1,6 @@
 # The apis in this file are unstandardized that may caused by a variety of reasons,
 # we are trying to fix these apis and will move standardized apis into ops.yaml.
 
-- op : abs
-  args : (Tensor x)
-  output : Tensor
-  infer_meta :
-    func : RealAndImagInferMeta
-  kernel :
-    func : abs
-  backward : abs_grad
-
 - op : adadelta_
   args : (Tensor param, Tensor grad, Tensor avg_squared_grad, Tensor avg_squared_update, Tensor learning_rate, Tensor master_param, float rho, float epsilon, bool multi_precision)
   output : Tensor(param_out), Tensor(moment_out), Tensor(inf_norm_out), Tensor(master_param_out)
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index 7fc3acc5614d2..b7b2521a6a394 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -5,6 +5,16 @@
 # are consistent and correspond one-to-one. It's forbidden that the
 # operator configured in this yaml file does not have Python API.
 
+- op : abs
+  args : (Tensor x)
+  output : Tensor(out)
+  infer_meta :
+    func : RealAndImagInferMeta
+  kernel :
+    func : abs
+    data_type : x
+  backward : abs_grad
+
 - op : accuracy
   args : (Tensor x, Tensor indices, Tensor label)
   output : Tensor(accuracy), Tensor(correct), Tensor(total)
diff --git a/paddle/phi/ops/compat/abs_sig.cc b/paddle/phi/ops/compat/abs_sig.cc
deleted file mode 100644
index 92d29dd0189b5..0000000000000
--- a/paddle/phi/ops/compat/abs_sig.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/core/compat/op_utils.h"
-
-namespace phi {
-
-KernelSignature AbsOpArgumentMapping(const ArgumentMappingContext& ctx) {
-  return KernelSignature("abs", {"X"}, {}, {"Out"});
-}
-
-KernelSignature AbsGradOpArgumentMapping(const ArgumentMappingContext& ctx) {
-  return KernelSignature("abs_grad", {"X", "Out@GRAD"}, {}, {"X@GRAD"});
-}
-
-KernelSignature AbsDoubleGradOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature("abs_double_grad", {"X", "DDX"}, {}, {"DDOut"});
-}
-
-}  // namespace phi
-
-PD_REGISTER_ARG_MAPPING_FN(abs, phi::AbsOpArgumentMapping);
-PD_REGISTER_ARG_MAPPING_FN(abs_grad, phi::AbsGradOpArgumentMapping);
-PD_REGISTER_ARG_MAPPING_FN(abs_double_grad,
-                           phi::AbsDoubleGradOpArgumentMapping);
diff --git a/python/paddle/tensor/layer_function_generator.py b/python/paddle/tensor/layer_function_generator.py
index 5436d2f515a4a..a2ab51d286d64 100644
--- a/python/paddle/tensor/layer_function_generator.py
+++ b/python/paddle/tensor/layer_function_generator.py
@@ -308,12 +308,28 @@ def func(x, name=None):
             return output
 
     func.__name__ = op_type
-    func.__doc__ = _generate_doc_string_(
-        op_proto,
-        additional_args_lines=[
-            "name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`."
-        ],
-    )
+    if op_type == 'abs':
+        func.__doc__ = r"""
+
+Abs Operator.
+Perform elementwise abs for input `X`.
+
+.. math::
+
+    out = |x|
+
+Args:
+    x (Tensor): The input tensor of abs op.
+    out (Tensor): The output tensor of abs op.
+    name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
+"""
+    else:
+        func.__doc__ = _generate_doc_string_(
+            op_proto,
+            additional_args_lines=[
+                "name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`."
+            ],
+        )
     return func
 
 

From 312f018730c31fd02ef291539ee59d08aea7ee86 Mon Sep 17 00:00:00 2001
From: Wang Xin <xinwang614@gmail.com>
Date: Tue, 16 May 2023 14:20:39 +0800
Subject: [PATCH 355/405] static graph autogen code support for softmax op
 (#53581)

* static graph autogen code support for softmax op

* bug fixed

* fix PR-CI-Windows error

* fix CI error

* bug fixed

* fix conflicts
---
 .../generator/get_expected_kernel_func.cc     |  39 ++++
 .../generator/get_expected_kernel_func.h      |   8 +
 paddle/fluid/operators/softmax_op.cc          | 199 ------------------
 paddle/fluid/operators/unity_build_rule.cmake |   3 +-
 paddle/phi/api/yaml/op_compat.yaml            |   7 +-
 paddle/phi/api/yaml/static_backward.yaml      |  11 +
 paddle/phi/api/yaml/static_ops.yaml           |  10 +
 paddle/phi/ops/compat/softmax_sig.cc          |  32 ---
 test/cpp/fluid/CMakeLists.txt                 |   2 +-
 test/cpp/fluid/mkldnn/CMakeLists.txt          |   9 +-
 10 files changed, 81 insertions(+), 239 deletions(-)
 delete mode 100644 paddle/fluid/operators/softmax_op.cc
 delete mode 100644 paddle/phi/ops/compat/softmax_sig.cc

diff --git a/paddle/fluid/operators/generator/get_expected_kernel_func.cc b/paddle/fluid/operators/generator/get_expected_kernel_func.cc
index 49697a48a176b..026d57bba4f7d 100644
--- a/paddle/fluid/operators/generator/get_expected_kernel_func.cc
+++ b/paddle/fluid/operators/generator/get_expected_kernel_func.cc
@@ -139,6 +139,45 @@ phi::KernelKey GetSgdExpectedKernelType(
   return phi::KernelKey(data_type, ctx.GetPlace());
 }
 
+phi::KernelKey GetSoftmaxExpectedKernelType(
+    const framework::ExecutionContext& ctx,
+    const framework::OperatorWithKernel* op_ptr) {
+  // choose cudnn kernel if the runtime supported.
+  std::string data_format = ctx.Attr<std::string>("data_format");
+  phi::DataLayout layout_ = phi::StringToDataLayout(data_format);
+  auto input_data_type = op_ptr->IndicateVarDataType(ctx, "X");
+  if (input_data_type == framework::proto::VarType::FP16) {
+    PADDLE_ENFORCE_EQ(
+        platform::is_gpu_place(ctx.GetPlace()) ||
+            platform::is_xpu_place(ctx.GetPlace()) ||
+            platform::is_custom_place(ctx.GetPlace()),
+        true,
+        platform::errors::InvalidArgument(
+            "float16 can only be used on GPU/XPU and custom place"));
+  }
+  return phi::KernelKey(
+      ctx.GetPlace(), layout_, phi::TransToPhiDataType(input_data_type));
+}
+
+phi::KernelKey GetSoftmaxGradExpectedKernelType(
+    const framework::ExecutionContext& ctx,
+    const framework::OperatorWithKernel* op_ptr) {
+  // choose cudnn kernel if the runtime supported.
+  std::string data_format = ctx.Attr<std::string>("data_format");
+  phi::DataLayout layout_ = phi::StringToDataLayout(data_format);
+  auto input_data_type =
+      op_ptr->IndicateVarDataType(ctx, framework::GradVarName("Out"));
+  if (input_data_type == framework::proto::VarType::FP16) {
+    if (!(platform::is_gpu_place(ctx.GetPlace()) ||
+          platform::is_xpu_place(ctx.GetPlace()) ||
+          platform::is_custom_place(ctx.GetPlace())))
+      PADDLE_THROW(platform::errors::InvalidArgument(
+          "float16 can only be used on GPU/XPU and custom place"));
+  }
+  return phi::KernelKey(
+      ctx.GetPlace(), layout_, phi::TransToPhiDataType(input_data_type));
+}
+
 phi::KernelKey GetUpdateLossScalingExpectedKernelType(
     const framework::ExecutionContext& ctx,
     const framework::OperatorWithKernel* op_ptr) {
diff --git a/paddle/fluid/operators/generator/get_expected_kernel_func.h b/paddle/fluid/operators/generator/get_expected_kernel_func.h
index 4ef889099846a..7923c8d79fb2d 100644
--- a/paddle/fluid/operators/generator/get_expected_kernel_func.h
+++ b/paddle/fluid/operators/generator/get_expected_kernel_func.h
@@ -36,6 +36,14 @@ phi::KernelKey GetSgdExpectedKernelType(
     const framework::ExecutionContext& ctx,
     const framework::OperatorWithKernel* op_ptr);
 
+phi::KernelKey GetSoftmaxExpectedKernelType(
+    const framework::ExecutionContext& ctx,
+    const framework::OperatorWithKernel* op_ptr);
+
+phi::KernelKey GetSoftmaxGradExpectedKernelType(
+    const framework::ExecutionContext& ctx,
+    const framework::OperatorWithKernel* op_ptr);
+
 phi::KernelKey GetUpdateLossScalingExpectedKernelType(
     const framework::ExecutionContext& ctx,
     const framework::OperatorWithKernel* op_ptr);
diff --git a/paddle/fluid/operators/softmax_op.cc b/paddle/fluid/operators/softmax_op.cc
deleted file mode 100644
index 2fb7883cb3f71..0000000000000
--- a/paddle/fluid/operators/softmax_op.cc
+++ /dev/null
@@ -1,199 +0,0 @@
-/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include <memory>
-#include <string>
-#include <unordered_map>
-
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/platform/device/gpu/gpu_dnn.h"
-#include "paddle/fluid/prim/api/composite_backward/composite_backward_api.h"
-#include "paddle/fluid/prim/utils/static/composite_grad_desc_maker.h"
-#include "paddle/fluid/prim/utils/static/desc_tensor.h"
-#include "paddle/phi/core/infermeta_utils.h"
-#include "paddle/phi/infermeta/backward.h"
-#include "paddle/phi/infermeta/unary.h"
-
-namespace paddle {
-namespace operators {
-
-class SoftmaxOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    // choose cudnn kernel if the runtime supported.
-    std::string data_format = ctx.Attr<std::string>("data_format");
-    phi::DataLayout layout_ = phi::StringToDataLayout(data_format);
-    auto input_data_type = OperatorWithKernel::IndicateVarDataType(ctx, "X");
-    if (input_data_type == framework::proto::VarType::FP16) {
-      PADDLE_ENFORCE_EQ(
-          platform::is_gpu_place(ctx.GetPlace()) ||
-              platform::is_xpu_place(ctx.GetPlace()) ||
-              platform::is_custom_place(ctx.GetPlace()),
-          true,
-          platform::errors::InvalidArgument(
-              "float16 can only be used on GPU/XPU and custom place"));
-    }
-    return phi::KernelKey(
-        ctx.GetPlace(), layout_, phi::TransToPhiDataType(input_data_type));
-  }
-};
-
-class SoftmaxOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput("X",
-             "The input tensor of softmax, "
-             "whose dimension :attr:`axis` is the input_feature_dimensions.");
-    AddOutput("Out", "The normalized values with the same shape as X.");
-    AddAttr<int>("axis",
-                 "The dimension index of Input(x) to perform softmax,"
-                 "default -1 for last dimension")
-        .SetDefault(-1);
-    AddAttr<std::string>(
-        "data_format",
-        "(string, default NCHW) Only used in "
-        "An optional string from: \"NHWC\", \"NCHW\". "
-        "Defaults to \"NHWC\". Specify the data format of the output data, "
-        "the input will be transformed automatically. ")
-        .SetDefault("AnyLayout");
-    AddAttr<bool>(
-        "use_cudnn",
-        "(bool, default false) Only used in cudnn kernel, need install cudnn")
-        .SetDefault(false)
-        .AsExtra();
-    AddComment(R"DOC(
-Softmax Operator.
-
-The input of the softmax operator is a tensor of any rank. The output tensor
-has the same shape as the input.
-
-The dimension :attr:`axis` of the input tensor will be permuted to the last.
-Then the input tensor will be logically flattened to a 2-D matrix. The matrix's
-second dimension(row length) is as same as the dimension :attr:`axis` of the input
-tensor, and the first dimension(column length) is the product of all other
-dimensions of the input tensor. For each row of the matrix, the softmax operator
-squashes the K-dimensional(K is the width of the matrix, which is also the size
-of the input tensor's dimension :attr:`axis`) vector of arbitrary real values to a
-K-dimensional vector of real values in the range [0, 1] that add up to 1.
-It computes the exponential of the given dimension and the sum of exponential
-values of all the other dimensions in the K-dimensional vector input.
-Then the ratio of the exponential of the given dimension and the sum of
-exponential values of all the other dimensions is the output of the softmax
-operator.
-
-For each row $i$ and each column $j$ in the matrix, we have:
-    $$Out[i, j] = \frac{\exp(X[i, j])}{\sum_j(exp(X[i, j])}$$
-
-)DOC");
-  }
-};
-
-class SoftmaxOpInferVarType : public framework::PassInDtypeAndVarTypeToOutput {
- protected:
-  std::unordered_map<std::string, std::string>& GetInputOutputWithSameType()
-      const override {
-    static std::unordered_map<std::string, std::string> m{{"X", /*->*/ "Out"}};
-    return m;
-  }
-};
-
-class SoftmaxOpGrad : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    // choose cudnn kernel if the runtime supported.
-    std::string data_format = ctx.Attr<std::string>("data_format");
-    phi::DataLayout layout_ = phi::StringToDataLayout(data_format);
-    auto input_data_type = OperatorWithKernel::IndicateVarDataType(
-        ctx, framework::GradVarName("Out"));
-    if (input_data_type == framework::proto::VarType::FP16) {
-      if (!(platform::is_gpu_place(ctx.GetPlace()) ||
-            platform::is_xpu_place(ctx.GetPlace()) ||
-            platform::is_custom_place(ctx.GetPlace())))
-        PADDLE_THROW(platform::errors::InvalidArgument(
-            "float16 can only be used on GPU/XPU and custom place"));
-    }
-    return phi::KernelKey(
-        ctx.GetPlace(), layout_, phi::TransToPhiDataType(input_data_type));
-  }
-};
-
-template <typename T>
-class SoftmaxOpGradMaker : public framework::SingleGradOpMaker<T> {
- public:
-  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
-
- protected:
-  void Apply(GradOpPtr<T> op) const override {
-    op->SetType("softmax_grad");
-
-    op->SetInput("Out", this->Output("Out"));
-    op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out"));
-
-    op->SetAttrMap(this->Attrs());
-
-    op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
-  }
-};
-
-class SoftmaxCompositeGradOpMaker : public prim::CompositeGradOpMakerBase {
-  using prim::CompositeGradOpMakerBase::CompositeGradOpMakerBase;
-
- public:
-  void Apply() override {
-    paddle::Tensor out = this->GetSingleForwardOutput("Out");
-    paddle::Tensor out_grad = this->GetSingleOutputGrad("Out");
-    paddle::Tensor dx = this->GetSingleInputGrad("X");
-    auto* dx_ptr = this->GetOutputPtr(&dx);
-    std::string dx_name = this->GetOutputName(dx);
-    int axis = static_cast<int>(this->Attr<int>("axis"));
-    VLOG(6) << "Runing softmax_grad composite func";
-    prim::softmax_grad<prim::DescTensor>(out, out_grad, axis, dx_ptr);
-    this->RecoverOutputName(dx, dx_name);
-  }
-};
-
-DECLARE_INPLACE_OP_INFERER(SoftmaxInplaceInferer, {"X", "Out"});
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-
-DECLARE_INFER_SHAPE_FUNCTOR(softmax,
-                            SoftmaxInferShapeFunctor,
-                            PD_INFER_META(phi::SoftmaxInferMeta));
-REGISTER_OPERATOR(softmax,
-                  ops::SoftmaxOp,
-                  ops::SoftmaxOpMaker,
-                  ops::SoftmaxOpInferVarType,
-                  ops::SoftmaxOpGradMaker<paddle::framework::OpDesc>,
-                  ops::SoftmaxOpGradMaker<paddle::imperative::OpBase>,
-                  ops::SoftmaxCompositeGradOpMaker,
-                  ops::SoftmaxInplaceInferer,
-                  SoftmaxInferShapeFunctor);
-DECLARE_INFER_SHAPE_FUNCTOR(softmax_grad,
-                            SoftmaxGradInferShapeFunctor,
-                            PD_INFER_META(phi::GeneralUnaryGradInferMeta));
-REGISTER_OPERATOR(softmax_grad,
-                  ops::SoftmaxOpGrad,
-                  SoftmaxGradInferShapeFunctor);
diff --git a/paddle/fluid/operators/unity_build_rule.cmake b/paddle/fluid/operators/unity_build_rule.cmake
index 6eab7b3f44eeb..740402b155fc2 100644
--- a/paddle/fluid/operators/unity_build_rule.cmake
+++ b/paddle/fluid/operators/unity_build_rule.cmake
@@ -259,8 +259,7 @@ register_unity_group(
   sign_op.cc
   similarity_focus_op.cc
   size_op.cc
-  slice_op.cc
-  softmax_op.cc)
+  slice_op.cc)
 register_unity_group(
   cc
   space_to_depth_op.cc
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index ab8274ac60181..7087e913e9450 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -2147,8 +2147,13 @@
   backward : softmax_grad
   inputs :
     x : X
+  outputs :
+    out : Out
+  get_expected_kernel_type :
+    softmax : GetSoftmaxExpectedKernelType
+    softmax_grad : GetSoftmaxGradExpectedKernelType
   extra :
-    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32", bool is_test = false]
+    attrs : [str data_format = "AnyLayout", bool use_cudnn=false, bool use_mkldnn = false, str mkldnn_data_type = "float32", bool is_test = false]
 
 - op : softplus
   backward : softplus_grad, softplus_double_grad
diff --git a/paddle/phi/api/yaml/static_backward.yaml b/paddle/phi/api/yaml/static_backward.yaml
index 3bda010909c35..5d48977ebabf9 100755
--- a/paddle/phi/api/yaml/static_backward.yaml
+++ b/paddle/phi/api/yaml/static_backward.yaml
@@ -53,3 +53,14 @@
   kernel :
     func : rnn_grad
     data_type: out_grad
+
+- backward_op : softmax_grad
+  forward : softmax (Tensor x, int axis=-1) -> Tensor(out)
+  args : (Tensor out, Tensor out_grad, int axis)
+  output : Tensor(x_grad)
+  infer_meta :
+    func : UnchangedInferMeta
+    param : [out]
+  kernel :
+    func : softmax_grad
+  composite : softmax_grad(out, out_grad, axis, x_grad)
diff --git a/paddle/phi/api/yaml/static_ops.yaml b/paddle/phi/api/yaml/static_ops.yaml
index d1d07681ed189..f481cedbc1c78 100755
--- a/paddle/phi/api/yaml/static_ops.yaml
+++ b/paddle/phi/api/yaml/static_ops.yaml
@@ -335,6 +335,16 @@
   kernel :
     func : share_buffer
 
+- op : softmax
+  args : (Tensor x, int axis = -1)
+  output : Tensor(out)
+  infer_meta :
+    func : SoftmaxInferMeta
+  kernel :
+    func : softmax
+  inplace : (x -> out)
+  backward : softmax_grad
+
 - op : tril_indices
   args : (int rows = 0, int cols = 0, int offset = 0, DataType dtype = DataType::INT64)
   output : Tensor(out)
diff --git a/paddle/phi/ops/compat/softmax_sig.cc b/paddle/phi/ops/compat/softmax_sig.cc
deleted file mode 100644
index a30a2a2b06fd5..0000000000000
--- a/paddle/phi/ops/compat/softmax_sig.cc
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include "paddle/phi/core/compat/op_utils.h"
-
-namespace phi {
-
-KernelSignature SoftmaxOpArgumentMapping(const ArgumentMappingContext& ctx) {
-  return KernelSignature("softmax", {"X"}, {"axis"}, {"Out"});
-}
-
-KernelSignature SoftmaxGradOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature(
-      "softmax_grad", {"Out", "Out@GRAD"}, {"axis"}, {"X@GRAD"});
-}
-
-}  // namespace phi
-
-PD_REGISTER_ARG_MAPPING_FN(softmax, phi::SoftmaxOpArgumentMapping);
-PD_REGISTER_ARG_MAPPING_FN(softmax_grad, phi::SoftmaxGradOpArgumentMapping);
diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt
index 7bc33dfda33e8..590816d1b5e15 100644
--- a/test/cpp/fluid/CMakeLists.txt
+++ b/test/cpp/fluid/CMakeLists.txt
@@ -42,7 +42,7 @@ cc_test(
   test_common_infer_shape_functions
   SRCS test_common_infer_shape_functions.cc
   DEPS common_infer_shape_functions ${COMMON_OP_DEPS} activation_op
-       elementwise_add_op softmax_op softmax)
+       elementwise_add_op softmax generated_static_op)
 cc_test(
   gather_test
   SRCS gather_test.cc
diff --git a/test/cpp/fluid/mkldnn/CMakeLists.txt b/test/cpp/fluid/mkldnn/CMakeLists.txt
index 34f3ce8959c68..dae56ea5eb6d3 100644
--- a/test/cpp/fluid/mkldnn/CMakeLists.txt
+++ b/test/cpp/fluid/mkldnn/CMakeLists.txt
@@ -4,26 +4,27 @@ cc_test(
   DEPS op_registry
        elementwise_add_op
        activation_op
-       softmax_op
        softmax
        scope
        device_context
        enforce
-       executor)
+       executor
+       generated_static_op)
 
 set(TEST_MKLDNN_CACHING_DEPS
     op_registry
     elementwise_mul_op
     elementwise_add_op
     activation_op
-    softmax_op
     conv_op
     im2col
     vol2col
     softmax
     scope
     device_context
-    enforce)
+    enforce
+    generated_static_op)
+
 if(WITH_GPU OR WITH_ROCM)
   set(TEST_MKLDNN_CACHING_DEPS ${TEST_MKLDNN_CACHING_DEPS} depthwise_conv)
 endif()

From 5e5481d8240be06cdd937c1f374fbe2095fd75a4 Mon Sep 17 00:00:00 2001
From: Sonder <55493212+AndSonder@users.noreply.github.com>
Date: Tue, 16 May 2023 14:30:28 +0800
Subject: [PATCH 356/405] Move fused batchnorm to Phi (#53476)

* trans fused batch norm Compute function

* trans batch norm register info to phi

* trans fused batch norm grad Compute

* trans batch norm grad register info

* add sig file

* update sig file

* Update fused_bn_activation_kernel.cu

* Update fused_bn_activation_grad_kernel.cu

* fix

* Rename fused_bn_activation_kernel_grad.cu to fused_bn_activation_kernel.cu

* fix

* fix

* fix CudnnDataType error

* fix

* fix include

* update

* add #if

* add fused bn act to cmakelist.txt

* update  cmakelist

* fix #ifdef error

* add timeout set

* add env set

* fix

* fix

* Update fused_bn_activation_sig.cc
---
 .../operators/fused/fused_bn_activation_op.cu | 444 ------------------
 .../kernels/fused_bn_activation_grad_kernel.h |  37 ++
 .../phi/kernels/fused_bn_activation_kernel.h  |  37 ++
 .../gpu/fused_bn_activation_grad_kernel.cu    | 237 ++++++++++
 .../fusion/gpu/fused_bn_activation_kernel.cu  | 237 ++++++++++
 .../phi/ops/compat/fused_bn_activation_sig.cc |  53 +++
 .../fluid/tests/unittests/CMakeLists.txt      |   8 +
 7 files changed, 609 insertions(+), 444 deletions(-)
 delete mode 100644 paddle/fluid/operators/fused/fused_bn_activation_op.cu
 create mode 100644 paddle/phi/kernels/fused_bn_activation_grad_kernel.h
 create mode 100644 paddle/phi/kernels/fused_bn_activation_kernel.h
 create mode 100644 paddle/phi/kernels/fusion/gpu/fused_bn_activation_grad_kernel.cu
 create mode 100644 paddle/phi/kernels/fusion/gpu/fused_bn_activation_kernel.cu
 create mode 100644 paddle/phi/ops/compat/fused_bn_activation_sig.cc

diff --git a/paddle/fluid/operators/fused/fused_bn_activation_op.cu b/paddle/fluid/operators/fused/fused_bn_activation_op.cu
deleted file mode 100644
index a93938fcfd043..0000000000000
--- a/paddle/fluid/operators/fused/fused_bn_activation_op.cu
+++ /dev/null
@@ -1,444 +0,0 @@
-// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <algorithm>
-#include <cfloat>
-#include <string>
-#include <vector>
-
-#include "cub/cub.cuh"
-#include "paddle/fluid/framework/data_layout.h"
-#include "paddle/fluid/operators/activation_op.h"
-#include "paddle/fluid/operators/fused/fused_bn_activation_op.h"
-#include "paddle/fluid/platform/device/gpu/gpu_dnn.h"
-#include "paddle/fluid/platform/float16.h"
-#include "paddle/phi/core/flags.h"
-#include "paddle/phi/kernels/funcs/math_function.h"
-#include "paddle/phi/kernels/funcs/norm_utils.h"
-
-PHI_DECLARE_bool(cudnn_batchnorm_spatial_persistent);
-
-namespace paddle {
-namespace operators {
-template <typename T>
-using CudnnDataType = platform::CudnnDataType<T>;
-template <typename T>
-using BatchNormParamType = typename CudnnDataType<T>::BatchNormParamType;
-
-template <typename T>
-class FusedBatchNormActKernel<T, phi::GPUContext>
-    : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext &ctx) const override {
-#if CUDNN_VERSION < 7401
-    PADDLE_THROW(phi::errors::Unimplemented(
-        "The fused_batch_norm_act operator is not supported on GPU "
-        "when CUDNN version < 7.4.1"));
-#endif
-    PADDLE_ENFORCE_EQ(
-        platform::is_gpu_place(ctx.GetPlace()),
-        true,
-        platform::errors::PreconditionNotMet("It must use CUDAPlace."));
-    auto &dev_ctx = ctx.template device_context<phi::GPUContext>();
-    double epsilon = static_cast<double>(ctx.Attr<float>("epsilon"));
-    float momentum = ctx.Attr<float>("momentum");
-    std::string act_type = ctx.Attr<std::string>("act_type");
-
-    if (epsilon <= CUDNN_BN_MIN_EPSILON - FLT_EPSILON) {
-      LOG(ERROR) << "Provided epsilon is smaller than "
-                 << "CUDNN_BN_MIN_EPSILON. Setting it to "
-                 << "CUDNN_BN_MIN_EPSILON instead.";
-    }
-    epsilon = std::max(epsilon, CUDNN_BN_MIN_EPSILON);
-
-    // Get the size for each dimension.
-    // NHWC [batch_size, in_height, in_width, in_channels]
-    const auto *x = ctx.Input<phi::DenseTensor>("X");
-    const auto &x_dims = x->dims();
-    PADDLE_ENFORCE_EQ(x_dims.size() >= 2 && x_dims.size() <= 5,
-                      true,
-                      platform::errors::PreconditionNotMet(
-                          "The Input dim size should be between 2 and 5"));
-
-    const auto *scale = ctx.Input<phi::DenseTensor>("Scale");
-    const auto *bias = ctx.Input<phi::DenseTensor>("Bias");
-
-    // Run training mode.
-    // obtain running mean and running inv var, and see if we need to
-    // initialize them.
-    auto *mean_out = ctx.Output<phi::DenseTensor>("MeanOut");
-    auto *variance_out = ctx.Output<phi::DenseTensor>("VarianceOut");
-    dev_ctx.Alloc<BatchNormParamType<T>>(
-        mean_out, mean_out->numel() * sizeof(BatchNormParamType<T>));
-    dev_ctx.Alloc<BatchNormParamType<T>>(
-        variance_out, variance_out->numel() * sizeof(BatchNormParamType<T>));
-
-    auto *saved_mean = ctx.Output<phi::DenseTensor>("SavedMean");
-    auto *saved_variance = ctx.Output<phi::DenseTensor>("SavedVariance");
-    dev_ctx.Alloc<BatchNormParamType<T>>(
-        saved_mean, saved_mean->numel() * sizeof(BatchNormParamType<T>));
-    dev_ctx.Alloc<BatchNormParamType<T>>(
-        saved_variance,
-        saved_variance->numel() * sizeof(BatchNormParamType<T>));
-
-    auto *y = ctx.Output<phi::DenseTensor>("Y");
-    dev_ctx.Alloc<T>(y, y->numel() * sizeof(T));
-
-    int N, C, H, W, D;
-    const DataLayout data_layout = DataLayout::kNHWC;
-    phi::funcs::ExtractNCWHD(x_dims, data_layout, &N, &C, &H, &W, &D);
-
-    if ((N * H * W * D) == 1) {
-      // Only 1 element in normalization dimension,
-      // skip the batch norm calculation, let y = act(x).
-      auto x_v = framework::EigenVector<T>::Flatten(*x);
-      auto y_v = framework::EigenVector<T>::Flatten(*y);
-      auto &dev = *dev_ctx.eigen_device();
-      if (act_type == "relu") {
-        ReluCUDAFunctor<T>()(dev, x_v, y_v);
-      } else {
-        PADDLE_THROW(
-            platform::errors::Unimplemented("Unsupported activation type"));
-      }
-      return;
-    }
-
-    // ------------------- cudnn descriptors ---------------------
-    auto handle = dev_ctx.cudnn_handle();
-    cudnnTensorDescriptor_t data_desc_;
-    cudnnTensorDescriptor_t bn_param_desc_;
-    cudnnBatchNormMode_t mode_ = CUDNN_BATCHNORM_SPATIAL_PERSISTENT;
-
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnCreateTensorDescriptor(&data_desc_));
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnCreateTensorDescriptor(&bn_param_desc_));
-
-    VLOG(3) << "Setting descriptors.";
-    std::vector<int> dims = {N, C, H, W, D};
-    std::vector<int> strides = {H * W * D * C, 1, W * D * C, D * C, C};
-
-    PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnSetTensorNdDescriptor(
-        data_desc_,
-        CudnnDataType<T>::type,
-        x_dims.size() > 3 ? x_dims.size() : 4,
-        dims.data(),
-        strides.data()));
-
-    PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnDeriveBNTensorDescriptor(
-        bn_param_desc_, data_desc_, mode_));
-
-    double this_factor = 1. - momentum;
-    cudnnBatchNormOps_t bnOps_ = CUDNN_BATCHNORM_OPS_BN_ACTIVATION;
-    platform::ScopedActivationDescriptor scope_act_desc;
-    cudnnActivationDescriptor_t activation_desc_ =
-        scope_act_desc.descriptor<T>(act_type);
-    size_t workspace_size = 0;
-    size_t reserve_space_size = 0;
-    void *reserve_space_ptr = nullptr;
-    void *workspace_ptr = nullptr;
-    phi::DenseTensor workspace_tensor;
-    // Create reserve space and workspace for batch norm.
-    // Create tensor for each batchnorm op, it will be used in the
-    // backward. Thus this tensor shouldn't be temp.
-    auto *reserve_space = ctx.Output<phi::DenseTensor>("ReserveSpace");
-    PADDLE_ENFORCE_NOT_NULL(
-        reserve_space,
-        platform::errors::NotFound(
-            "The argument ReserveSpace of batch_norm op is not found."));
-
-    // --------------- cudnn batchnorm workspace ---------------
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::
-            cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize(
-                /*handle=*/handle,
-                /*mode=*/mode_,
-                /*bnOps=*/bnOps_,
-                /*xDesc=*/data_desc_,
-                /*zDesc=*/nullptr,
-                /*yDesc=*/data_desc_,
-                /*bnScaleBiasMeanVarDesc=*/bn_param_desc_,
-                /*activationDesc=*/activation_desc_,
-                /*sizeInBytes=*/&workspace_size));
-
-    // -------------- cudnn batchnorm reserve space --------------
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnGetBatchNormalizationTrainingExReserveSpaceSize(
-            /*handle=*/handle,
-            /*mode=*/mode_,
-            /*bnOps=*/bnOps_,
-            /*activationDesc=*/activation_desc_,
-            /*xDesc=*/data_desc_,
-            /*sizeInBytes=*/&reserve_space_size));
-
-    reserve_space->Resize({static_cast<int64_t>(
-        (reserve_space_size + phi::SizeOf(x->dtype()) - 1) /
-        phi::SizeOf(x->dtype()))});
-    reserve_space_ptr =
-        dev_ctx.Alloc<T>(reserve_space, reserve_space->numel() * sizeof(T));
-    workspace_tensor.Resize(
-        {static_cast<int64_t>((workspace_size + phi::SizeOf(x->dtype()) - 1) /
-                              phi::SizeOf(x->dtype()))});
-    workspace_ptr = dev_ctx.Alloc<T>(&workspace_tensor,
-                                     workspace_tensor.numel() * sizeof(T));
-
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnBatchNormalizationForwardTrainingEx(
-            handle,
-            mode_,
-            bnOps_,
-            CudnnDataType<T>::kOne(),
-            CudnnDataType<T>::kZero(),
-            data_desc_,
-            x->template data<T>(),
-            nullptr,
-            nullptr,
-            data_desc_,
-            y->template data<T>(),
-            bn_param_desc_,
-            scale->template data<BatchNormParamType<T>>(),
-            bias->template data<BatchNormParamType<T>>(),
-            this_factor,
-            dev_ctx.template Alloc<BatchNormParamType<T>>(
-                mean_out, mean_out->numel() * sizeof(BatchNormParamType<T>)),
-            dev_ctx.template Alloc<BatchNormParamType<T>>(
-                variance_out,
-                variance_out->numel() * sizeof(BatchNormParamType<T>)),
-            epsilon,
-            dev_ctx.template Alloc<BatchNormParamType<T>>(
-                saved_mean,
-                saved_mean->numel() * sizeof(BatchNormParamType<T>)),
-            dev_ctx.template Alloc<BatchNormParamType<T>>(
-                saved_variance,
-                saved_variance->numel() * sizeof(BatchNormParamType<T>)),
-            activation_desc_,
-            workspace_ptr,
-            workspace_size,
-            reserve_space_ptr,
-            reserve_space_size));
-
-    // clean when exit.
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnDestroyTensorDescriptor(data_desc_));
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnDestroyTensorDescriptor(bn_param_desc_));
-  }
-};
-
-template <typename T>
-class FusedBatchNormActGradKernel<T, phi::GPUContext>
-    : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext &ctx) const override {
-#if CUDNN_VERSION < 7401
-    PADDLE_THROW(phi::errors::Unimplemented(
-        "The fused_batch_norm_act operator is not supported on GPU "
-        "when CUDNN version < 7.4.1"));
-#endif
-    PADDLE_ENFORCE_EQ(
-        platform::is_gpu_place(ctx.GetPlace()),
-        true,
-        platform::errors::PreconditionNotMet("It must use CUDAPlace."));
-    double epsilon = static_cast<double>(ctx.Attr<float>("epsilon"));
-    std::string act_type = ctx.Attr<std::string>("act_type");
-    auto &dev_ctx = ctx.template device_context<phi::GPUContext>();
-    const auto *x = ctx.Input<phi::DenseTensor>("X");
-    const auto *y = ctx.Input<phi::DenseTensor>("Y");
-    const auto *d_y = ctx.Input<phi::DenseTensor>(framework::GradVarName("Y"));
-    const auto *scale = ctx.Input<phi::DenseTensor>("Scale");
-    const auto *bias = ctx.Input<phi::DenseTensor>("Bias");
-    const auto *reserve_space = ctx.Input<phi::DenseTensor>("ReserveSpace");
-
-    const auto &x_dims = x->dims();
-
-    PADDLE_ENFORCE_EQ(x_dims.size() >= 2 && x_dims.size() <= 5,
-                      true,
-                      platform::errors::PreconditionNotMet(
-                          "The Input dim size should be between 2 and 5"));
-    int N, C, H, W, D;
-    const DataLayout data_layout = DataLayout::kNHWC;
-    phi::funcs::ExtractNCWHD(x_dims, data_layout, &N, &C, &H, &W, &D);
-
-    // init output
-    auto *d_x = ctx.Output<phi::DenseTensor>(framework::GradVarName("X"));
-    auto *d_scale =
-        ctx.Output<phi::DenseTensor>(framework::GradVarName("Scale"));
-    auto *d_bias = ctx.Output<phi::DenseTensor>(framework::GradVarName("Bias"));
-
-    dev_ctx.Alloc<T>(d_x, d_x->numel() * sizeof(T));
-    PADDLE_ENFORCE_EQ(
-        d_scale && d_bias,
-        true,
-        platform::errors::PreconditionNotMet(
-            "Both the scale grad and the bias grad must not be null."));
-    dev_ctx.Alloc<BatchNormParamType<T>>(
-        d_scale, d_scale->numel() * sizeof(BatchNormParamType<T>));
-    dev_ctx.Alloc<BatchNormParamType<T>>(
-        d_bias, d_bias->numel() * sizeof(BatchNormParamType<T>));
-    PADDLE_ENFORCE_EQ(scale->dims().size(),
-                      1UL,
-                      platform::errors::PreconditionNotMet(
-                          "The scale only has one dimension."));
-    PADDLE_ENFORCE_EQ(
-        scale->dims()[0],
-        C,
-        platform::errors::PreconditionNotMet(
-            "The size of scale is equal to the channel of Input(X)."));
-
-    if ((N * H * W * D) == 1) {
-      if (act_type == "relu") {
-        auto x_v = framework::EigenVector<T>::Flatten(*x);
-        auto y_v = framework::EigenVector<T>::Flatten(*y);
-        auto dx_v = framework::EigenVector<T>::Flatten(*d_x);
-        auto dy_v = framework::EigenVector<T>::Flatten(*d_y);
-        auto &dev = *dev_ctx.eigen_device();
-        ReluGradFunctor<T>()(dev, x_v, y_v, dy_v, dx_v);
-      } else {
-        PADDLE_THROW(
-            platform::errors::Unimplemented("Unsupported activation type"));
-      }
-      phi::funcs::SetConstant<phi::GPUContext, BatchNormParamType<T>> functor;
-      functor(dev_ctx, d_scale, static_cast<BatchNormParamType<T>>(0));
-      functor(dev_ctx, d_bias, static_cast<BatchNormParamType<T>>(0));
-      return;
-    }
-
-    std::vector<int> dims = {N, C, H, W, D};
-    std::vector<int> strides = {H * W * C * D, 1, W * D * C, D * C, C};
-    // ------------------- cudnn descriptors ---------------------
-    cudnnTensorDescriptor_t data_desc_;
-    cudnnTensorDescriptor_t bn_param_desc_;
-    cudnnBatchNormMode_t mode_ = CUDNN_BATCHNORM_SPATIAL_PERSISTENT;
-
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnCreateTensorDescriptor(&data_desc_));
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnCreateTensorDescriptor(&bn_param_desc_));
-    if (epsilon <= CUDNN_BN_MIN_EPSILON - FLT_EPSILON) {
-      LOG(ERROR) << "Provided epsilon is smaller than "
-                 << "CUDNN_BN_MIN_EPSILON. Setting it to "
-                 << "CUDNN_BN_MIN_EPSILON instead.";
-    }
-    epsilon = std::max(epsilon, CUDNN_BN_MIN_EPSILON);
-
-    PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnSetTensorNdDescriptor(
-        data_desc_,
-        CudnnDataType<T>::type,
-        x_dims.size() > 3 ? x_dims.size() : 4,
-        dims.data(),
-        strides.data()));
-    PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnDeriveBNTensorDescriptor(
-        bn_param_desc_, data_desc_, mode_));
-
-    const auto *saved_mean = ctx.Input<phi::DenseTensor>("SavedMean");
-    const auto *saved_var = ctx.Input<phi::DenseTensor>("SavedVariance");
-    const auto *saved_mean_data =
-        saved_mean->template data<BatchNormParamType<T>>();
-    const auto *saved_var_data =
-        saved_var->template data<BatchNormParamType<T>>();
-
-    size_t workspace_size = 0;
-    void *workspace_ptr = nullptr;
-    phi::DenseTensor workspace_tensor;
-    auto reserve_space_size = reserve_space->memory_size();
-    cudnnBatchNormOps_t bnOps_ = CUDNN_BATCHNORM_OPS_BN_ACTIVATION;
-    platform::ScopedActivationDescriptor scope_act_desc;
-    cudnnActivationDescriptor_t activation_desc_ =
-        scope_act_desc.descriptor<T>(act_type);
-    // --------------- cudnn batchnorm workspace ---------------
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnGetBatchNormalizationBackwardExWorkspaceSize(
-            /*handle=*/dev_ctx.cudnn_handle(),
-            /*mode=*/mode_,
-            /*bnOps=*/bnOps_,
-            /*xDesc=*/data_desc_,
-            /*yDesc=*/data_desc_,
-            /*dyDesc=*/data_desc_,
-            /*dzDesc=*/nullptr,
-            /*dxDesc=*/data_desc_,
-            /*bnScaleBiasMeanVarDesc=*/bn_param_desc_,
-            /*activationDesc=*/activation_desc_,
-            /*sizeInBytes=*/&workspace_size));
-
-    workspace_tensor.Resize(
-        {static_cast<int64_t>((workspace_size + phi::SizeOf(x->dtype()) - 1) /
-                              phi::SizeOf(x->dtype()))});
-    workspace_ptr = dev_ctx.Alloc<T>(&workspace_tensor,
-                                     workspace_tensor.numel() * sizeof(T));
-
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnBatchNormalizationBackwardEx(
-            /*handle=*/dev_ctx.cudnn_handle(),
-            /*mode=*/mode_,
-            /*bnOps=*/bnOps_,
-            /*alphaDataDiff=*/CudnnDataType<T>::kOne(),
-            /*betaDataDiff=*/CudnnDataType<T>::kZero(),
-            /*alphaParamDiff=*/CudnnDataType<T>::kOne(),
-            /*betaParamDiff=*/CudnnDataType<T>::kZero(),
-            /*xDesc=*/data_desc_,
-            /*xData=*/x->template data<T>(),
-            /*yDesc=*/data_desc_,
-            /*yData=*/y->template data<T>(),
-            /*dyDesc=*/data_desc_,
-            /*dyData=*/d_y->template data<T>(),
-            /*dzDesc=*/nullptr,
-            /*dzData=*/nullptr,
-            /*dxDesc=*/data_desc_,
-            /*dxData=*/
-            dev_ctx.template Alloc<T>(d_x, d_x->numel() * sizeof(T)),
-            /*dBnScaleBiasDesc=*/bn_param_desc_,
-            /*bnScaleData=*/scale->template data<BatchNormParamType<T>>(),
-            /*bnBiasData=*/bias->template data<BatchNormParamType<T>>(),
-            /*dBnScaleData=*/
-            dev_ctx.template Alloc<BatchNormParamType<T>>(
-                d_scale, d_scale->numel() * sizeof(BatchNormParamType<T>)),
-            /*dBnBiasData=*/
-            dev_ctx.template Alloc<BatchNormParamType<T>>(
-                d_bias, d_bias->numel() * sizeof(BatchNormParamType<T>)),
-            /*epsilon=*/epsilon,
-            /*savedMean=*/saved_mean_data,
-            /*savedInvVariance=*/saved_var_data,
-            /*activationDesc=*/activation_desc_,
-            /*workspace=*/workspace_ptr,
-            /*workSpaceSizeInBytes=*/workspace_size,
-            /*reserveSpace=*/const_cast<T *>(reserve_space->template data<T>()),
-            /*reserveSpaceSizeInBytes=*/reserve_space_size));
-
-    // clean when exit.
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnDestroyTensorDescriptor(data_desc_));
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnDestroyTensorDescriptor(bn_param_desc_));
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-namespace plat = paddle::platform;
-PD_REGISTER_STRUCT_KERNEL(fused_batch_norm_act,
-                          GPU,
-                          ALL_LAYOUT,
-                          ops::FusedBatchNormActKernel,
-                          float,
-                          double,
-                          plat::float16) {}
-PD_REGISTER_STRUCT_KERNEL(fused_batch_norm_act_grad,
-                          GPU,
-                          ALL_LAYOUT,
-                          ops::FusedBatchNormActGradKernel,
-                          float,
-                          double,
-                          plat::float16) {}
diff --git a/paddle/phi/kernels/fused_bn_activation_grad_kernel.h b/paddle/phi/kernels/fused_bn_activation_grad_kernel.h
new file mode 100644
index 0000000000000..1f74e44cbb009
--- /dev/null
+++ b/paddle/phi/kernels/fused_bn_activation_grad_kernel.h
@@ -0,0 +1,37 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void FusedBatchNormActGradKernel(const Context &dev_ctx,
+                                 const DenseTensor &x,
+                                 const DenseTensor &scale,
+                                 const DenseTensor &bias,
+                                 const DenseTensor &saved_mean,
+                                 const DenseTensor &saved_variance,
+                                 const DenseTensor &reserve_space,
+                                 const DenseTensor &y,
+                                 const DenseTensor &y_grad,
+                                 float momentum,
+                                 float epsilon,
+                                 const std::string &act_type,
+                                 DenseTensor *x_grad,
+                                 DenseTensor *scale_grad,
+                                 DenseTensor *bias_grad);
+}  // namespace phi
diff --git a/paddle/phi/kernels/fused_bn_activation_kernel.h b/paddle/phi/kernels/fused_bn_activation_kernel.h
new file mode 100644
index 0000000000000..d3f26d8c4e341
--- /dev/null
+++ b/paddle/phi/kernels/fused_bn_activation_kernel.h
@@ -0,0 +1,37 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void FusedBatchNormActKernel(const Context &dev_ctx,
+                             const DenseTensor &x,
+                             const DenseTensor &scale,
+                             const DenseTensor &bias,
+                             const DenseTensor &mean,
+                             const DenseTensor &variance,
+                             float momentum,
+                             float epsilon,
+                             const std::string &act_type,
+                             DenseTensor *y,
+                             DenseTensor *mean_out,
+                             DenseTensor *variance_out,
+                             DenseTensor *saved_mean,
+                             DenseTensor *saved_variance,
+                             DenseTensor *reserve_space);
+}  // namespace phi
diff --git a/paddle/phi/kernels/fusion/gpu/fused_bn_activation_grad_kernel.cu b/paddle/phi/kernels/fusion/gpu/fused_bn_activation_grad_kernel.cu
new file mode 100644
index 0000000000000..a27eb5149308e
--- /dev/null
+++ b/paddle/phi/kernels/fusion/gpu/fused_bn_activation_grad_kernel.cu
@@ -0,0 +1,237 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <cfloat>
+#include <string>
+#include <vector>
+
+#ifdef __NVCC__
+#include "cub/cub.cuh"
+#endif
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/backends/gpu/gpu_dnn.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/flags.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/activation_functor.h"
+#include "paddle/phi/kernels/funcs/eigen/common.h"
+#include "paddle/phi/kernels/funcs/math_function.h"
+#include "paddle/phi/kernels/funcs/norm_utils.h"
+#include "paddle/phi/kernels/fused_bn_activation_grad_kernel.h"
+
+PHI_DECLARE_bool(cudnn_batchnorm_spatial_persistent);
+
+namespace phi {
+namespace fusion {
+
+template <typename T, typename Context>
+void FusedBatchNormActGradKernel(const Context &dev_ctx,
+                                 const DenseTensor &x,
+                                 const DenseTensor &scale,
+                                 const DenseTensor &bias,
+                                 const DenseTensor &saved_mean,
+                                 const DenseTensor &saved_variance,
+                                 const DenseTensor &reserve_space,
+                                 const DenseTensor &y,
+                                 const DenseTensor &y_grad,
+                                 float momentum,
+                                 float epsilon,
+                                 const std::string &act_type,
+                                 DenseTensor *x_grad,
+                                 DenseTensor *scale_grad,
+                                 DenseTensor *bias_grad) {
+// Note(andsonder): Fused bn activation only used in the gpu place.
+#if defined(PADDLE_WITH_CUDA) and CUDNN_VERSION >= 7401
+  using CudnnDataType = phi::backends::gpu::CudnnDataType<T>;
+  using BatchNormParamType = typename CudnnDataType::BatchNormParamType;
+  bool is_gpu_place = dev_ctx.GetPlace().GetType() == phi::AllocationType::GPU;
+  PADDLE_ENFORCE_EQ(is_gpu_place,
+                    true,
+                    phi::errors::PreconditionNotMet("It must use CUDAPlace."));
+  double epsilon1 = static_cast<double>(epsilon);
+
+  const auto *d_y = &y_grad;
+
+  const auto &x_dims = x.dims();
+
+  PADDLE_ENFORCE_EQ(x_dims.size() >= 2 && x_dims.size() <= 5,
+                    true,
+                    phi::errors::PreconditionNotMet(
+                        "The Input dim size should be between 2 and 5"));
+  int N, C, H, W, D;
+  const phi::DataLayout data_layout = phi::DataLayout::kNHWC;
+  phi::funcs::ExtractNCWHD(x_dims, data_layout, &N, &C, &H, &W, &D);
+
+  // init output
+  auto *d_x = x_grad;
+  auto *d_scale = scale_grad;
+  auto *d_bias = bias_grad;
+
+  dev_ctx.template Alloc<T>(d_x);
+  PADDLE_ENFORCE_EQ(
+      d_scale && d_bias,
+      true,
+      phi::errors::PreconditionNotMet(
+          "Both the scale grad and the bias grad must not be null."));
+  dev_ctx.template Alloc<BatchNormParamType>(d_scale);
+  dev_ctx.template Alloc<BatchNormParamType>(d_bias);
+  PADDLE_ENFORCE_EQ(
+      scale.dims().size(),
+      1UL,
+      phi::errors::PreconditionNotMet("The scale only has one dimension."));
+  PADDLE_ENFORCE_EQ(
+      scale.dims()[0],
+      C,
+      phi::errors::PreconditionNotMet(
+          "The size of scale is equal to the channel of Input(X)."));
+
+  if ((N * H * W * D) == 1) {
+    if (act_type == "relu") {
+      auto x_v = phi::EigenVector<T>::Flatten(x);
+      auto y_v = phi::EigenVector<T>::Flatten(y);
+      auto dx_v = phi::EigenVector<T>::Flatten(*d_x);
+      auto dy_v = phi::EigenVector<T>::Flatten(*d_y);
+      auto &dev = *dev_ctx.eigen_device();
+      phi::funcs::ReluGradFunctor<T>()(dev, x_v, y_v, dy_v, dx_v);
+    } else {
+      PADDLE_THROW(phi::errors::Unimplemented("Unsupported activation type"));
+    }
+    phi::funcs::SetConstant<phi::GPUContext, BatchNormParamType> functor;
+    functor(dev_ctx, d_scale, static_cast<BatchNormParamType>(0));
+    functor(dev_ctx, d_bias, static_cast<BatchNormParamType>(0));
+    return;
+  }
+
+  std::vector<int> dims = {N, C, H, W, D};
+  std::vector<int> strides = {H * W * C * D, 1, W * D * C, D * C, C};
+  // ------------------- cudnn descriptors ---------------------
+  cudnnTensorDescriptor_t data_desc_;
+  cudnnTensorDescriptor_t bn_param_desc_;
+  cudnnBatchNormMode_t mode_ = CUDNN_BATCHNORM_SPATIAL_PERSISTENT;
+
+  PADDLE_ENFORCE_GPU_SUCCESS(
+      phi::dynload::cudnnCreateTensorDescriptor(&data_desc_));
+  PADDLE_ENFORCE_GPU_SUCCESS(
+      phi::dynload::cudnnCreateTensorDescriptor(&bn_param_desc_));
+  if (epsilon1 <= CUDNN_BN_MIN_EPSILON - FLT_EPSILON) {
+    LOG(ERROR) << "Provided epsilon is smaller than "
+               << "CUDNN_BN_MIN_EPSILON. Setting it to "
+               << "CUDNN_BN_MIN_EPSILON instead.";
+  }
+  epsilon1 = std::max(epsilon1, CUDNN_BN_MIN_EPSILON);
+
+  PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnSetTensorNdDescriptor(
+      data_desc_,
+      CudnnDataType::type,
+      x_dims.size() > 3 ? x_dims.size() : 4,
+      dims.data(),
+      strides.data()));
+  PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnDeriveBNTensorDescriptor(
+      bn_param_desc_, data_desc_, mode_));
+
+  const auto *saved_mean_data = saved_mean.template data<BatchNormParamType>();
+  const auto *saved_var_data =
+      saved_variance.template data<BatchNormParamType>();
+
+  size_t workspace_size = 0;
+  void *workspace_ptr = nullptr;
+  phi::DenseTensor workspace_tensor;
+  auto reserve_space_size = reserve_space.memory_size();
+  cudnnBatchNormOps_t bnOps_ = CUDNN_BATCHNORM_OPS_BN_ACTIVATION;
+  phi::backends::gpu::ScopedActivationDescriptor scope_act_desc;
+  cudnnActivationDescriptor_t activation_desc_ =
+      scope_act_desc.descriptor<T>(act_type);
+  // --------------- cudnn batchnorm workspace ---------------
+  PADDLE_ENFORCE_GPU_SUCCESS(
+      phi::dynload::cudnnGetBatchNormalizationBackwardExWorkspaceSize(
+          /*handle=*/dev_ctx.cudnn_handle(),
+          /*mode=*/mode_,
+          /*bnOps=*/bnOps_,
+          /*xDesc=*/data_desc_,
+          /*yDesc=*/data_desc_,
+          /*dyDesc=*/data_desc_,
+          /*dzDesc=*/nullptr,
+          /*dxDesc=*/data_desc_,
+          /*bnScaleBiasMeanVarDesc=*/bn_param_desc_,
+          /*activationDesc=*/activation_desc_,
+          /*sizeInBytes=*/&workspace_size));
+
+  workspace_tensor.Resize({static_cast<int64_t>(
+      (workspace_size + phi::SizeOf(x.dtype()) - 1) / phi::SizeOf(x.dtype()))});
+  workspace_ptr = dev_ctx.template Alloc<T>(&workspace_tensor);
+
+  PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnBatchNormalizationBackwardEx(
+      /*handle=*/dev_ctx.cudnn_handle(),
+      /*mode=*/mode_,
+      /*bnOps=*/bnOps_,
+      /*alphaDataDiff=*/CudnnDataType::kOne(),
+      /*betaDataDiff=*/CudnnDataType::kZero(),
+      /*alphaParamDiff=*/CudnnDataType::kOne(),
+      /*betaParamDiff=*/CudnnDataType::kZero(),
+      /*xDesc=*/data_desc_,
+      /*xData=*/x.template data<T>(),
+      /*yDesc=*/data_desc_,
+      /*yData=*/y.template data<T>(),
+      /*dyDesc=*/data_desc_,
+      /*dyData=*/d_y->template data<T>(),
+      /*dzDesc=*/nullptr,
+      /*dzData=*/nullptr,
+      /*dxDesc=*/data_desc_,
+      /*dxData=*/
+      dev_ctx.template Alloc<T>(d_x, d_x->numel() * sizeof(T)),
+      /*dBnScaleBiasDesc=*/bn_param_desc_,
+      /*bnScaleData=*/scale.template data<BatchNormParamType>(),
+      /*bnBiasData=*/bias.template data<BatchNormParamType>(),
+      /*dBnScaleData=*/
+      dev_ctx.template Alloc<BatchNormParamType>(d_scale),
+      /*dBnBiasData=*/
+      dev_ctx.template Alloc<BatchNormParamType>(d_bias),
+      /*epsilon=*/epsilon1,
+      /*savedMean=*/saved_mean_data,
+      /*savedInvVariance=*/saved_var_data,
+      /*activationDesc=*/activation_desc_,
+      /*workspace=*/workspace_ptr,
+      /*workSpaceSizeInBytes=*/workspace_size,
+      /*reserveSpace=*/const_cast<T *>(reserve_space.template data<T>()),
+      /*reserveSpaceSizeInBytes=*/reserve_space_size));
+
+  // clean when exit.
+  PADDLE_ENFORCE_GPU_SUCCESS(
+      phi::dynload::cudnnDestroyTensorDescriptor(data_desc_));
+  PADDLE_ENFORCE_GPU_SUCCESS(
+      phi::dynload::cudnnDestroyTensorDescriptor(bn_param_desc_));
+#else
+  PADDLE_THROW(phi::errors::Unimplemented(
+      "The fused_batch_norm_act operator is not supported on GPU "
+      "when CUDNN version < 7.4.1"));
+#endif
+}
+
+}  // namespace fusion
+}  // namespace phi
+
+PD_REGISTER_KERNEL(fused_batch_norm_act_grad,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::fusion::FusedBatchNormActGradKernel,
+                   float,
+                   double,
+                   phi::dtype::float16) {
+  if (kernel_key.dtype() == phi::DataType::FLOAT16) {
+    kernel->OutputAt(1).SetDataType(phi::DataType::FLOAT32);
+    kernel->OutputAt(2).SetDataType(phi::DataType::FLOAT32);
+  }
+}
diff --git a/paddle/phi/kernels/fusion/gpu/fused_bn_activation_kernel.cu b/paddle/phi/kernels/fusion/gpu/fused_bn_activation_kernel.cu
new file mode 100644
index 0000000000000..700141f1e0331
--- /dev/null
+++ b/paddle/phi/kernels/fusion/gpu/fused_bn_activation_kernel.cu
@@ -0,0 +1,237 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <cfloat>
+#include <string>
+#include <vector>
+
+#ifdef __NVCC__
+#include "cub/cub.cuh"
+#endif
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/backends/gpu/gpu_dnn.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/flags.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/activation_functor.h"
+#include "paddle/phi/kernels/funcs/eigen/common.h"
+#include "paddle/phi/kernels/funcs/math_function.h"
+#include "paddle/phi/kernels/funcs/norm_utils.h"
+
+PHI_DECLARE_bool(cudnn_batchnorm_spatial_persistent);
+
+namespace phi {
+namespace fusion {
+
+template <typename T, typename Context>
+void FusedBatchNormActKernel(const Context &dev_ctx,
+                             const DenseTensor &x,
+                             const DenseTensor &scale,
+                             const DenseTensor &bias,
+                             const DenseTensor &mean,
+                             const DenseTensor &variance,
+                             float momentum,
+                             float epsilon,
+                             const std::string &act_type,
+                             DenseTensor *y,
+                             DenseTensor *mean_out,
+                             DenseTensor *variance_out,
+                             DenseTensor *saved_mean,
+                             DenseTensor *saved_variance,
+                             DenseTensor *reserve_space) {
+// Note(andsonder): Fused bn activation only used in the gpu place.
+#if defined(PADDLE_WITH_CUDA) and CUDNN_VERSION >= 7401
+  using CudnnDataType = phi::backends::gpu::CudnnDataType<T>;
+  using BatchNormParamType = typename CudnnDataType::BatchNormParamType;
+  bool is_gpu_place = dev_ctx.GetPlace().GetType() == phi::AllocationType::GPU;
+  PADDLE_ENFORCE_EQ(is_gpu_place,
+                    true,
+                    phi::errors::PreconditionNotMet("It must use CUDAPlace."));
+  double epsilon1 = static_cast<double>(epsilon);
+
+  if (epsilon1 <= CUDNN_BN_MIN_EPSILON - FLT_EPSILON) {
+    LOG(ERROR) << "Provided epsilon is smaller than "
+               << "CUDNN_BN_MIN_EPSILON. Setting it to "
+               << "CUDNN_BN_MIN_EPSILON instead.";
+  }
+  epsilon1 = std::max(epsilon1, CUDNN_BN_MIN_EPSILON);
+
+  // Get the size for each dimension.
+  // NHWC [batch_size, in_height, in_width, in_channels]
+  const auto &x_dims = x.dims();
+  PADDLE_ENFORCE_EQ(x_dims.size() >= 2 && x_dims.size() <= 5,
+                    true,
+                    phi::errors::PreconditionNotMet(
+                        "The Input dim size should be between 2 and 5"));
+
+  // Run training mode.
+  // obtain running mean and running inv var, and see if we need to
+  // initialize them.
+  dev_ctx.template Alloc<BatchNormParamType>(mean_out);
+  dev_ctx.template Alloc<BatchNormParamType>(variance_out);
+
+  dev_ctx.template Alloc<BatchNormParamType>(saved_mean);
+  dev_ctx.template Alloc<BatchNormParamType>(saved_variance);
+
+  dev_ctx.template Alloc<T>(y);
+
+  int N, C, H, W, D;
+  const DataLayout data_layout = phi::DataLayout::kNHWC;
+  phi::funcs::ExtractNCWHD(x_dims, data_layout, &N, &C, &H, &W, &D);
+
+  if ((N * H * W * D) == 1) {
+    // Only 1 element in normalization dimension,
+    // skip the batch norm calculation, let y = act(x).
+    auto x_v = phi::EigenVector<T>::Flatten(x);
+    auto y_v = phi::EigenVector<T>::Flatten(*y);
+    auto &dev = *dev_ctx.eigen_device();
+    if (act_type == "relu") {
+      phi::funcs::ReluCUDAFunctor<T>()(dev, x_v, y_v);
+    } else {
+      PADDLE_THROW(phi::errors::Unimplemented("Unsupported activation type"));
+    }
+    return;
+  }
+
+  // ------------------- cudnn descriptors ---------------------
+  auto handle = dev_ctx.cudnn_handle();
+  cudnnTensorDescriptor_t data_desc_;
+  cudnnTensorDescriptor_t bn_param_desc_;
+  cudnnBatchNormMode_t mode_ = CUDNN_BATCHNORM_SPATIAL_PERSISTENT;
+
+  PADDLE_ENFORCE_GPU_SUCCESS(
+      phi::dynload::cudnnCreateTensorDescriptor(&data_desc_));
+  PADDLE_ENFORCE_GPU_SUCCESS(
+      phi::dynload::cudnnCreateTensorDescriptor(&bn_param_desc_));
+
+  VLOG(3) << "Setting descriptors.";
+  std::vector<int> dims = {N, C, H, W, D};
+  std::vector<int> strides = {H * W * D * C, 1, W * D * C, D * C, C};
+
+  PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnSetTensorNdDescriptor(
+      data_desc_,
+      CudnnDataType::type,
+      x_dims.size() > 3 ? x_dims.size() : 4,
+      dims.data(),
+      strides.data()));
+
+  PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnDeriveBNTensorDescriptor(
+      bn_param_desc_, data_desc_, mode_));
+
+  double this_factor = 1. - momentum;
+  cudnnBatchNormOps_t bnOps_ = CUDNN_BATCHNORM_OPS_BN_ACTIVATION;
+  phi::backends::gpu::ScopedActivationDescriptor scope_act_desc;
+  cudnnActivationDescriptor_t activation_desc_ =
+      scope_act_desc.descriptor<T>(act_type);
+  size_t workspace_size = 0;
+  size_t reserve_space_size = 0;
+  void *reserve_space_ptr = nullptr;
+  void *workspace_ptr = nullptr;
+  phi::DenseTensor workspace_tensor;
+
+  PADDLE_ENFORCE_NOT_NULL(
+      reserve_space,
+      phi::errors::NotFound(
+          "The argument ReserveSpace of batch_norm op is not found."));
+
+  // --------------- cudnn batchnorm workspace ---------------
+  PADDLE_ENFORCE_GPU_SUCCESS(
+      phi::dynload::cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize(
+          /*handle=*/handle,
+          /*mode=*/mode_,
+          /*bnOps=*/bnOps_,
+          /*xDesc=*/data_desc_,
+          /*zDesc=*/nullptr,
+          /*yDesc=*/data_desc_,
+          /*bnScaleBiasMeanVarDesc=*/bn_param_desc_,
+          /*activationDesc=*/activation_desc_,
+          /*sizeInBytes=*/&workspace_size));
+
+  // -------------- cudnn batchnorm reserve space --------------
+  PADDLE_ENFORCE_GPU_SUCCESS(
+      phi::dynload::cudnnGetBatchNormalizationTrainingExReserveSpaceSize(
+          /*handle=*/handle,
+          /*mode=*/mode_,
+          /*bnOps=*/bnOps_,
+          /*activationDesc=*/activation_desc_,
+          /*xDesc=*/data_desc_,
+          /*sizeInBytes=*/&reserve_space_size));
+
+  reserve_space->Resize(
+      {static_cast<int64_t>((reserve_space_size + phi::SizeOf(x.dtype()) - 1) /
+                            phi::SizeOf(x.dtype()))});
+  reserve_space_ptr = dev_ctx.template Alloc<T>(reserve_space);
+  workspace_tensor.Resize({static_cast<int64_t>(
+      (workspace_size + phi::SizeOf(x.dtype()) - 1) / phi::SizeOf(x.dtype()))});
+  workspace_ptr = dev_ctx.template Alloc<T>(&workspace_tensor);
+
+  PADDLE_ENFORCE_GPU_SUCCESS(
+      phi::dynload::cudnnBatchNormalizationForwardTrainingEx(
+          handle,
+          mode_,
+          bnOps_,
+          CudnnDataType::kOne(),
+          CudnnDataType::kZero(),
+          data_desc_,
+          x.template data<T>(),
+          nullptr,
+          nullptr,
+          data_desc_,
+          y->template data<T>(),
+          bn_param_desc_,
+          scale.template data<BatchNormParamType>(),
+          bias.template data<BatchNormParamType>(),
+          this_factor,
+          dev_ctx.template Alloc<BatchNormParamType>(mean_out),
+          dev_ctx.template Alloc<BatchNormParamType>(variance_out),
+          epsilon1,
+          dev_ctx.template Alloc<BatchNormParamType>(saved_mean),
+          dev_ctx.template Alloc<BatchNormParamType>(saved_variance),
+          activation_desc_,
+          workspace_ptr,
+          workspace_size,
+          reserve_space_ptr,
+          reserve_space_size));
+
+  // clean when exit.
+  PADDLE_ENFORCE_GPU_SUCCESS(
+      phi::dynload::cudnnDestroyTensorDescriptor(data_desc_));
+  PADDLE_ENFORCE_GPU_SUCCESS(
+      phi::dynload::cudnnDestroyTensorDescriptor(bn_param_desc_));
+#else
+  PADDLE_THROW(phi::errors::Unimplemented(
+      "The fused_batch_norm_act operator is not supported on GPU "
+      "when CUDNN version < 7.4.1"));
+#endif
+}
+
+}  // namespace fusion
+}  // namespace phi
+
+PD_REGISTER_KERNEL(fused_batch_norm_act,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::fusion::FusedBatchNormActKernel,
+                   float,
+                   double,
+                   phi::dtype::float16) {
+  if (kernel_key.dtype() == phi::DataType::FLOAT16) {
+    kernel->OutputAt(1).SetDataType(phi::DataType::FLOAT32);
+    kernel->OutputAt(2).SetDataType(phi::DataType::FLOAT32);
+    kernel->OutputAt(3).SetDataType(phi::DataType::FLOAT32);
+    kernel->OutputAt(4).SetDataType(phi::DataType::FLOAT32);
+  }
+}
diff --git a/paddle/phi/ops/compat/fused_bn_activation_sig.cc b/paddle/phi/ops/compat/fused_bn_activation_sig.cc
new file mode 100644
index 0000000000000..1d478057debc2
--- /dev/null
+++ b/paddle/phi/ops/compat/fused_bn_activation_sig.cc
@@ -0,0 +1,53 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/core/compat/op_utils.h"
+
+namespace phi {
+
+KernelSignature BatchNormActFuseOpArgumentMapping(
+    const ArgumentMappingContext& ctx) {
+  return KernelSignature("fused_batch_norm_act",
+                         {"X", "Scale", "Bias", "Mean", "Variance"},
+                         {"momentum", "epsilon", "act_type"},
+                         {"Y",
+                          "MeanOut",
+                          "VarianceOut",
+                          "SavedMean",
+                          "SavedVariance",
+                          "ReserveSpace"});
+}
+
+KernelSignature BatchNormActGradFuseOpArgumentMapping(
+    const ArgumentMappingContext& ctx) {
+  return KernelSignature("fused_batch_norm_act_grad",
+                         {"X",
+                          "Scale",
+                          "Bias",
+                          "SavedMean",
+                          "SavedVariance",
+                          "ReserveSpace",
+                          "Y",
+                          "Y@GRAD"},
+                         {"momentum", "epsilon", "act_type"},
+                         {"X@GRAD", "Scale@GRAD", "Bias@GRAD"});
+}
+
+}  // namespace phi
+
+PD_REGISTER_ARG_MAPPING_FN(fused_batch_norm_act,
+                           phi::BatchNormActFuseOpArgumentMapping);
+
+PD_REGISTER_ARG_MAPPING_FN(fused_batch_norm_act_grad,
+                           phi::BatchNormActGradFuseOpArgumentMapping);
diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt
index 8ca0bc01ab31b..0cf206084a21c 100755
--- a/python/paddle/fluid/tests/unittests/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt
@@ -1163,6 +1163,7 @@ set(STATIC_BUILD_TESTS
     test_fetch_lod_tensor_array
     test_fused_attention_op
     test_fused_attention_op_api
+    test_fuse_bn_act_pass
     test_fused_feedforward_op
     test_fused_feedforward_pass
     test_imperative_optimizer
@@ -1206,6 +1207,13 @@ endforeach()
 
 set_tests_properties(test_decoupled_py_reader_static_build PROPERTIES TIMEOUT
                                                                       120)
+set_tests_properties(test_fuse_bn_act_pass_static_build PROPERTIES TIMEOUT 120)
+set_tests_properties(
+  test_fuse_bn_act_pass_static_build
+  PROPERTIES
+    ENVIRONMENT
+    "FLAGS_cudnn_deterministic=1;FLAGS_cudnn_batchnorm_spatial_persistent=1;FLAGS_conv_workspace_size_limit=1000"
+)
 set_tests_properties(test_imperative_optimizer_static_build PROPERTIES TIMEOUT
                                                                        250)
 set_tests_properties(test_matmul_op_static_build PROPERTIES TIMEOUT 120)

From 0689e2a51ce1b847fc453f99103e86a4efd3c4bb Mon Sep 17 00:00:00 2001
From: cxxly <chenxx_id@163.com>
Date: Fri, 12 May 2023 12:30:31 +0000
Subject: [PATCH 357/405] fix _strip_grad_suffix_ bugs when input patten is
 'x@GRAD@RENAME'

---
 python/paddle/fluid/backward.py                    |  5 +++--
 .../paddle/fluid/tests/unittests/test_backward.py  | 14 ++++++++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py
index ff57966eefbfb..51fe6294a5077 100755
--- a/python/paddle/fluid/backward.py
+++ b/python/paddle/fluid/backward.py
@@ -464,9 +464,10 @@ def _strip_grad_suffix_(name):
          x@GRAD@GRAD ==> x
          y@GRAD@RENAME@1 ==> y
          z@GRAD_slice_0@GRAD ==> z@GRAD_slice_0
+         grad/grad/z@GRAD@RENAME@block0@1@GRAD ==> z
     """
-    pos = re.search(f'{core.grad_var_suffix()}$', name) or re.search(
-        f'{core.grad_var_suffix()}@', name
+    pos = re.search(f'{core.grad_var_suffix()}+@', name) or re.search(
+        f'{core.grad_var_suffix()}$', name
     )
     new_name = name[: pos.start()] if pos is not None else name
     new_pos = name.rfind('grad/')
diff --git a/python/paddle/fluid/tests/unittests/test_backward.py b/python/paddle/fluid/tests/unittests/test_backward.py
index b2abf39fecac5..031664e4cb35d 100644
--- a/python/paddle/fluid/tests/unittests/test_backward.py
+++ b/python/paddle/fluid/tests/unittests/test_backward.py
@@ -19,6 +19,7 @@
 import paddle
 import paddle.nn.functional as F
 from paddle import fluid, static
+from paddle.fluid import backward
 
 
 class BackwardNet:
@@ -449,6 +450,19 @@ def test(self):
             print(out)
 
 
+class TestStripGradSuffix(unittest.TestCase):
+    def test_strip_grad_suffix(self):
+        cases = (
+            ('x@GRAD', 'x'),
+            ('x@GRAD@GRAD', 'x'),
+            ('x@GRAD@RENAME@1', 'x'),
+            ('x@GRAD_slice_0@GRAD', 'x@GRAD_slice_0'),
+            ('grad/grad/x@GRAD@RENAME@block0@1@GRAD', 'x'),
+        )
+        for input_, desired in cases:
+            self.assertEqual(backward._strip_grad_suffix_(input_), desired)
+
+
 if __name__ == '__main__':
     paddle.enable_static()
     unittest.main()

From 5b054d2f64b36d32b66401f0b3e18816cdc824e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?=
 <83450930+Liyulingyue@users.noreply.github.com>
Date: Tue, 16 May 2023 14:38:55 +0800
Subject: [PATCH 358/405] =?UTF-8?q?=E6=98=87=E8=85=BE=E5=92=8C=E5=AF=92?=
 =?UTF-8?q?=E6=AD=A6=E7=BA=AA=E7=9B=B8=E5=85=B3=E4=BB=A3=E7=A0=81=E9=80=80?=
 =?UTF-8?q?=E5=9C=BA=20npu=E7=9B=B8=E5=85=B3=E4=BB=A3=E7=A0=81=E9=80=80?=
 =?UTF-8?q?=E5=9C=BA3=20(#53699)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* rm npu

* rm use_npu

* rm npuid

* rm use_npu

* rm npuid

* delete npupinned

* roll back sth.

* roll back sth.

* delete npupinned

* roll back sth.

* roll back sth.

* rm npu

* rollback something

* rollback npu identity

* rollback npu identity
---
 paddle/fluid/framework/library_type.h                 | 2 --
 paddle/fluid/operators/beam_search_decode_op.h        | 8 +-------
 paddle/fluid/operators/memcpy_op.cc                   | 1 -
 paddle/fluid/operators/memcpy_op.h                    | 2 --
 paddle/fluid/operators/reader/buffered_reader.cc      | 1 -
 paddle/fluid/operators/reader/buffered_reader.h       | 1 -
 paddle/fluid/platform/CMakeLists.txt                  | 1 -
 paddle/fluid/platform/device_context.h                | 1 -
 paddle/phi/common/backend.h                           | 1 -
 paddle/phi/common/place.h                             | 1 -
 python/paddle/distributed/auto_parallel/cluster.py    | 5 +----
 python/paddle/distributed/auto_parallel/cluster_v2.py | 1 -
 12 files changed, 2 insertions(+), 23 deletions(-)

diff --git a/paddle/fluid/framework/library_type.h b/paddle/fluid/framework/library_type.h
index c5dddbc614dd3..d3d90f6b93fef 100644
--- a/paddle/fluid/framework/library_type.h
+++ b/paddle/fluid/framework/library_type.h
@@ -68,8 +68,6 @@ inline LibraryType StringToLibraryType(const char* ctype) {
     return LibraryType::kPlain;
   } else if (s == std::string("IPU")) {
     return LibraryType::kPlain;
-  } else if (s == std::string("NPU")) {
-    return LibraryType::kPlain;
   } else if (s == std::string("CUDA")) {
     return LibraryType::kPlain;
   } else {
diff --git a/paddle/fluid/operators/beam_search_decode_op.h b/paddle/fluid/operators/beam_search_decode_op.h
index 07a1c46ac0923..7347e228780b4 100644
--- a/paddle/fluid/operators/beam_search_decode_op.h
+++ b/paddle/fluid/operators/beam_search_decode_op.h
@@ -34,13 +34,10 @@ struct BeamSearchDecodeFunctor {
         id_tensor_(id_tensor),
         score_tensor_(score_tensor) {
     tensor_on_gpu_ = false;
-    tensor_on_npu_ = false;
     // First make a copy of GPU data on CPU
     if (platform::is_gpu_place(step_ids_origin_[0].place())) {
       if (platform::is_gpu_place(step_ids_origin_[0].place())) {
         tensor_on_gpu_ = true;
-      } else {
-        tensor_on_npu_ = true;
       }
       platform::DeviceContextPool& pool =
           platform::DeviceContextPool::Instance();
@@ -63,8 +60,6 @@ struct BeamSearchDecodeFunctor {
     if (platform::is_gpu_place(step_scores_origin_[0].place())) {
       if (platform::is_gpu_place(step_scores_origin_[0].place())) {
         tensor_on_gpu_ = true;
-      } else {
-        tensor_on_npu_ = true;
       }
       platform::DeviceContextPool& pool =
           platform::DeviceContextPool::Instance();
@@ -96,7 +91,7 @@ struct BeamSearchDecodeFunctor {
     } else {
       BeamSearchDecoder<T> beam_search_decoder(beam_size_, end_id_);
       // Check if the tensor is on GPU. If so, use the CPU copy instead
-      if (tensor_on_gpu_ || tensor_on_npu_) {
+      if (tensor_on_gpu_) {
         beam_search_decoder.Backtrace(
             step_ids_, step_scores_, id_tensor_, score_tensor_);
       } else {
@@ -107,7 +102,6 @@ struct BeamSearchDecodeFunctor {
   }
 
   bool tensor_on_gpu_;
-  bool tensor_on_npu_;
   size_t beam_size_;
   int end_id_;
   // TODO(Superjomn) Here might result serious performance issue in the
diff --git a/paddle/fluid/operators/memcpy_op.cc b/paddle/fluid/operators/memcpy_op.cc
index c1ee17789ac46..8568974580be5 100644
--- a/paddle/fluid/operators/memcpy_op.cc
+++ b/paddle/fluid/operators/memcpy_op.cc
@@ -111,7 +111,6 @@ class MemcpyOpProtoMaker : public framework::OpProtoAndCheckerMaker {
                  "1: dst is on CUDAPlace. "
                  "2: dst is on CUDAPinnedPlace. "
                  "3: dst is on XPUPlace. "
-                 "4: dst is on NPUPinnerPlace. "
                  "5: dst is on CustomDevicePlace");
     AddComment(R"DOC(
     Memcpy Operator.
diff --git a/paddle/fluid/operators/memcpy_op.h b/paddle/fluid/operators/memcpy_op.h
index 1cb077ee8a4b9..bfdd43eaaa519 100644
--- a/paddle/fluid/operators/memcpy_op.h
+++ b/paddle/fluid/operators/memcpy_op.h
@@ -39,8 +39,6 @@ class MemcpyFunctor {
     CUDA = 1,
     CUDA_PINNED = 2,
     XPU = 3,
-    NPU = 4,
-    NPU_PINNED = 5,
     CUSTOM_DEVICE = 6,
   };
 
diff --git a/paddle/fluid/operators/reader/buffered_reader.cc b/paddle/fluid/operators/reader/buffered_reader.cc
index b1e29ea69249d..b01b062279bfa 100644
--- a/paddle/fluid/operators/reader/buffered_reader.cc
+++ b/paddle/fluid/operators/reader/buffered_reader.cc
@@ -98,7 +98,6 @@ BufferedReader::BufferedReader(
 
   cpu_buffer_.resize(buffer_size);
   cuda_buffer_.resize(buffer_size);
-  npu_buffer_.resize(buffer_size);
   xpu_buffer_.resize(buffer_size);
   custom_device_buffer_.resize(buffer_size);
   ReadTillBufferFullAsync();
diff --git a/paddle/fluid/operators/reader/buffered_reader.h b/paddle/fluid/operators/reader/buffered_reader.h
index 67304c2244ecd..032a74b7e23f1 100644
--- a/paddle/fluid/operators/reader/buffered_reader.h
+++ b/paddle/fluid/operators/reader/buffered_reader.h
@@ -77,7 +77,6 @@ class BufferedReader : public framework::DecoratedReader {
   // buffers and prevent alloc every time.
   std::vector<TensorVec> cpu_buffer_;
   std::vector<TensorVec> cuda_buffer_;
-  std::vector<TensorVec> npu_buffer_;
   std::vector<TensorVec> xpu_buffer_;
   std::vector<TensorVec> custom_device_buffer_;
   size_t prev_pos_{-1UL};
diff --git a/paddle/fluid/platform/CMakeLists.txt b/paddle/fluid/platform/CMakeLists.txt
index baac9605504fc..ffebe1012b35f 100644
--- a/paddle/fluid/platform/CMakeLists.txt
+++ b/paddle/fluid/platform/CMakeLists.txt
@@ -123,7 +123,6 @@ cc_library(
        framework_proto
        ${IPU_CTX_DEPS}
        ${GPU_CTX_DEPS}
-       ${NPU_CTX_DEPS}
        ${MKLDNN_CTX_DEPS}
        ${dgc_deps}
        dlpack
diff --git a/paddle/fluid/platform/device_context.h b/paddle/fluid/platform/device_context.h
index f6cdb19f8d1f8..b07b3f29dafde 100644
--- a/paddle/fluid/platform/device_context.h
+++ b/paddle/fluid/platform/device_context.h
@@ -93,7 +93,6 @@ namespace platform {
 enum DeviceType {
   CPU = 0,
   CUDA = 1,
-  NPU = 2,
   XPU = 3,
   IPU = 4,
   CUSTOM_DEVICE = 6,
diff --git a/paddle/phi/common/backend.h b/paddle/phi/common/backend.h
index dd76eccca7af2..5540592d5013c 100644
--- a/paddle/phi/common/backend.h
+++ b/paddle/phi/common/backend.h
@@ -52,7 +52,6 @@ enum class Backend : uint8_t {
 
   // various acceleration devices' backends
   XPU,  // XPU currently does not exist at the same time as CUDA
-  NPU,  // NPU currently does not exist at the same time as CUDA
   IPU,
 
   // paddle kernel primitives backend
diff --git a/paddle/phi/common/place.h b/paddle/phi/common/place.h
index d8709c73d5ad6..09f7c53cde80d 100644
--- a/paddle/phi/common/place.h
+++ b/paddle/phi/common/place.h
@@ -31,7 +31,6 @@ enum class AllocationType : int8_t {
   GPU = 2,
   GPUPINNED = 3,
   XPU = 4,
-  NPU = 5,
   IPU = 7,
   CUSTOM = 9,
 };
diff --git a/python/paddle/distributed/auto_parallel/cluster.py b/python/paddle/distributed/auto_parallel/cluster.py
index 6a35894900afe..725037d20592c 100644
--- a/python/paddle/distributed/auto_parallel/cluster.py
+++ b/python/paddle/distributed/auto_parallel/cluster.py
@@ -450,9 +450,8 @@ def gen_default_config_cluster(
         """Generate cluster by default config."""
         gpu_models = ["V100", "A100", "H100", "A2", "A10", "A16", "A30", "A40"]
         xpu_models = ["XPU"]
-        npu_models = ["NPU"]
         dcu_models = ["DCU"]
-        all_gpu_models = gpu_models + xpu_models + npu_models + dcu_models
+        all_gpu_models = gpu_models + xpu_models + dcu_models
         self._num_devices_per_machine = device_count
 
         def _convert_to_type(gpu_model):
@@ -461,8 +460,6 @@ def _convert_to_type(gpu_model):
                 type = "GPU"
             elif gpu_model in xpu_models:
                 type = "XPU"
-            elif gpu_model in npu_models:
-                type = "NPU"
             elif gpu_model in dcu_models:
                 type = "DCU"
             else:
diff --git a/python/paddle/distributed/auto_parallel/cluster_v2.py b/python/paddle/distributed/auto_parallel/cluster_v2.py
index 7770aa3ab47a8..3f08d240efb05 100644
--- a/python/paddle/distributed/auto_parallel/cluster_v2.py
+++ b/python/paddle/distributed/auto_parallel/cluster_v2.py
@@ -25,7 +25,6 @@ class DeviceType(IntEnum):
     CPU = 1
     GPU = 2
     XPU = 3
-    NPU = 4
     DCU = 5
     NIC = 6
 

From 52889e385f2d5e8e831a5add98e26c36a4e3e164 Mon Sep 17 00:00:00 2001
From: huangjiyi <43315610+huangjiyi@users.noreply.github.com>
Date: Tue, 16 May 2023 15:19:56 +0800
Subject: [PATCH 359/405] move cudnn_lstm kernel to phi (#53730)

* update

* fix bug

* test

* test

* update

* update mutable_data

* fix bug

* update

* fix bug

* update output type reg

* update

* update
---
 paddle/fluid/operators/cudnn_lstm_op.cc       |  12 -
 paddle/fluid/operators/cudnn_lstm_op.cu.cc    | 743 ------------------
 paddle/phi/kernels/cpu/cudnn_lstm_kernel.cc   |  49 ++
 paddle/phi/kernels/cudnn_lstm_grad_kernel.h   |  47 ++
 paddle/phi/kernels/cudnn_lstm_kernel.h        |  45 ++
 .../kernels/gpu}/cudnn_lstm_cache.h           |  77 +-
 .../phi/kernels/gpu/cudnn_lstm_grad_kernel.cu | 312 ++++++++
 paddle/phi/kernels/gpu/cudnn_lstm_kernel.cu   | 374 +++++++++
 paddle/phi/kernels/gpu/cudnn_lstm_utils.h     |  76 ++
 .../kernels/gpu}/miopen_lstm_cache.h          |  70 +-
 paddle/phi/ops/compat/cudnn_lstm_sig.cc       |  59 ++
 11 files changed, 1035 insertions(+), 829 deletions(-)
 delete mode 100644 paddle/fluid/operators/cudnn_lstm_op.cu.cc
 create mode 100644 paddle/phi/kernels/cpu/cudnn_lstm_kernel.cc
 create mode 100644 paddle/phi/kernels/cudnn_lstm_grad_kernel.h
 create mode 100644 paddle/phi/kernels/cudnn_lstm_kernel.h
 rename paddle/{fluid/operators => phi/kernels/gpu}/cudnn_lstm_cache.h (72%)
 create mode 100644 paddle/phi/kernels/gpu/cudnn_lstm_grad_kernel.cu
 create mode 100644 paddle/phi/kernels/gpu/cudnn_lstm_kernel.cu
 create mode 100644 paddle/phi/kernels/gpu/cudnn_lstm_utils.h
 rename paddle/{fluid/operators => phi/kernels/gpu}/miopen_lstm_cache.h (71%)
 create mode 100644 paddle/phi/ops/compat/cudnn_lstm_sig.cc

diff --git a/paddle/fluid/operators/cudnn_lstm_op.cc b/paddle/fluid/operators/cudnn_lstm_op.cc
index c0386a36901ee..a3cd48cc50c0d 100644
--- a/paddle/fluid/operators/cudnn_lstm_op.cc
+++ b/paddle/fluid/operators/cudnn_lstm_op.cc
@@ -292,15 +292,6 @@ class CudnnLSTMGradOpMaker : public framework::SingleGradOpMaker<T> {
   }
 };
 
-template <typename T, typename DeviceContext>
-class NotImpleKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
-    PADDLE_THROW(platform::errors::Unimplemented(
-        "CPU is not support for this kernel now. Will be add in the future"));
-  }
-};
-
 }  // namespace operators
 }  // namespace paddle
 
@@ -312,9 +303,6 @@ REGISTER_OPERATOR(cudnn_lstm,
                   ops::CudnnLSTMGradOpMaker<paddle::imperative::OpBase>);
 REGISTER_OPERATOR(cudnn_lstm_grad, ops::CudnnLSTMGradOp);
 
-PD_REGISTER_STRUCT_KERNEL(
-    cudnn_lstm, CPU, ALL_LAYOUT, ops::NotImpleKernel, float) {}
-
 // TODO(Shixiaowei02) Add ModifyInput support
 REGISTER_OP_VERSION(cudnn_lstm)
     .AddCheckpoint(
diff --git a/paddle/fluid/operators/cudnn_lstm_op.cu.cc b/paddle/fluid/operators/cudnn_lstm_op.cu.cc
deleted file mode 100644
index dd9f8bc629742..0000000000000
--- a/paddle/fluid/operators/cudnn_lstm_op.cu.cc
+++ /dev/null
@@ -1,743 +0,0 @@
-/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/phi/core/generator.h"
-#include "paddle/phi/core/tensor_utils.h"
-#include "paddle/phi/kernels/funcs/math_function.h"
-#ifdef PADDLE_WITH_CUDA
-#include "paddle/fluid/operators/cudnn_lstm_cache.h"
-#endif
-#ifdef PADDLE_WITH_HIP
-#include "paddle/fluid/operators/miopen_lstm_cache.h"
-#endif
-
-namespace paddle {
-namespace operators {
-
-template <typename T, typename Type>
-bool is_continuous(const Type &weight_list) {
-  bool continuous = true;
-  for (size_t i = 0; i < weight_list.size() - 1; ++i) {
-    auto *in_data = weight_list[i]->template data<T>();
-    auto *in_after_data = weight_list[i + 1]->template data<T>();
-    auto in_size = weight_list[i]->numel();
-    bool temp = in_data + in_size == in_after_data;
-    continuous = continuous && temp;
-  }
-  return continuous;
-}
-
-int size_sum(const std::vector<const phi::DenseTensor *> &weight_list) {
-  int size = 0;
-  for (size_t i = 0; i < weight_list.size(); ++i) {
-    auto in_size = weight_list[i]->numel();
-    size += in_size;
-  }
-  return size;
-}
-
-template <typename T>
-void weight_to_tensor(const platform::Place &place,
-                      gpuStream_t stream,
-                      const std::vector<const phi::DenseTensor *> &weight_list,
-                      phi::DenseTensor *weight) {
-  auto weight_data = weight->data<T>();
-  int weight_offset = 0;
-  for (size_t i = 0; i < weight_list.size(); ++i) {
-    const T *in_data = weight_list[i]->data<T>();
-    auto in_size = weight_list[i]->numel();
-
-    memory::Copy(weight->place(),
-                 weight_data + weight_offset,
-                 weight_list[i]->place(),
-                 in_data,
-                 in_size * sizeof(T),
-                 stream);
-    weight_offset += in_size;
-  }
-}
-
-template <typename T>
-void weight_to_tensor_list(
-    const platform::Place &place,
-    gpuStream_t stream,
-    std::vector<phi::DenseTensor *> *weight_grad,
-    const std::vector<const phi::DenseTensor *> &weight_input,
-    const phi::DenseTensor *weight) {
-  int weight_offset = 0;
-  auto *weight_data = weight->data<T>();
-  for (size_t i = 0; i < weight_input.size(); ++i) {
-    auto in_size = weight_input[i]->numel();
-    T *weight_grad_data = (*weight_grad)[i]->mutable_data<T>(place);
-    const T *src = weight_data + weight_offset;
-
-    memory::Copy((*weight_grad)[i]->place(),
-                 weight_grad_data,
-                 weight->place(),
-                 src,
-                 in_size * sizeof(T),
-                 stream);
-    weight_offset += in_size;
-  }
-}
-
-template <typename T>
-#ifdef PADDLE_WITH_HIP
-void LSTMInferece(const bool &has_seq_length,
-                  const miopenHandle_t &handle,
-#else
-void LSTMInferece(const bool &has_seq_length,
-                  const cudnnHandle_t &handle,
-#endif
-                  const int &seq_length,
-                  ScopedRNNBase *rnn,
-                  const T *x_data,
-                  const T *init_h_data,
-                  const T *init_c_data,
-                  const T *w_data,
-                  T *out_data,
-                  T *last_h_data,
-                  T *last_c_data,
-                  phi::DenseTensor *workspace_data,
-                  const size_t &workspace_size) {
-  if (!has_seq_length) {
-// for inference
-// This interface is used when the input/output is unpadded.
-#ifdef PADDLE_WITH_HIP
-    PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::miopenRNNForwardInference(
-        handle,
-        rnn->rnn_desc(),
-        seq_length,
-        rnn->x_descs(),
-        x_data,
-        rnn->init_h_desc(),
-        init_h_data,
-        rnn->init_c_desc(),
-        init_c_data,
-        rnn->weight_desc(),
-        w_data,
-        rnn->y_descs(),
-        out_data,
-        rnn->last_h_desc(),
-        last_h_data,
-        rnn->last_c_desc(),
-        last_c_data,
-        workspace_data->data<uint8_t>(),
-        workspace_size));
-#else
-    PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnRNNForwardInference(
-        handle,
-        rnn->rnn_desc(),
-        seq_length,
-        rnn->x_descs(),
-        x_data,
-        rnn->init_h_desc(),
-        init_h_data,
-        rnn->init_c_desc(),
-        init_c_data,
-        rnn->weight_desc(),
-        w_data,
-        rnn->y_descs(),
-        out_data,
-        rnn->last_h_desc(),
-        last_h_data,
-        rnn->last_c_desc(),
-        last_c_data,
-        workspace_data->data<uint8_t>(),
-        workspace_size));
-#endif
-  } else {
-#if !defined(PADDLE_WITH_HIP) && CUDNN_VERSION >= 7201
-    // for inference
-    // This interface is used when the input/output is padded.
-    PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnRNNForwardInferenceEx(
-        handle,
-        rnn->rnn_desc(),
-        rnn->x_seq_desc(),
-        x_data,
-        rnn->init_h_desc(),
-        init_h_data,
-        rnn->init_c_desc(),
-        init_c_data,
-        rnn->weight_desc(),
-        w_data,
-        rnn->y_seq_desc(),
-        out_data,
-        rnn->last_h_desc(),
-        last_h_data,
-        rnn->last_c_desc(),
-        last_c_data,
-        nullptr,
-        nullptr,
-        nullptr,
-        nullptr,
-        nullptr,
-        nullptr,
-        nullptr,
-        nullptr,
-        workspace_data->data<uint8_t>(),
-        workspace_size));
-#else
-    // CUDNN VERSION has to >=7.2.1
-    PADDLE_THROW(platform::errors::Unavailable(
-        "The padded input is supported by "
-        "cudnnRNNForwardInferenceEx, but it only works when "
-        "the version of cudnn is larger than 7.2.1"));
-#endif
-  }
-}
-
-template <typename T, typename DeviceContext>
-class CudnnLSTMGPUKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext &ctx) const override {
-    const phi::DenseTensor *x = ctx.Input<phi::DenseTensor>("Input");
-    const phi::DenseTensor *init_h = ctx.Input<phi::DenseTensor>("InitH");
-    const phi::DenseTensor *init_c = ctx.Input<phi::DenseTensor>("InitC");
-
-    phi::DenseTensor *out = ctx.Output<phi::DenseTensor>("Out");
-    phi::DenseTensor *last_h = ctx.Output<phi::DenseTensor>("LastH");
-    phi::DenseTensor *last_c = ctx.Output<phi::DenseTensor>("LastC");
-    phi::DenseTensor *reserve = ctx.Output<phi::DenseTensor>("Reserve");
-    phi::DenseTensor *state_out = ctx.Output<phi::DenseTensor>("StateOut");
-
-    const T *x_data = x->data<T>();
-    const T *init_h_data = init_h->data<T>();
-    const T *init_c_data = init_c->data<T>();
-
-    T *out_data = out->mutable_data<T>(ctx.GetPlace());
-    T *last_h_data = last_h->mutable_data<T>(ctx.GetPlace());
-    T *last_c_data = last_c->mutable_data<T>(ctx.GetPlace());
-
-    float dropout_prob = ctx.Attr<float>("dropout_prob");
-    bool is_bidirec = ctx.Attr<bool>("is_bidirec");
-    int hidden_size = ctx.Attr<int>("hidden_size");
-    int num_layers = ctx.Attr<int>("num_layers");
-    bool is_test = ctx.Attr<bool>("is_test");
-    int seed = ctx.Attr<int>("seed");
-
-    if (!is_test) {
-      if (seed == 0) {
-        // If not specify seed, use global Generator to generate seed.
-        int device_id = ctx.GetPlace().GetDeviceId();
-        auto gen_cuda = phi::DefaultCUDAGenerator(device_id);
-        seed = static_cast<int>(gen_cuda->Random64());
-      }
-      // else use `ctx.Attr<int>("seed")` specified seed
-    }
-
-    bool has_seq_length = ctx.HasInput("SequenceLength");
-    std::vector<int> SequenceLength;
-    if (has_seq_length) {
-      auto *sequence_length = ctx.Input<phi::DenseTensor>("SequenceLength");
-      SequenceLength = phi::GetVectorFromTensor<int>(sequence_length);
-    }
-
-    auto &dev_ctx = ctx.template device_context<phi::GPUContext>();
-    auto handle = dev_ctx.cudnn_handle();
-
-    int seq_length = x->dims()[0];
-    int batch_size = x->dims()[1];
-    int input_size = x->dims()[2];
-    bool state_initialized = state_out->IsInitialized() ? true : false;
-
-    size_t workspace_size;
-    size_t reserve_size;
-    phi::DenseTensor weight_whole;
-    T *w_data = nullptr;
-    int weight_numel;
-    bool w_initialized = false;
-    auto place = ctx.GetPlace();
-    auto stream =
-        reinterpret_cast<const phi::GPUContext &>(ctx.device_context())
-            .stream();
-    if (is_test && ctx.HasInput("W")) {
-      auto *W = ctx.Input<phi::DenseTensor>("W");
-      w_initialized = W->IsInitialized() ? true : false;
-      weight_numel = W->numel();
-    }
-    if (!w_initialized) {
-      auto weight_list = ctx.MultiInput<phi::DenseTensor>("WeightList");
-      bool continuous =
-          is_continuous<T, std::vector<const phi::DenseTensor *>>(weight_list);
-      weight_numel = size_sum(weight_list);
-
-      if (!continuous) {
-        LOG_FIRST_N(WARNING, 2)
-            << "If the memory space of the Input WeightList is not continuous, "
-               "less efficient calculation will be called. Please call "
-               "flatten_parameters() to make the input memory continuous.";
-        weight_whole.mutable_data<T>({weight_numel}, place);
-        weight_to_tensor<T>(place, stream, weight_list, &weight_whole);
-        w_data = weight_whole.data<T>();
-        if (is_test) {  // maybe also reset small weights' ptr for training
-          int offset = 0;
-          for (size_t i = 0; i < weight_list.size(); ++i) {
-            size_t len = weight_list[i]->numel();
-            auto dim = weight_list[i]->dims();
-            const_cast<phi::DenseTensor *>(weight_list[i])
-                ->ShareDataWith(
-                    weight_whole.Slice(static_cast<int64_t>(offset),
-                                       static_cast<int64_t>(offset + len)))
-                .Resize(dim);
-            offset += len;
-          }
-        }
-      } else {
-        w_data = const_cast<T *>(weight_list[0]->data<T>());
-      }
-    } else {
-      auto *W = ctx.Input<phi::DenseTensor>("W");
-      w_data = const_cast<T *>(W->data<T>());
-    }
-
-    ScopedRNNBase rnn(seq_length,
-                      batch_size,
-                      input_size,
-                      hidden_size,
-                      num_layers,
-                      dropout_prob,
-                      seed,
-                      weight_numel,
-                      state_initialized,
-                      is_bidirec);
-    rnn.Create<T>(handle,
-                  ctx.GetPlace(),
-                  SequenceLength,
-                  &workspace_size,
-                  &reserve_size,
-                  state_out);
-
-    phi::DenseTensor workspace_data_;
-    workspace_data_.mutable_data<uint8_t>(
-        {static_cast<int64_t>(workspace_size)}, ctx.GetPlace());
-
-    auto *reserve_data = reserve->mutable_data<uint8_t>(
-        {static_cast<int64_t>(reserve_size)}, ctx.GetPlace());
-
-    if (is_test) {
-      LSTMInferece<T>(has_seq_length,
-                      handle,
-                      seq_length,
-                      &rnn,
-                      x_data,
-                      init_h_data,
-                      init_c_data,
-                      w_data,
-                      out_data,
-                      last_h_data,
-                      last_c_data,
-                      &workspace_data_,
-                      workspace_size);
-    } else {
-      if (!has_seq_length) {
-// for train
-// This interface is used when the input/output is unpadded.
-#ifdef PADDLE_WITH_HIP
-        PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::miopenRNNForwardTraining(
-            handle,
-            rnn.rnn_desc(),
-            seq_length,
-            rnn.x_descs(),
-            x_data,
-            rnn.init_h_desc(),
-            init_h_data,
-            rnn.init_c_desc(),
-            init_c_data,
-            rnn.weight_desc(),
-            w_data,
-            rnn.y_descs(),
-            out_data,
-            rnn.last_h_desc(),
-            last_h_data,
-            rnn.last_c_desc(),
-            last_c_data,
-            workspace_data_.data<uint8_t>(),
-            workspace_size,
-            reserve_data,
-            reserve_size));
-#else
-        PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnRNNForwardTraining(
-            handle,
-            rnn.rnn_desc(),
-            seq_length,
-            rnn.x_descs(),
-            x_data,
-            rnn.init_h_desc(),
-            init_h_data,
-            rnn.init_c_desc(),
-            init_c_data,
-            rnn.weight_desc(),
-            w_data,
-            rnn.y_descs(),
-            out_data,
-            rnn.last_h_desc(),
-            last_h_data,
-            rnn.last_c_desc(),
-            last_c_data,
-            workspace_data_.data<uint8_t>(),
-            workspace_size,
-            reserve_data,
-            reserve_size));
-#endif
-      } else {
-#if !defined(PADDLE_WITH_HIP) && CUDNN_VERSION >= 7201
-        // for train
-        // This interface is used when the input/output is padded.
-        PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnRNNForwardTrainingEx(
-            handle,
-            rnn.rnn_desc(),
-            rnn.x_seq_desc(),
-            x_data,
-            rnn.init_h_desc(),
-            init_h_data,
-            rnn.init_c_desc(),
-            init_c_data,
-            rnn.weight_desc(),
-            w_data,
-            rnn.y_seq_desc(),
-            out_data,
-            rnn.last_h_desc(),
-            last_h_data,
-            rnn.last_c_desc(),
-            last_c_data,
-            nullptr,
-            nullptr,
-            nullptr,
-            nullptr,
-            nullptr,
-            nullptr,
-            nullptr,
-            nullptr,
-            workspace_data_.data<uint8_t>(),
-            workspace_size,
-            reserve_data,
-            reserve_size));
-#else
-        PADDLE_THROW(platform::errors::Unavailable(
-            "The padded input is supported by "
-            "cudnnRNNForwardTrainingEx, but it only works when "
-            "the version of cudnn is larger than 7.2.1"));
-#endif
-      }
-    }
-  }
-};
-
-template <typename T, typename DeviceContext>
-class CudnnLSTMGPUGradKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext &ctx) const override {
-    auto *input = ctx.Input<phi::DenseTensor>("Input");
-    auto *init_h = ctx.Input<phi::DenseTensor>("InitH");
-    auto *init_c = ctx.Input<phi::DenseTensor>("InitC");
-    auto *reserve = ctx.Input<phi::DenseTensor>("Reserve");
-    auto *state_out = ctx.Input<phi::DenseTensor>("StateOut");
-    auto weight_list = ctx.MultiInput<phi::DenseTensor>("WeightList");
-
-    auto *out = ctx.Input<phi::DenseTensor>("Out");
-    auto *out_grad = ctx.Input<phi::DenseTensor>(framework::GradVarName("Out"));
-    auto *last_h_grad =
-        ctx.Input<phi::DenseTensor>(framework::GradVarName("LastH"));
-    auto *last_c_grad =
-        ctx.Input<phi::DenseTensor>(framework::GradVarName("LastC"));
-
-    auto *in_grad =
-        ctx.Output<phi::DenseTensor>(framework::GradVarName("Input"));
-    auto *init_h_grad =
-        ctx.Output<phi::DenseTensor>(framework::GradVarName("InitH"));
-    auto *init_c_grad =
-        ctx.Output<phi::DenseTensor>(framework::GradVarName("InitC"));
-    auto weight_grad_list =
-        ctx.MultiOutput<phi::DenseTensor>(framework::GradVarName("WeightList"));
-
-    auto &dev_ctx = ctx.template device_context<phi::GPUContext>();
-    auto handle = dev_ctx.cudnn_handle();
-
-    auto input_dims = input->dims();
-    auto init_h_dims = init_h->dims();
-    auto init_c_dims = init_c->dims();
-
-    auto *init_h_data = init_h->data<T>();
-    auto *init_c_data = init_c->data<T>();
-    auto *out_data = out->data<T>();
-    auto *out_grad_data = out_grad->data<T>();
-    auto *last_h_grad_data = last_h_grad->data<T>();
-    auto *last_c_grad_data = last_c_grad->data<T>();
-
-    auto place = ctx.GetPlace();
-    int weight_numel = size_sum(weight_list);
-    bool continuous =
-        is_continuous<T, std::vector<const phi::DenseTensor *>>(weight_list);
-
-    auto stream =
-        reinterpret_cast<const phi::GPUContext &>(ctx.device_context())
-            .stream();
-    phi::DenseTensor weight_whole;
-    T *weight_data = nullptr;
-
-    if (!continuous) {
-      weight_whole.mutable_data<T>({weight_numel}, place);
-      weight_to_tensor<T>(place, stream, weight_list, &weight_whole);
-      weight_data = weight_whole.data<T>();
-    } else {
-      weight_data = const_cast<T *>(weight_list[0]->data<T>());
-    }
-
-    phi::DenseTensor weight_grad;
-    phi::funcs::SetConstant<phi::GPUContext, T> zero;
-    weight_grad.mutable_data<T>({weight_numel}, ctx.GetPlace());
-    zero(dev_ctx, &weight_grad, static_cast<T>(0.0));
-    T *weight_grad_data = weight_grad.data<T>();
-
-    int offset = 0;
-    for (size_t i = 0; i < weight_grad_list.size(); ++i) {
-      size_t len = weight_grad_list[i]->numel();
-      auto dim = weight_grad_list[i]->dims();
-      weight_grad_list[i]
-          ->ShareDataWith(weight_grad.Slice(static_cast<int64_t>(offset),
-                                            static_cast<int64_t>(offset + len)))
-          .Resize(dim);
-      offset += len;
-    }
-
-    in_grad->mutable_data<T>(input_dims, ctx.GetPlace());
-    auto *in_grad_data = in_grad->data<T>();
-
-    if (init_h_grad) init_h_grad->mutable_data<T>(init_h_dims, ctx.GetPlace());
-    auto *init_h_grad_data = init_h_grad ? init_h_grad->data<T>() : nullptr;
-
-    if (init_c_grad) init_c_grad->mutable_data<T>(init_c_dims, ctx.GetPlace());
-    auto *init_c_grad_data = init_c_grad ? init_c_grad->data<T>() : nullptr;
-
-    float dropout_prob = ctx.Attr<float>("dropout_prob");
-    bool is_bidirec = ctx.Attr<bool>("is_bidirec");
-    int hidden_size = ctx.Attr<int>("hidden_size");
-    int num_layers = ctx.Attr<int>("num_layers");
-    int seed = ctx.Attr<int>("seed");
-
-    bool has_seq_length = ctx.HasInput("SequenceLength");
-    std::vector<int> SequenceLength;
-    if (has_seq_length) {
-      auto *sequence_length = ctx.Input<phi::DenseTensor>("SequenceLength");
-      SequenceLength = phi::GetVectorFromTensor<int>(sequence_length);
-    }
-
-    int seq_length = input_dims[0];
-    int batch_size = input->dims()[1];
-    int input_size = input->dims()[2];
-
-    size_t workspace_size;
-    size_t reserve_size;
-
-    ScopedRNNBase rnn(seq_length,
-                      batch_size,
-                      input_size,
-                      hidden_size,
-                      num_layers,
-                      dropout_prob,
-                      seed,
-                      weight_numel,
-                      true,
-                      is_bidirec);
-
-    rnn.Create<T>(handle,
-                  ctx.GetPlace(),
-                  SequenceLength,
-                  &workspace_size,
-                  &reserve_size,
-                  const_cast<phi::DenseTensor *>(state_out));
-
-    phi::DenseTensor workspace_data_;
-    workspace_data_.mutable_data<uint8_t>(
-        {static_cast<int64_t>(workspace_size)}, ctx.GetPlace());
-    const uint8_t *reserve_data = reserve->data<uint8_t>();
-
-    if (!has_seq_length) {
-// This interface is used when the input/output is unpadded.
-#ifdef PADDLE_WITH_HIP
-      PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::miopenRNNBackwardData(
-          handle,
-          rnn.rnn_desc(),
-          seq_length,
-          rnn.y_descs(),
-          out_data,
-          rnn.y_descs(),
-          out_grad_data,
-          rnn.last_h_desc(),
-          last_h_grad_data,
-          rnn.last_c_desc(),
-          last_c_grad_data,
-          rnn.weight_desc(),
-          weight_data,
-          rnn.init_h_desc(),
-          init_h_data,
-          rnn.init_c_desc(),
-          init_c_data,
-          rnn.x_descs(),
-          in_grad_data,
-          rnn.init_h_desc(),
-          init_h_grad_data,
-          rnn.init_c_desc(),
-          init_c_grad_data,
-          workspace_data_.data<uint8_t>(),
-          workspace_size,
-          const_cast<uint8_t *>(reserve_data),
-          reserve_size));
-
-      PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::miopenRNNBackwardWeights(
-          handle,
-          rnn.rnn_desc(),
-          seq_length,
-          rnn.x_descs(),
-          input->data<T>(),
-          rnn.init_h_desc(),
-          init_h->data<T>(),
-          rnn.y_descs(),
-          out->data<T>(),
-          rnn.weight_desc(),
-          weight_grad_data,
-          workspace_data_.data<uint8_t>(),
-          workspace_size,
-          const_cast<uint8_t *>(reserve_data),
-          reserve_size));
-#else
-      PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnRNNBackwardData(
-          handle,
-          rnn.rnn_desc(),
-          seq_length,
-          rnn.y_descs(),
-          out_data,
-          rnn.y_descs(),
-          out_grad_data,
-          rnn.last_h_desc(),
-          last_h_grad_data,
-          rnn.last_c_desc(),
-          last_c_grad_data,
-          rnn.weight_desc(),
-          weight_data,
-          rnn.init_h_desc(),
-          init_h_data,
-          rnn.init_c_desc(),
-          init_c_data,
-          rnn.x_descs(),
-          in_grad_data,
-          rnn.init_h_desc(),
-          init_h_grad_data,
-          rnn.init_c_desc(),
-          init_c_grad_data,
-          workspace_data_.data<uint8_t>(),
-          workspace_size,
-          const_cast<uint8_t *>(reserve_data),
-          reserve_size));
-
-      PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnRNNBackwardWeights(
-          handle,
-          rnn.rnn_desc(),
-          seq_length,
-          rnn.x_descs(),
-          input->data<T>(),
-          rnn.init_h_desc(),
-          init_h->data<T>(),
-          rnn.y_descs(),
-          out->data<T>(),
-          workspace_data_.data<uint8_t>(),
-          workspace_size,
-          rnn.weight_desc(),
-          weight_grad_data,
-          const_cast<uint8_t *>(reserve_data),
-          reserve_size));
-#endif
-    } else {
-#if !defined(PADDLE_WITH_HIP) && CUDNN_VERSION >= 7201
-      // for train
-      // This interface is used when the input/output is padded.
-      PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnRNNBackwardDataEx(
-          handle,
-          rnn.rnn_desc(),
-          rnn.y_seq_desc(),
-          out_data,
-          rnn.y_seq_desc(),
-          out_grad_data,
-          nullptr,
-          nullptr,
-          rnn.last_h_desc(),
-          last_h_grad_data,
-          rnn.last_c_desc(),
-          last_c_grad_data,
-          rnn.weight_desc(),
-          weight_data,
-          rnn.init_h_desc(),
-          init_h_data,
-          rnn.init_c_desc(),
-          init_c_data,
-          rnn.x_seq_desc(),
-          in_grad_data,
-          rnn.init_h_desc(),
-          init_h_grad_data,
-          rnn.init_c_desc(),
-          init_c_grad_data,
-          nullptr,
-          nullptr,
-          workspace_data_.data<uint8_t>(),
-          workspace_size,
-          const_cast<uint8_t *>(reserve_data),
-          reserve_size));
-
-      PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnRNNBackwardWeightsEx(
-          handle,
-          rnn.rnn_desc(),
-          rnn.x_seq_desc(),
-          input->data<T>(),
-          rnn.init_h_desc(),
-          init_h->data<T>(),
-          rnn.y_seq_desc(),
-          out->data<T>(),
-          workspace_data_.data<uint8_t>(),
-          workspace_size,
-          rnn.weight_desc(),
-          weight_grad_data,
-          const_cast<uint8_t *>(reserve_data),
-          reserve_size));
-#else
-      PADDLE_THROW(platform::errors::Unavailable(
-          "The padded input of rnn is supported by cudnnRNNBackwardDataEx, "
-          "cudnnRNNBackwardWeightsEx, but it only works when the version "
-          "of cudnn is larger than 7.2.1"));
-#endif
-    }
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-#ifdef PADDLE_WITH_HIP
-// MIOPEN do not support double
-PD_REGISTER_STRUCT_KERNEL(
-    cudnn_lstm, GPU, ALL_LAYOUT, ops::CudnnLSTMGPUKernel, float) {}
-PD_REGISTER_STRUCT_KERNEL(
-    cudnn_lstm_grad, GPU, ALL_LAYOUT, ops::CudnnLSTMGPUGradKernel, float) {}
-#else
-PD_REGISTER_STRUCT_KERNEL(
-    cudnn_lstm, GPU, ALL_LAYOUT, ops::CudnnLSTMGPUKernel, float, double) {}
-PD_REGISTER_STRUCT_KERNEL(cudnn_lstm_grad,
-                          GPU,
-                          ALL_LAYOUT,
-                          ops::CudnnLSTMGPUGradKernel,
-                          float,
-                          double) {}
-#endif
diff --git a/paddle/phi/kernels/cpu/cudnn_lstm_kernel.cc b/paddle/phi/kernels/cpu/cudnn_lstm_kernel.cc
new file mode 100644
index 0000000000000..cd709fe2bf465
--- /dev/null
+++ b/paddle/phi/kernels/cpu/cudnn_lstm_kernel.cc
@@ -0,0 +1,49 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/cudnn_lstm_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/enforce.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void CudnnLSTMKernel(
+    const Context& ctx,
+    const DenseTensor& x,
+    const DenseTensor& init_h,
+    const DenseTensor& init_c,
+    const paddle::optional<DenseTensor>& w,
+    const paddle::optional<std::vector<const DenseTensor*>>& weight_list,
+    const paddle::optional<DenseTensor>& sequence_length,
+    float dropout_prob,
+    bool is_bidirec,
+    int hidden_size,
+    int num_layers,
+    bool is_test,
+    int seed,
+    DenseTensor* out,
+    DenseTensor* last_h,
+    DenseTensor* last_c,
+    DenseTensor* reserve,
+    DenseTensor* state_out) {
+  PADDLE_THROW(phi::errors::Unimplemented(
+      "CPU is not support for cudnn_lstm now. Will be add in the future"));
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(cudnn_lstm, CPU, ALL_LAYOUT, phi::CudnnLSTMKernel, float) {}
diff --git a/paddle/phi/kernels/cudnn_lstm_grad_kernel.h b/paddle/phi/kernels/cudnn_lstm_grad_kernel.h
new file mode 100644
index 0000000000000..848d3376ee3bf
--- /dev/null
+++ b/paddle/phi/kernels/cudnn_lstm_grad_kernel.h
@@ -0,0 +1,47 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/utils/optional.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void CudnnLSTMGradKernel(
+    const Context& ctx,
+    const DenseTensor& x,
+    const DenseTensor& init_h,
+    const DenseTensor& init_c,
+    const paddle::optional<std::vector<const DenseTensor*>>& weight_list,
+    const paddle::optional<DenseTensor>& sequence_length,
+    const DenseTensor& out,
+    const DenseTensor& reserve,
+    const DenseTensor& state_out,
+    const DenseTensor& out_grad,
+    const DenseTensor& last_h_grad,
+    const DenseTensor& last_c_grad,
+    float dropout_prob,
+    bool is_bidirec,
+    int hidden_size,
+    int num_layers,
+    bool is_test,
+    int seed,
+    DenseTensor* x_grad,
+    DenseTensor* init_h_grad,
+    DenseTensor* init_c_grad,
+    std::vector<DenseTensor*> weight_list_grad);
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/cudnn_lstm_kernel.h b/paddle/phi/kernels/cudnn_lstm_kernel.h
new file mode 100644
index 0000000000000..441808235fd7e
--- /dev/null
+++ b/paddle/phi/kernels/cudnn_lstm_kernel.h
@@ -0,0 +1,45 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <vector>
+
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/utils/optional.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void CudnnLSTMKernel(
+    const Context& ctx,
+    const DenseTensor& x,
+    const DenseTensor& init_h,
+    const DenseTensor& init_c,
+    const paddle::optional<DenseTensor>& w,
+    const paddle::optional<std::vector<const DenseTensor*>>& weight_list,
+    const paddle::optional<DenseTensor>& sequence_length,
+    float dropout_prob,
+    bool is_bidirec,
+    int hidden_size,
+    int num_layers,
+    bool is_test,
+    int seed,
+    DenseTensor* out,
+    DenseTensor* last_h,
+    DenseTensor* last_c,
+    DenseTensor* reserve,
+    DenseTensor* state_out);
+
+}  // namespace phi
diff --git a/paddle/fluid/operators/cudnn_lstm_cache.h b/paddle/phi/kernels/gpu/cudnn_lstm_cache.h
similarity index 72%
rename from paddle/fluid/operators/cudnn_lstm_cache.h
rename to paddle/phi/kernels/gpu/cudnn_lstm_cache.h
index 32f1b46dbbd39..197049452f97f 100644
--- a/paddle/fluid/operators/cudnn_lstm_cache.h
+++ b/paddle/phi/kernels/gpu/cudnn_lstm_cache.h
@@ -16,12 +16,13 @@ limitations under the License. */
 
 #include <vector>
 
-#include "paddle/fluid/framework/tensor.h"
-#include "paddle/fluid/platform/device/gpu/gpu_dnn.h"
-#include "paddle/fluid/platform/dynload/cudnn.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/dynload/cudnn.h"
+#include "paddle/phi/backends/gpu/forwards.h"
+#include "paddle/phi/backends/gpu/gpu_dnn.h"
+#include "paddle/phi/core/dense_tensor.h"
 
-namespace paddle {
-namespace operators {
+namespace phi {
 
 class ScopedRNNBase {
  public:
@@ -48,13 +49,13 @@ class ScopedRNNBase {
 
   template <typename T>
   void Create(const cudnnHandle_t& handle,
-              const platform::Place& place,
+              const phi::Place& place,
               const std::vector<int>& sequence_length,
               size_t* workspace_size,
               size_t* reserve_size,
               phi::DenseTensor* dropout_state) {
     int numDirections = is_bidirec_ ? 2 : 1;
-    cudnnDataType_t cudnn_type = platform::CudnnDataType<T>::type;
+    cudnnDataType_t cudnn_type = phi::backends::gpu::CudnnDataType<T>::type;
 
     // ------------------- cudnn x, y descriptors ---------------------
     std::vector<int> dims_x = {batch_size_, input_size_, 1};
@@ -91,9 +92,11 @@ class ScopedRNNBase {
     size_t state_size;
     if (!initialized_) {
       PADDLE_ENFORCE_GPU_SUCCESS(
-          platform::dynload::cudnnDropoutGetStatesSize(handle, &state_size));
-      dropout_state->mutable_data<uint8_t>({static_cast<int64_t>(state_size)},
-                                           place);
+          phi::dynload::cudnnDropoutGetStatesSize(handle, &state_size));
+      phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+      auto* dev_ctx = reinterpret_cast<phi::GPUContext*>(pool.Get(place));
+      dropout_state->Resize({static_cast<int64_t>(state_size)});
+      dev_ctx->template Alloc<uint8_t>(dropout_state);
     }
     dropout_desc_.descriptor(handle,
                              place,
@@ -104,7 +107,7 @@ class ScopedRNNBase {
                              state_size);
 
     // ------------------- cudnn rnn descriptors ---------------------
-    PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnSetRNNDescriptor_v6(
+    PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnSetRNNDescriptor_v6(
         handle,
         rnn_desc_.desc(),
         hidden_size_,
@@ -118,38 +121,35 @@ class ScopedRNNBase {
 
 #if CUDNN_VERSION >= 7201
     if (!sequence_length.empty()) {
-      PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnSetRNNPaddingMode(
+      PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnSetRNNPaddingMode(
           rnn_desc_.desc(), CUDNN_RNN_PADDED_IO_ENABLED));
     }
 #endif
 
     // ------------------- cudnn weights_size ---------------------
     size_t weights_size_;
-    PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cudnnGetRNNParamsSize(
+    PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnGetRNNParamsSize(
         handle, rnn_desc_.desc(), x_descs_[0], &weights_size_, cudnn_type));
     PADDLE_ENFORCE_EQ(
         weights_size_,
         sizeof(T) * weight_numel_,
-        platform::errors::InvalidArgument(
+        phi::errors::InvalidArgument(
             "The cudnn lstm and setting weight size should be same."));
     // ------------------- cudnn weight descriptors ---------------------
-    platform::DataLayout layout = platform::DataLayout::kNCHW;
+    phi::backends::gpu::DataLayout layout =
+        phi::backends::gpu::DataLayout::kNCHW;
     int dim_tmp = weights_size_ / sizeof(T);
     std::vector<int> dim_w = {dim_tmp, 1, 1};
     weight_desc_.descriptor<T>(layout, dim_w);
     // ------------------- cudnn workspace, reserve size ---------------------
     PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnGetRNNWorkspaceSize(handle,
-                                                    rnn_desc_.desc(),
-                                                    seq_length_,
-                                                    x_descs_.data(),
-                                                    workspace_size));
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::cudnnGetRNNTrainingReserveSize(handle,
-                                                          rnn_desc_.desc(),
-                                                          seq_length_,
-                                                          x_descs_.data(),
-                                                          reserve_size));
+        phi::dynload::cudnnGetRNNWorkspaceSize(handle,
+                                               rnn_desc_.desc(),
+                                               seq_length_,
+                                               x_descs_.data(),
+                                               workspace_size));
+    PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnGetRNNTrainingReserveSize(
+        handle, rnn_desc_.desc(), seq_length_, x_descs_.data(), reserve_size));
   }
   cudnnTensorDescriptor_t* x_descs() { return x_descs_.data(); }
   cudnnTensorDescriptor_t* y_descs() { return y_descs_.data(); }
@@ -179,20 +179,19 @@ class ScopedRNNBase {
   std::vector<cudnnTensorDescriptor_t> x_descs_;
   std::vector<cudnnTensorDescriptor_t> y_descs_;
 
-  platform::ScopedTensorDescriptor x_desc_;
-  platform::ScopedTensorDescriptor y_desc_;
+  phi::backends::gpu::ScopedTensorDescriptor x_desc_;
+  phi::backends::gpu::ScopedTensorDescriptor y_desc_;
 #if CUDNN_VERSION >= 7201
-  platform::ScopedRNNTensorDescriptor x_seq_desc_;
-  platform::ScopedRNNTensorDescriptor y_seq_desc_;
+  phi::backends::gpu::ScopedRNNTensorDescriptor x_seq_desc_;
+  phi::backends::gpu::ScopedRNNTensorDescriptor y_seq_desc_;
 #endif
-  platform::ScopedTensorDescriptor init_h_desc_;
-  platform::ScopedTensorDescriptor init_c_desc_;
-  platform::ScopedTensorDescriptor last_h_desc_;
-  platform::ScopedTensorDescriptor last_c_desc_;
-  platform::ScopedDropoutDescriptor dropout_desc_;
-  platform::ScopedFilterDescriptor weight_desc_;
-  platform::ScopedRNNDescriptor rnn_desc_;
+  phi::backends::gpu::ScopedTensorDescriptor init_h_desc_;
+  phi::backends::gpu::ScopedTensorDescriptor init_c_desc_;
+  phi::backends::gpu::ScopedTensorDescriptor last_h_desc_;
+  phi::backends::gpu::ScopedTensorDescriptor last_c_desc_;
+  phi::backends::gpu::ScopedDropoutDescriptor dropout_desc_;
+  phi::backends::gpu::ScopedFilterDescriptor weight_desc_;
+  phi::backends::gpu::ScopedRNNDescriptor rnn_desc_;
 };
 
-}  // namespace operators
-}  // namespace paddle
+}  // namespace phi
diff --git a/paddle/phi/kernels/gpu/cudnn_lstm_grad_kernel.cu b/paddle/phi/kernels/gpu/cudnn_lstm_grad_kernel.cu
new file mode 100644
index 0000000000000..661a1dd90e7e9
--- /dev/null
+++ b/paddle/phi/kernels/gpu/cudnn_lstm_grad_kernel.cu
@@ -0,0 +1,312 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/cudnn_lstm_grad_kernel.h"
+
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/gpu/cudnn_lstm_utils.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void CudnnLSTMGradKernel(
+    const Context &ctx,
+    const DenseTensor &x,
+    const DenseTensor &init_h,
+    const DenseTensor &init_c,
+    const paddle::optional<std::vector<const DenseTensor *>> &weight_list,
+    const paddle::optional<DenseTensor> &sequence_length,
+    const DenseTensor &out,
+    const DenseTensor &reserve,
+    const DenseTensor &state_out,
+    const DenseTensor &out_grad,
+    const DenseTensor &last_h_grad,
+    const DenseTensor &last_c_grad,
+    float dropout_prob,
+    bool is_bidirec,
+    int hidden_size,
+    int num_layers,
+    bool is_test,
+    int seed,
+    DenseTensor *x_grad,
+    DenseTensor *init_h_grad,
+    DenseTensor *init_c_grad,
+    std::vector<DenseTensor *> weight_grad_list) {
+  auto input_dims = x.dims();
+  auto init_h_dims = init_h.dims();
+  auto init_c_dims = init_c.dims();
+
+  auto *init_h_data = init_h.data<T>();
+  auto *init_c_data = init_c.data<T>();
+  auto *out_data = out.data<T>();
+  auto *out_grad_data = out_grad.data<T>();
+  auto *last_h_grad_data = last_h_grad.data<T>();
+  auto *last_c_grad_data = last_c_grad.data<T>();
+
+  auto running_weight_list = *weight_list.get_ptr();
+  int weight_numel = size_sum(running_weight_list);
+  bool continuous = is_continuous<T, std::vector<const phi::DenseTensor *>>(
+      running_weight_list);
+
+  auto handle = ctx.cudnn_handle();
+  auto place = ctx.GetPlace();
+  auto stream = ctx.stream();
+  phi::DenseTensor weight_whole;
+  T *weight_data = nullptr;
+
+  if (!continuous) {
+    weight_whole.Resize({weight_numel});
+    ctx.template Alloc<T>(&weight_whole);
+    weight_to_tensor<T>(place, stream, running_weight_list, &weight_whole);
+    weight_data = weight_whole.data<T>();
+  } else {
+    weight_data = const_cast<T *>(running_weight_list[0]->data<T>());
+  }
+
+  phi::DenseTensor weight_grad;
+  phi::funcs::SetConstant<phi::GPUContext, T> zero;
+  weight_grad.Resize({weight_numel});
+  ctx.template Alloc<T>(&weight_grad);
+  zero(ctx, &weight_grad, static_cast<T>(0.0));
+  T *weight_grad_data = weight_grad.data<T>();
+
+  int offset = 0;
+  for (size_t i = 0; i < weight_grad_list.size(); ++i) {
+    size_t len = weight_grad_list[i]->numel();
+    auto dim = weight_grad_list[i]->dims();
+    weight_grad_list[i]
+        ->ShareDataWith(weight_grad.Slice(static_cast<int64_t>(offset),
+                                          static_cast<int64_t>(offset + len)))
+        .Resize(dim);
+    offset += len;
+  }
+
+  x_grad->Resize(input_dims);
+  ctx.template Alloc<T>(x_grad);
+  auto *in_grad_data = x_grad->data<T>();
+
+  if (init_h_grad) {
+    init_h_grad->Resize(init_h_dims);
+    ctx.template Alloc<T>(init_h_grad);
+  }
+  auto *init_h_grad_data = init_h_grad ? init_h_grad->data<T>() : nullptr;
+
+  if (init_c_grad) {
+    init_c_grad->Resize(init_c_dims);
+    ctx.template Alloc<T>(init_c_grad);
+  }
+  auto *init_c_grad_data = init_c_grad ? init_c_grad->data<T>() : nullptr;
+
+  auto running_seq_length = sequence_length.get_ptr();
+  bool has_seq_length = running_seq_length != nullptr;
+  std::vector<int> SequenceLength;
+  if (has_seq_length) {
+    SequenceLength = phi::GetVectorFromTensor<int>(running_seq_length);
+  }
+
+  int seq_length = input_dims[0];
+  int batch_size = x.dims()[1];
+  int input_size = x.dims()[2];
+
+  size_t workspace_size;
+  size_t reserve_size;
+
+  ScopedRNNBase rnn(seq_length,
+                    batch_size,
+                    input_size,
+                    hidden_size,
+                    num_layers,
+                    dropout_prob,
+                    seed,
+                    weight_numel,
+                    true,
+                    is_bidirec);
+
+  rnn.Create<T>(handle,
+                ctx.GetPlace(),
+                SequenceLength,
+                &workspace_size,
+                &reserve_size,
+                const_cast<phi::DenseTensor *>(&state_out));
+
+  phi::DenseTensor workspace_data_;
+  workspace_data_.Resize({static_cast<int64_t>(workspace_size)});
+  ctx.template Alloc<uint8_t>(&workspace_data_);
+  const uint8_t *reserve_data = reserve.data<uint8_t>();
+
+  if (!has_seq_length) {
+// This interface is used when the input/output is unpadded.
+#ifdef PADDLE_WITH_HIP
+    PADDLE_ENFORCE_GPU_SUCCESS(
+        phi::dynload::miopenRNNBackwardData(handle,
+                                            rnn.rnn_desc(),
+                                            seq_length,
+                                            rnn.y_descs(),
+                                            out_data,
+                                            rnn.y_descs(),
+                                            out_grad_data,
+                                            rnn.last_h_desc(),
+                                            last_h_grad_data,
+                                            rnn.last_c_desc(),
+                                            last_c_grad_data,
+                                            rnn.weight_desc(),
+                                            weight_data,
+                                            rnn.init_h_desc(),
+                                            init_h_data,
+                                            rnn.init_c_desc(),
+                                            init_c_data,
+                                            rnn.x_descs(),
+                                            in_grad_data,
+                                            rnn.init_h_desc(),
+                                            init_h_grad_data,
+                                            rnn.init_c_desc(),
+                                            init_c_grad_data,
+                                            workspace_data_.data<uint8_t>(),
+                                            workspace_size,
+                                            const_cast<uint8_t *>(reserve_data),
+                                            reserve_size));
+
+    PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::miopenRNNBackwardWeights(
+        handle,
+        rnn.rnn_desc(),
+        seq_length,
+        rnn.x_descs(),
+        x.data<T>(),
+        rnn.init_h_desc(),
+        init_h.data<T>(),
+        rnn.y_descs(),
+        out.data<T>(),
+        rnn.weight_desc(),
+        weight_grad_data,
+        workspace_data_.data<uint8_t>(),
+        workspace_size,
+        const_cast<uint8_t *>(reserve_data),
+        reserve_size));
+#else
+    PADDLE_ENFORCE_GPU_SUCCESS(
+        phi::dynload::cudnnRNNBackwardData(handle,
+                                           rnn.rnn_desc(),
+                                           seq_length,
+                                           rnn.y_descs(),
+                                           out_data,
+                                           rnn.y_descs(),
+                                           out_grad_data,
+                                           rnn.last_h_desc(),
+                                           last_h_grad_data,
+                                           rnn.last_c_desc(),
+                                           last_c_grad_data,
+                                           rnn.weight_desc(),
+                                           weight_data,
+                                           rnn.init_h_desc(),
+                                           init_h_data,
+                                           rnn.init_c_desc(),
+                                           init_c_data,
+                                           rnn.x_descs(),
+                                           in_grad_data,
+                                           rnn.init_h_desc(),
+                                           init_h_grad_data,
+                                           rnn.init_c_desc(),
+                                           init_c_grad_data,
+                                           workspace_data_.data<uint8_t>(),
+                                           workspace_size,
+                                           const_cast<uint8_t *>(reserve_data),
+                                           reserve_size));
+
+    PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnRNNBackwardWeights(
+        handle,
+        rnn.rnn_desc(),
+        seq_length,
+        rnn.x_descs(),
+        x.data<T>(),
+        rnn.init_h_desc(),
+        init_h.data<T>(),
+        rnn.y_descs(),
+        out.data<T>(),
+        workspace_data_.data<uint8_t>(),
+        workspace_size,
+        rnn.weight_desc(),
+        weight_grad_data,
+        const_cast<uint8_t *>(reserve_data),
+        reserve_size));
+#endif
+  } else {
+#if !defined(PADDLE_WITH_HIP) && CUDNN_VERSION >= 7201
+    // for train
+    // This interface is used when the input/output is padded.
+    PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnRNNBackwardDataEx(
+        handle,
+        rnn.rnn_desc(),
+        rnn.y_seq_desc(),
+        out_data,
+        rnn.y_seq_desc(),
+        out_grad_data,
+        nullptr,
+        nullptr,
+        rnn.last_h_desc(),
+        last_h_grad_data,
+        rnn.last_c_desc(),
+        last_c_grad_data,
+        rnn.weight_desc(),
+        weight_data,
+        rnn.init_h_desc(),
+        init_h_data,
+        rnn.init_c_desc(),
+        init_c_data,
+        rnn.x_seq_desc(),
+        in_grad_data,
+        rnn.init_h_desc(),
+        init_h_grad_data,
+        rnn.init_c_desc(),
+        init_c_grad_data,
+        nullptr,
+        nullptr,
+        workspace_data_.data<uint8_t>(),
+        workspace_size,
+        const_cast<uint8_t *>(reserve_data),
+        reserve_size));
+
+    PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnRNNBackwardWeightsEx(
+        handle,
+        rnn.rnn_desc(),
+        rnn.x_seq_desc(),
+        x.data<T>(),
+        rnn.init_h_desc(),
+        init_h.data<T>(),
+        rnn.y_seq_desc(),
+        out.data<T>(),
+        workspace_data_.data<uint8_t>(),
+        workspace_size,
+        rnn.weight_desc(),
+        weight_grad_data,
+        const_cast<uint8_t *>(reserve_data),
+        reserve_size));
+#else
+    PADDLE_THROW(phi::errors::Unavailable(
+        "The padded input of rnn is supported by cudnnRNNBackwardDataEx, "
+        "cudnnRNNBackwardWeightsEx, but it only works when the version "
+        "of cudnn is larger than 7.2.1"));
+#endif
+  }
+}
+
+}  // namespace phi
+
+#ifdef PADDLE_WITH_HIP
+PD_REGISTER_KERNEL(
+    cudnn_lstm_grad, GPU, ALL_LAYOUT, phi::CudnnLSTMGradKernel, float) {}
+#else
+PD_REGISTER_KERNEL(
+    cudnn_lstm_grad, GPU, ALL_LAYOUT, phi::CudnnLSTMGradKernel, float, double) {
+}
+#endif
diff --git a/paddle/phi/kernels/gpu/cudnn_lstm_kernel.cu b/paddle/phi/kernels/gpu/cudnn_lstm_kernel.cu
new file mode 100644
index 0000000000000..e3f2b780f3ffc
--- /dev/null
+++ b/paddle/phi/kernels/gpu/cudnn_lstm_kernel.cu
@@ -0,0 +1,374 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/cudnn_lstm_kernel.h"
+
+#include "glog/logging.h"
+
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/gpu/cudnn_lstm_utils.h"
+
+namespace phi {
+
+template <typename T>
+#ifdef PADDLE_WITH_HIP
+void LSTMInferece(const bool &has_seq_length,
+                  const miopenHandle_t &handle,
+#else
+void LSTMInferece(const bool &has_seq_length,
+                  const cudnnHandle_t &handle,
+#endif
+                  const int &seq_length,
+                  ScopedRNNBase *rnn,
+                  const T *x_data,
+                  const T *init_h_data,
+                  const T *init_c_data,
+                  const T *w_data,
+                  T *out_data,
+                  T *last_h_data,
+                  T *last_c_data,
+                  phi::DenseTensor *workspace_data,
+                  const size_t &workspace_size) {
+  if (!has_seq_length) {
+// for inference
+// This interface is used when the input/output is unpadded.
+#ifdef PADDLE_WITH_HIP
+    PADDLE_ENFORCE_GPU_SUCCESS(
+        phi::dynload::miopenRNNForwardInference(handle,
+                                                rnn->rnn_desc(),
+                                                seq_length,
+                                                rnn->x_descs(),
+                                                x_data,
+                                                rnn->init_h_desc(),
+                                                init_h_data,
+                                                rnn->init_c_desc(),
+                                                init_c_data,
+                                                rnn->weight_desc(),
+                                                w_data,
+                                                rnn->y_descs(),
+                                                out_data,
+                                                rnn->last_h_desc(),
+                                                last_h_data,
+                                                rnn->last_c_desc(),
+                                                last_c_data,
+                                                workspace_data->data<uint8_t>(),
+                                                workspace_size));
+#else
+    PADDLE_ENFORCE_GPU_SUCCESS(
+        phi::dynload::cudnnRNNForwardInference(handle,
+                                               rnn->rnn_desc(),
+                                               seq_length,
+                                               rnn->x_descs(),
+                                               x_data,
+                                               rnn->init_h_desc(),
+                                               init_h_data,
+                                               rnn->init_c_desc(),
+                                               init_c_data,
+                                               rnn->weight_desc(),
+                                               w_data,
+                                               rnn->y_descs(),
+                                               out_data,
+                                               rnn->last_h_desc(),
+                                               last_h_data,
+                                               rnn->last_c_desc(),
+                                               last_c_data,
+                                               workspace_data->data<uint8_t>(),
+                                               workspace_size));
+#endif
+  } else {
+#if !defined(PADDLE_WITH_HIP) && CUDNN_VERSION >= 7201
+    // for inference
+    // This interface is used when the input/output is padded.
+    PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnRNNForwardInferenceEx(
+        handle,
+        rnn->rnn_desc(),
+        rnn->x_seq_desc(),
+        x_data,
+        rnn->init_h_desc(),
+        init_h_data,
+        rnn->init_c_desc(),
+        init_c_data,
+        rnn->weight_desc(),
+        w_data,
+        rnn->y_seq_desc(),
+        out_data,
+        rnn->last_h_desc(),
+        last_h_data,
+        rnn->last_c_desc(),
+        last_c_data,
+        nullptr,
+        nullptr,
+        nullptr,
+        nullptr,
+        nullptr,
+        nullptr,
+        nullptr,
+        nullptr,
+        workspace_data->data<uint8_t>(),
+        workspace_size));
+#else
+    // CUDNN VERSION has to >=7.2.1
+    PADDLE_THROW(phi::errors::Unavailable(
+        "The padded input is supported by "
+        "cudnnRNNForwardInferenceEx, but it only works when "
+        "the version of cudnn is larger than 7.2.1"));
+#endif
+  }
+}
+
+template <typename T, typename Context>
+void CudnnLSTMKernel(
+    const Context &ctx,
+    const DenseTensor &x,
+    const DenseTensor &init_h,
+    const DenseTensor &init_c,
+    const paddle::optional<DenseTensor> &w,
+    const paddle::optional<std::vector<const DenseTensor *>> &weight_list,
+    const paddle::optional<DenseTensor> &sequence_length,
+    float dropout_prob,
+    bool is_bidirec,
+    int hidden_size,
+    int num_layers,
+    bool is_test,
+    int seed,
+    DenseTensor *out,
+    DenseTensor *last_h,
+    DenseTensor *last_c,
+    DenseTensor *reserve,
+    DenseTensor *state_out) {
+  const T *x_data = x.data<T>();
+  const T *init_h_data = init_h.data<T>();
+  const T *init_c_data = init_c.data<T>();
+
+  T *out_data = ctx.template Alloc<T>(out);
+  T *last_h_data = ctx.template Alloc<T>(last_h);
+  T *last_c_data = ctx.template Alloc<T>(last_c);
+
+  if (!is_test) {
+    if (seed == 0) {
+      // If not specify seed, use global Generator to generate seed.
+      int device_id = ctx.GetPlace().GetDeviceId();
+      auto gen_cuda = phi::DefaultCUDAGenerator(device_id);
+      seed = static_cast<int>(gen_cuda->Random64());
+    }
+  }
+
+  auto *running_sequence_length = sequence_length.get_ptr();
+  bool has_seq_length = running_sequence_length != nullptr;
+  std::vector<int> SequenceLength;
+  if (has_seq_length) {
+    SequenceLength = phi::GetVectorFromTensor<int>(running_sequence_length);
+  }
+
+  auto handle = ctx.cudnn_handle();
+
+  int seq_length = x.dims()[0];
+  int batch_size = x.dims()[1];
+  int input_size = x.dims()[2];
+  bool state_initialized = state_out->IsInitialized() ? true : false;
+
+  size_t workspace_size;
+  size_t reserve_size;
+  phi::DenseTensor weight_whole;
+  T *w_data = nullptr;
+  int weight_numel;
+  bool w_initialized = false;
+  auto place = ctx.GetPlace();
+  auto stream = ctx.stream();
+  auto *running_w = w.get_ptr();
+  if (is_test && running_w != nullptr) {
+    w_initialized = running_w->IsInitialized() ? true : false;
+    weight_numel = running_w->numel();
+  }
+  if (!w_initialized) {
+    auto running_weight_list = *weight_list.get_ptr();
+    bool continuous = is_continuous<T, std::vector<const phi::DenseTensor *>>(
+        running_weight_list);
+    weight_numel = size_sum(running_weight_list);
+
+    if (!continuous) {
+      LOG_FIRST_N(WARNING, 2)
+          << "If the memory space of the Input WeightList is not continuous, "
+             "less efficient calculation will be called. Please call "
+             "flatten_parameters() to make the input memory continuous.";
+      weight_whole.Resize({weight_numel});
+      ctx.template Alloc<T>(&weight_whole);
+      weight_to_tensor<T>(place, stream, running_weight_list, &weight_whole);
+      w_data = weight_whole.data<T>();
+      if (is_test) {  // maybe also reset small weights' ptr for training
+        int offset = 0;
+        for (size_t i = 0; i < running_weight_list.size(); ++i) {
+          size_t len = running_weight_list[i]->numel();
+          auto dim = running_weight_list[i]->dims();
+          const_cast<phi::DenseTensor *>(running_weight_list[i])
+              ->ShareDataWith(
+                  weight_whole.Slice(static_cast<int64_t>(offset),
+                                     static_cast<int64_t>(offset + len)))
+              .Resize(dim);
+          offset += len;
+        }
+      }
+    } else {
+      w_data = const_cast<T *>(running_weight_list[0]->data<T>());
+    }
+  } else {
+    w_data = const_cast<T *>(running_w->data<T>());
+  }
+
+  ScopedRNNBase rnn(seq_length,
+                    batch_size,
+                    input_size,
+                    hidden_size,
+                    num_layers,
+                    dropout_prob,
+                    seed,
+                    weight_numel,
+                    state_initialized,
+                    is_bidirec);
+  rnn.Create<T>(handle,
+                ctx.GetPlace(),
+                SequenceLength,
+                &workspace_size,
+                &reserve_size,
+                state_out);
+
+  phi::DenseTensor workspace_data_;
+  workspace_data_.Resize({static_cast<int64_t>(workspace_size)});
+  ctx.template Alloc<uint8_t>(&workspace_data_);
+
+  reserve->Resize({static_cast<int64_t>(reserve_size)});
+  auto *reserve_data = ctx.template Alloc<uint8_t>(reserve);
+
+  if (is_test) {
+    LSTMInferece<T>(has_seq_length,
+                    handle,
+                    seq_length,
+                    &rnn,
+                    x_data,
+                    init_h_data,
+                    init_c_data,
+                    w_data,
+                    out_data,
+                    last_h_data,
+                    last_c_data,
+                    &workspace_data_,
+                    workspace_size);
+  } else {
+    if (!has_seq_length) {
+// for train
+// This interface is used when the input/output is unpadded.
+#ifdef PADDLE_WITH_HIP
+      PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::miopenRNNForwardTraining(
+          handle,
+          rnn.rnn_desc(),
+          seq_length,
+          rnn.x_descs(),
+          x_data,
+          rnn.init_h_desc(),
+          init_h_data,
+          rnn.init_c_desc(),
+          init_c_data,
+          rnn.weight_desc(),
+          w_data,
+          rnn.y_descs(),
+          out_data,
+          rnn.last_h_desc(),
+          last_h_data,
+          rnn.last_c_desc(),
+          last_c_data,
+          workspace_data_.data<uint8_t>(),
+          workspace_size,
+          reserve_data,
+          reserve_size));
+#else
+      PADDLE_ENFORCE_GPU_SUCCESS(
+          phi::dynload::cudnnRNNForwardTraining(handle,
+                                                rnn.rnn_desc(),
+                                                seq_length,
+                                                rnn.x_descs(),
+                                                x_data,
+                                                rnn.init_h_desc(),
+                                                init_h_data,
+                                                rnn.init_c_desc(),
+                                                init_c_data,
+                                                rnn.weight_desc(),
+                                                w_data,
+                                                rnn.y_descs(),
+                                                out_data,
+                                                rnn.last_h_desc(),
+                                                last_h_data,
+                                                rnn.last_c_desc(),
+                                                last_c_data,
+                                                workspace_data_.data<uint8_t>(),
+                                                workspace_size,
+                                                reserve_data,
+                                                reserve_size));
+#endif
+    } else {
+#if !defined(PADDLE_WITH_HIP) && CUDNN_VERSION >= 7201
+      // for train
+      // This interface is used when the input/output is padded.
+      PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::cudnnRNNForwardTrainingEx(
+          handle,
+          rnn.rnn_desc(),
+          rnn.x_seq_desc(),
+          x_data,
+          rnn.init_h_desc(),
+          init_h_data,
+          rnn.init_c_desc(),
+          init_c_data,
+          rnn.weight_desc(),
+          w_data,
+          rnn.y_seq_desc(),
+          out_data,
+          rnn.last_h_desc(),
+          last_h_data,
+          rnn.last_c_desc(),
+          last_c_data,
+          nullptr,
+          nullptr,
+          nullptr,
+          nullptr,
+          nullptr,
+          nullptr,
+          nullptr,
+          nullptr,
+          workspace_data_.data<uint8_t>(),
+          workspace_size,
+          reserve_data,
+          reserve_size));
+#else
+      PADDLE_THROW(phi::errors::Unavailable(
+          "The padded input is supported by "
+          "cudnnRNNForwardTrainingEx, but it only works when "
+          "the version of cudnn is larger than 7.2.1"));
+#endif
+    }
+  }
+}
+
+}  // namespace phi
+
+#ifdef PADDLE_WITH_HIP
+PD_REGISTER_KERNEL(cudnn_lstm, GPU, ALL_LAYOUT, phi::CudnnLSTMKernel, float) {
+  kernel->OutputAt(3).SetDataType(phi::DataType::UINT8);
+  kernel->OutputAt(4).SetDataType(phi::DataType::UINT8);
+}
+#else
+PD_REGISTER_KERNEL(
+    cudnn_lstm, GPU, ALL_LAYOUT, phi::CudnnLSTMKernel, float, double) {
+  kernel->OutputAt(3).SetDataType(phi::DataType::UINT8);
+  kernel->OutputAt(4).SetDataType(phi::DataType::UINT8);
+}
+#endif
diff --git a/paddle/phi/kernels/gpu/cudnn_lstm_utils.h b/paddle/phi/kernels/gpu/cudnn_lstm_utils.h
new file mode 100644
index 0000000000000..e5fc51849454d
--- /dev/null
+++ b/paddle/phi/kernels/gpu/cudnn_lstm_utils.h
@@ -0,0 +1,76 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <vector>
+
+#include "paddle/phi/common/memory_utils.h"
+#include "paddle/phi/core/generator.h"
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/kernels/funcs/math_function.h"
+#ifdef PADDLE_WITH_CUDA
+#include "paddle/phi/kernels/gpu/cudnn_lstm_cache.h"
+#endif
+#ifdef PADDLE_WITH_HIP
+#include "paddle/phi/kernels/gpu/miopen_lstm_cache.h"
+#endif
+
+namespace phi {
+
+template <typename T, typename Type>
+inline bool is_continuous(const Type &weight_list) {
+  bool continuous = true;
+  for (size_t i = 0; i < weight_list.size() - 1; ++i) {
+    auto *in_data = weight_list[i]->template data<T>();
+    auto *in_after_data = weight_list[i + 1]->template data<T>();
+    auto in_size = weight_list[i]->numel();
+    bool temp = in_data + in_size == in_after_data;
+    continuous = continuous && temp;
+  }
+  return continuous;
+}
+
+inline int size_sum(const std::vector<const phi::DenseTensor *> &weight_list) {
+  int size = 0;
+  for (size_t i = 0; i < weight_list.size(); ++i) {
+    auto in_size = weight_list[i]->numel();
+    size += in_size;
+  }
+  return size;
+}
+
+template <typename T>
+inline void weight_to_tensor(
+    const phi::Place &place,
+    gpuStream_t stream,
+    const std::vector<const phi::DenseTensor *> &weight_list,
+    phi::DenseTensor *weight) {
+  auto weight_data = weight->data<T>();
+  int weight_offset = 0;
+  for (size_t i = 0; i < weight_list.size(); ++i) {
+    const T *in_data = weight_list[i]->data<T>();
+    auto in_size = weight_list[i]->numel();
+
+    memory_utils::Copy(weight->place(),
+                       weight_data + weight_offset,
+                       weight_list[i]->place(),
+                       in_data,
+                       in_size * sizeof(T),
+                       stream);
+    weight_offset += in_size;
+  }
+}
+
+}  // namespace phi
diff --git a/paddle/fluid/operators/miopen_lstm_cache.h b/paddle/phi/kernels/gpu/miopen_lstm_cache.h
similarity index 71%
rename from paddle/fluid/operators/miopen_lstm_cache.h
rename to paddle/phi/kernels/gpu/miopen_lstm_cache.h
index a9a6482fd485c..1ce6acc44fcfa 100644
--- a/paddle/fluid/operators/miopen_lstm_cache.h
+++ b/paddle/phi/kernels/gpu/miopen_lstm_cache.h
@@ -16,11 +16,13 @@ limitations under the License. */
 
 #include <vector>
 
-#include "paddle/fluid/framework/tensor.h"
-#include "paddle/fluid/platform/device/gpu/gpu_dnn.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/dynload/miopen.h"
+#include "paddle/phi/backends/gpu/forwards.h"
+#include "paddle/phi/backends/gpu/gpu_dnn.h"
+#include "paddle/phi/core/dense_tensor.h"
 
-namespace paddle {
-namespace operators {
+namespace phi {
 
 class ScopedRNNBase {
  public:
@@ -47,13 +49,13 @@ class ScopedRNNBase {
 
   template <typename T>
   void Create(const miopenHandle_t& handle,
-              const platform::Place& place,
+              const phi::Place& place,
               const std::vector<int>& sequence_length,
               size_t* workspace_size,
               size_t* reserve_size,
               phi::DenseTensor* dropout_state) {
     int numDirections = is_bidirec_ ? 2 : 1;
-    miopenDataType_t miopen_type = platform::CudnnDataType<T>::type;
+    miopenDataType_t miopen_type = phi::backends::gpu::CudnnDataType<T>::type;
 
     // ------------------- miopen x, y descriptors ---------------------
     std::vector<int> dims_x = {batch_size_, input_size_, 1};
@@ -78,9 +80,11 @@ class ScopedRNNBase {
     size_t state_size;
     if (!initialized_) {
       PADDLE_ENFORCE_GPU_SUCCESS(
-          platform::dynload::miopenDropoutGetStatesSize(handle, &state_size));
-      dropout_state->mutable_data<uint8_t>({static_cast<int64_t>(state_size)},
-                                           place);
+          phi::dynload::miopenDropoutGetStatesSize(handle, &state_size));
+      phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+      auto* dev_ctx = reinterpret_cast<phi::GPUContext*>(pool.Get(place));
+      dropout_state->Resize({static_cast<int64_t>(state_size)});
+      dev_ctx->template Alloc<uint8_t>(dropout_state);
     }
     dropout_desc_.descriptor(handle,
                              place,
@@ -91,7 +95,7 @@ class ScopedRNNBase {
                              state_size);
 
     // ------------------- miopen rnn descriptors ---------------------
-    PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::miopenSetRNNDescriptor_V2(
+    PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::miopenSetRNNDescriptor_V2(
         rnn_desc_.desc(),
         hidden_size_,
         num_layers_,
@@ -105,31 +109,28 @@ class ScopedRNNBase {
 
     // ------------------- miopen weights_size ---------------------
     size_t weights_size_;
-    PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::miopenGetRNNParamsSize(
+    PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::miopenGetRNNParamsSize(
         handle, rnn_desc_.desc(), x_descs_[0], &weights_size_, miopen_type));
     PADDLE_ENFORCE_EQ(
         weights_size_,
         sizeof(T) * weight_numel_,
-        platform::errors::InvalidArgument(
+        phi::errors::InvalidArgument(
             "The miopen lstm and setting weight size should be same."));
     // ------------------- miopen weight descriptors ---------------------
-    platform::DataLayout layout = platform::DataLayout::kNCHW;
+    phi::backends::gpu::DataLayout layout =
+        phi::backends::gpu::DataLayout::kNCHW;
     int dim_tmp = weights_size_ / sizeof(T);
     std::vector<int> dim_w = {dim_tmp, 1, 1};
     weight_desc_.descriptor<T>(layout, dim_w);
     // ------------------- miopen workspace, reserve size ---------------------
     PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::miopenGetRNNWorkspaceSize(handle,
-                                                     rnn_desc_.desc(),
-                                                     seq_length_,
-                                                     x_descs_.data(),
-                                                     workspace_size));
-    PADDLE_ENFORCE_GPU_SUCCESS(
-        platform::dynload::miopenGetRNNTrainingReserveSize(handle,
-                                                           rnn_desc_.desc(),
-                                                           seq_length_,
-                                                           x_descs_.data(),
-                                                           reserve_size));
+        phi::dynload::miopenGetRNNWorkspaceSize(handle,
+                                                rnn_desc_.desc(),
+                                                seq_length_,
+                                                x_descs_.data(),
+                                                workspace_size));
+    PADDLE_ENFORCE_GPU_SUCCESS(phi::dynload::miopenGetRNNTrainingReserveSize(
+        handle, rnn_desc_.desc(), seq_length_, x_descs_.data(), reserve_size));
   }
   miopenTensorDescriptor_t* x_descs() { return x_descs_.data(); }
   miopenTensorDescriptor_t* y_descs() { return y_descs_.data(); }
@@ -155,16 +156,15 @@ class ScopedRNNBase {
   std::vector<miopenTensorDescriptor_t> x_descs_;
   std::vector<miopenTensorDescriptor_t> y_descs_;
 
-  platform::ScopedTensorDescriptor x_desc_;
-  platform::ScopedTensorDescriptor y_desc_;
-  platform::ScopedTensorDescriptor init_h_desc_;
-  platform::ScopedTensorDescriptor init_c_desc_;
-  platform::ScopedTensorDescriptor last_h_desc_;
-  platform::ScopedTensorDescriptor last_c_desc_;
-  platform::ScopedDropoutDescriptor dropout_desc_;
-  platform::ScopedFilterDescriptor weight_desc_;
-  platform::ScopedRNNDescriptor rnn_desc_;
+  phi::backends::gpu::ScopedTensorDescriptor x_desc_;
+  phi::backends::gpu::ScopedTensorDescriptor y_desc_;
+  phi::backends::gpu::ScopedTensorDescriptor init_h_desc_;
+  phi::backends::gpu::ScopedTensorDescriptor init_c_desc_;
+  phi::backends::gpu::ScopedTensorDescriptor last_h_desc_;
+  phi::backends::gpu::ScopedTensorDescriptor last_c_desc_;
+  phi::backends::gpu::ScopedDropoutDescriptor dropout_desc_;
+  phi::backends::gpu::ScopedFilterDescriptor weight_desc_;
+  phi::backends::gpu::ScopedRNNDescriptor rnn_desc_;
 };
 
-}  // namespace operators
-}  // namespace paddle
+}  // namespace phi
diff --git a/paddle/phi/ops/compat/cudnn_lstm_sig.cc b/paddle/phi/ops/compat/cudnn_lstm_sig.cc
new file mode 100644
index 0000000000000..83e61b396ee53
--- /dev/null
+++ b/paddle/phi/ops/compat/cudnn_lstm_sig.cc
@@ -0,0 +1,59 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "paddle/phi/core/compat/op_utils.h"
+
+namespace phi {
+
+KernelSignature CudnnLSTMOpArgumentMapping(const ArgumentMappingContext& ctx) {
+  return KernelSignature(
+      "cudnn_lstm",
+      {"Input", "InitH", "InitC", "W", "WeightList", "SequenceLength"},
+      {"dropout_prob",
+       "is_bidirec",
+       "hidden_size",
+       "num_layers",
+       "is_test",
+       "seed"},
+      {"Out", "LastH", "LastC", "Reserve", "StateOut"});
+}
+
+KernelSignature CudnnLSTMGradOpArgumentMapping(
+    const ArgumentMappingContext& ctx) {
+  return KernelSignature(
+      "cudnn_lstm_grad",
+      {"Input",
+       "InitH",
+       "InitC",
+       "WeightList",
+       "SequenceLength",
+       "Out",
+       "Reserve",
+       "StateOut",
+       "Out@GRAD",
+       "LastH@GRAD",
+       "LastC@GRAD"},
+      {"dropout_prob",
+       "is_bidirec",
+       "hidden_size",
+       "num_layers",
+       "is_test",
+       "seed"},
+      {"Input@GRAD", "InitH@GRAD", "InitC@GRAD", "WeightList@GRAD"});
+}
+
+}  // namespace phi
+
+PD_REGISTER_ARG_MAPPING_FN(cudnn_lstm, phi::CudnnLSTMOpArgumentMapping);
+PD_REGISTER_ARG_MAPPING_FN(cudnn_lstm_grad,
+                           phi::CudnnLSTMGradOpArgumentMapping);

From c2c3bd43c6d6ebb88d0da961ae618ab6e6c400de Mon Sep 17 00:00:00 2001
From: niuliling123 <51102941+niuliling123@users.noreply.github.com>
Date: Tue, 16 May 2023 16:09:25 +0800
Subject: [PATCH 360/405] [AMP] support OD level for static (#53768)

---
 python/paddle/static/amp/decorator.py  | 44 +++++++++-------
 python/paddle/static/amp/fp16_lists.py | 16 +++---
 python/paddle/static/amp/fp16_utils.py | 10 +++-
 test/amp/test_amp_api.py               | 70 +++++++++++++++++++++++++-
 4 files changed, 113 insertions(+), 27 deletions(-)

diff --git a/python/paddle/static/amp/decorator.py b/python/paddle/static/amp/decorator.py
index 43dc849925a9a..dca2a4e024c35 100644
--- a/python/paddle/static/amp/decorator.py
+++ b/python/paddle/static/amp/decorator.py
@@ -61,12 +61,11 @@ class OptimizerWithMixedPrecision:
     Args:
         optimizer (Optimizer): A common Optimizer object.
         amp_lists (AutoMixedPrecisionLists): An AutoMixedPrecisionLists object.
-        level(str): Auto mixed precision level. Accepted values are
-            "O1" and "O2": O1 represent mixed precision, the input data type
-            of each operator will be casted by white_list and black_list;
-            O2 represent Pure fp16 or bf16, all operators parameters and input
-            data will be casted to fp16 or bf16, except operators in black_list,
-            don't support fp16 or bf16 kernel and batch_norm.
+        level(str): Auto mixed precision level. Accepted values are "O1", "O2" and "OD": At the O1 level, operators in the white list
+             will use float16/bfloat16 inputs for calculations, and operators in the black list will use float32 inputs for calculations. At the O2
+             level, model's parameters will be casted to float16/bfloat16 by using `decorator`, and operators that have all float16/bfloat16 inputs
+             will be converted to float16/bfloat16, and that have any float32 input will be converted to float32. For the OD level, operators in
+             default white list will compute in float16/bfloat16.
         dtype(str): Whether to use 'float16' or 'bfloat16'.
         init_loss_scaling (float): The initial loss scaling factor.
         use_dynamic_loss_scaling (bool): Whether to use dynamic loss scaling.
@@ -123,6 +122,7 @@ def __init__(
         self._learning_rate = optimizer._learning_rate
         self._learning_rate_map = optimizer._learning_rate_map
         self._use_pure_fp16 = level == "O2"
+        self._amp_level = level
         self._use_fp16_guard = use_amp_guard
         self._to_fp16_var_names = None
         if self._use_dynamic_loss_scaling:
@@ -241,7 +241,7 @@ def backward(
                     self._amp_lists,
                     use_fp16_guard=False,
                     dest_type=self._amp_vartype,
-                    level='O1',
+                    level=self._amp_level,
                     use_promote=self.use_promote,
                 )
 
@@ -380,7 +380,7 @@ def run_example_code():
                     self._amp_lists,
                     use_fp16_guard=False,
                     dest_type=self._amp_vartype,
-                    level='O1',
+                    level=self._amp_level,
                     use_promote=self.use_promote,
                 )
 
@@ -773,12 +773,11 @@ def decorate(
         amp_lists(CustomOpLists, optional): An CustomOpLists object. The default
             white_list and black_list will be used for AMP training when it is
             not set. Default is None.
-        level(str, optional): Auto mixed precision level. Accepted values are
-            "O1" and "O2": O1 represent mixed precision, the input data type of
-            each operator will be casted by white_list and black_list;
-            O2 represent pure FP16 / BF16 training, all operators parameters
-            and input data will be casted to FP16 / BF16, except operators in
-            black_list, don't support FP16 / BF16 kernel and batch_norm. Default is O1.
+        level(str, optional): Auto mixed precision level. Accepted values are "O1", "O2" and "OD": At the O1 level, operators in the white list
+             will use float16/bfloat16 inputs for calculations, and operators in the black list will use float32 inputs for calculations. At the O2
+             level, model's parameters will be casted to float16/bfloat16 by using `decorator`, and operators that have all float16/bfloat16 inputs
+             will be converted to float16/bfloat16, and that have any float32 input will be converted to float32. For the OD level, operators in
+             default white list will compute in float16/bfloat16, and the others will compute in float32. Default is O1.
         dtype(str, optional): Whether to use 'float16' or 'bfloat16'. Default is 'float16'.
         master_weight(bool, optinal): For level='O2', whether to use multi-precision
             during weight updating. If master_weight is None, in O2 level optimizer
@@ -847,15 +846,22 @@ def forward(self, x):
     """
     # check amp_level: O0-O2
     level = level.upper()
-    if not (level in ['O0', 'O1', 'O2']):
-        raise ValueError(
-            "level should be O0, O1 or O2. O0 represents fp32 train mode, O1 represents AMP train mode, O2 represents pure fp16/bf16 train mode."
-        )
+    if not (level in ['O0', 'OD', 'O1', 'O2']):
+        raise ValueError("level should be O0, OD, O1 or O2.")
 
     amp_dtype = check_amp_dtype(dtype)
-    if amp_lists is None:
+    if amp_lists is None or level == 'OD':
         amp_lists = AutoMixedPrecisionLists(dtype=amp_dtype)
 
+    if level == 'OD':
+        if amp_lists is not None:
+            warnings.warn(
+                "If the Amp level is set to OD, the amp list will not be used."
+            )
+
+        amp_lists.white_list = {"conv2d", "matmul_v2"}
+        amp_lists.black_list = amp_lists.all_list - amp_lists.white_list
+
     if use_dynamic_loss_scaling is None:
         use_dynamic_loss_scaling = dtype == "float16"
 
diff --git a/python/paddle/static/amp/fp16_lists.py b/python/paddle/static/amp/fp16_lists.py
index dcf13c0847d6e..96ad079879a50 100644
--- a/python/paddle/static/amp/fp16_lists.py
+++ b/python/paddle/static/amp/fp16_lists.py
@@ -99,7 +99,7 @@ def _get_sys_unsupported_list(dtype):
         device = 'XPU'
     else:
         device = 'GPU'
-    _, _, sys_unsupported_list = core.op_supported_infos(device, var_type)
+    all_ops, _, sys_unsupported_list = core.op_supported_infos(device, var_type)
 
     # sys_unsupported_list will include the following ops.
     supported_fp16_list = {
@@ -114,13 +114,13 @@ def _get_sys_unsupported_list(dtype):
     }
     sys_unsupported_list -= supported_fp16_list
 
-    return device, sys_unsupported_list
+    return device, sys_unsupported_list, all_ops
 
 
 def _get_unsupported_list(dtype):
     # The set of ops that don't support fp16 calculation
-    _, _sys_unsupported_list = _get_sys_unsupported_list(dtype)
-    return _sys_unsupported_list
+    _, _sys_unsupported_list, _sys_all_list = _get_sys_unsupported_list(dtype)
+    return _sys_unsupported_list, _sys_all_list
 
 
 # The three sets listed below are changed dynamiclly. They don't contain all
@@ -200,7 +200,9 @@ def __init__(
         self.white_list = copy.copy(_get_white_list(self.amp_dtype))
         self.black_list = copy.copy(_get_black_list())
         self.gray_list = copy.copy(gray_list)
-        self.unsupported_list = copy.copy(_get_unsupported_list(self.amp_dtype))
+        unsupported_list, sys_all_list = _get_unsupported_list(self.amp_dtype)
+        self.unsupported_list = copy.copy(unsupported_list)
+        self.all_list = copy.copy(sys_all_list)
         self.black_varnames = copy.copy(custom_black_varnames)
         self._update_list()
 
@@ -232,7 +234,9 @@ def _update_list(self):
                     self.gray_list.remove(op_name)
                 self.black_list.add(op_name)
                 self.unsupported_list.add(op_name)
-        device, sys_unsupported_list = _get_sys_unsupported_list(self.amp_dtype)
+        device, sys_unsupported_list, _ = _get_sys_unsupported_list(
+            self.amp_dtype
+        )
         actual_unsupported_list = []
         for op_name in sys_unsupported_list:
             if op_name in self.white_list:
diff --git a/python/paddle/static/amp/fp16_utils.py b/python/paddle/static/amp/fp16_utils.py
index 0ba4db12a78fe..342a2618258df 100644
--- a/python/paddle/static/amp/fp16_utils.py
+++ b/python/paddle/static/amp/fp16_utils.py
@@ -426,7 +426,7 @@ def set_var_dst_dtype(
 
 def set_param_dtype(program, dtype, amp_lists, use_fp16_guard, level):
     keep_fp32_var_names = set()
-    if level == "O1":
+    if level == "O1" or level == "OD":
         return keep_fp32_var_names
     all_parameters = []
     for block in program.blocks:
@@ -618,6 +618,14 @@ def cast_model_to_fp16(
     if level == 'O2':
         amp_lists.black_list = amp_lists.black_list - black_list
 
+    if level == 'OD':
+        if amp_lists is not None:
+            dtype = get_low_precision_dtypestr(dest_type)
+            amp_lists = AutoMixedPrecisionLists(dtype)
+
+        amp_lists.white_list = {"conv2d", "matmul_v2"}
+        amp_lists.black_list = amp_lists.all_list - amp_lists.white_list
+
     global_block = program.global_block()
     keep_fp32_ops = set()
     keep_fp16_ops = set()
diff --git a/test/amp/test_amp_api.py b/test/amp/test_amp_api.py
index 4120dcce93904..ecef047ad6ce7 100644
--- a/test/amp/test_amp_api.py
+++ b/test/amp/test_amp_api.py
@@ -14,9 +14,11 @@
 
 import unittest
 
-from amp_base_models import AmpTestBase
+import numpy as np
+from amp_base_models import AmpTestBase, build_conv_model
 
 import paddle
+from paddle.static import amp
 
 
 class TestAutoCast(AmpTestBase):
@@ -37,6 +39,72 @@ def test_amp_OD_level(self):
         self.assertEqual(out3.dtype, paddle.float32)
 
 
+class TestStaticDecorate(AmpTestBase):
+    def check_results(
+        self, use_amp, dtype, level, use_promote, expected_op_calls
+    ):
+        (
+            main_program,
+            startup_program,
+            optimizer,
+            feed_vars,
+            fetch_vars,
+        ) = build_conv_model(use_amp, dtype, level, use_promote)
+        self.assertEqual(main_program.num_blocks, 1)
+        optimizer = paddle.fluid.optimizer.Adadelta(learning_rate=0.001)
+        optimizer = paddle.static.amp.decorate(
+            optimizer,
+            init_loss_scaling=128.0,
+            use_dynamic_loss_scaling=True,
+            level=level,
+        )
+
+        amp.debugging.collect_operator_stats(main_program)
+        op_stats_list = amp.debugging._get_op_stats_list(main_program)
+
+        self._check_op_calls(
+            op_stats_list[0], expected_fp16_calls=expected_op_calls
+        )
+
+        place = paddle.CUDAPlace(0)
+        exe = paddle.static.Executor(place)
+
+        max_iters = 2
+        x_fp32 = np.random.random(size=[1, 1, 6, 6]).astype("float32")
+        losses_o1 = self.run_program(
+            main_program,
+            startup_program,
+            optimizer,
+            feed_vars,
+            fetch_vars,
+            place,
+            exe,
+            x_fp32,
+            max_iters,
+            level,
+        )
+
+    def test_static_amp_o1(self):
+        paddle.enable_static()
+        expected_fp16_calls = {
+            "conv2d": 1,
+            "elementwise_add": 0,
+            "relu": 0,
+            "matmul_v2": 1,
+            "softmax": 0,
+            "reduce_mean": 0,
+            "adamw": 0,
+        }
+        self.check_results(
+            True,
+            'float16',
+            'OD',
+            use_promote=True,
+            expected_op_calls=expected_fp16_calls,
+        )
+        paddle.disable_static()
+
+
 class TestGradScaler(AmpTestBase):
     def test_amp_grad_scaler(self):
         model = paddle.nn.Conv2D(3, 2, 3)

From 32e36b15dcc2b6d284d79e4c82984ffccfd0e68e Mon Sep 17 00:00:00 2001
From: sprouteer <89541335+sprouteer@users.noreply.github.com>
Date: Tue, 16 May 2023 16:10:13 +0800
Subject: [PATCH 361/405] [XPU] Add sigmoid_elementmul_xpu_fuse_pass (#53580)

---
 paddle/fluid/framework/ir/CMakeLists.txt      |   1 +
 .../ir/sigmoid_elementmul_fuse_pass.cc        | 127 ++++++++++++++++++
 .../ir/sigmoid_elementmul_fuse_pass.h         |  62 +++++++++
 .../inference/api/paddle_pass_builder.cc      |   1 +
 .../test_xpu_sigmoid_elementmul_fuse_pass.py  |  71 ++++++++++
 5 files changed, 262 insertions(+)
 create mode 100644 paddle/fluid/framework/ir/sigmoid_elementmul_fuse_pass.cc
 create mode 100644 paddle/fluid/framework/ir/sigmoid_elementmul_fuse_pass.h
 create mode 100644 test/ir/inference/test_xpu_sigmoid_elementmul_fuse_pass.py

diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt
index 5eb94dbbce435..8130c91a31373 100755
--- a/paddle/fluid/framework/ir/CMakeLists.txt
+++ b/paddle/fluid/framework/ir/CMakeLists.txt
@@ -127,6 +127,7 @@ pass_library(dense_multihead_matmul_to_sparse_pass inference)
 pass_library(delete_cast_op_pass inference)
 pass_library(delete_elementwise_mul_op_pass inference)
 pass_library(delete_repeated_ops_pass inference)
+pass_library(sigmoid_elementmul_fuse_pass inference)
 pass_library(generate_pass DEPS pass_desc_proto)
 target_link_libraries(generate_pass pass_desc_proto)
 
diff --git a/paddle/fluid/framework/ir/sigmoid_elementmul_fuse_pass.cc b/paddle/fluid/framework/ir/sigmoid_elementmul_fuse_pass.cc
new file mode 100644
index 0000000000000..6904e5604fb5c
--- /dev/null
+++ b/paddle/fluid/framework/ir/sigmoid_elementmul_fuse_pass.cc
@@ -0,0 +1,127 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "glog/logging.h"
+
+#include "paddle/fluid/framework/ir/fuse_pass_base.h"
+#include "paddle/fluid/framework/ir/graph_pattern_detector.h"
+#include "paddle/fluid/framework/ir/pass.h"
+#include "paddle/fluid/framework/ir/sigmoid_elementmul_fuse_pass.h"
+#include "paddle/fluid/framework/op_version_registry.h"
+#include "paddle/fluid/platform/enforce.h"
+
+namespace paddle {
+namespace framework {
+namespace ir {
+namespace patterns {
+
+struct SigmoidElementmulFusePattern : public PatternBase {
+  SigmoidElementmulFusePattern(PDPattern* pattern,
+                               const std::string& name_scope);
+
+  // declare operator node's name
+  PATTERN_DECL_NODE(sigmoid);
+  PATTERN_DECL_NODE(elementwise_mul);
+  // declare variable node's name
+  PATTERN_DECL_NODE(sigmoid_x);
+  PATTERN_DECL_NODE(sigmoid_out);
+  PATTERN_DECL_NODE(elemul_out);
+};
+
+SigmoidElementmulFusePattern::SigmoidElementmulFusePattern(
+    PDPattern* pattern, const std::string& name_scope)
+    : PatternBase(pattern, name_scope, name_scope) {
+  auto* sigmoid_x = pattern->NewNode(sigmoid_x_repr())
+                        ->assert_is_op_input("sigmoid", "X")
+                        ->assert_var_not_persistable();
+
+  auto* sigmoid_op = pattern->NewNode(sigmoid_repr())->assert_is_op("sigmoid");
+
+  auto* sigmoid_out = pattern->NewNode(sigmoid_out_repr())
+                          ->assert_is_op_output("sigmoid", "Out")
+                          ->assert_var_not_persistable();
+
+  auto* elemul_op =
+      pattern->NewNode(elementwise_mul_repr())->assert_is_op("elementwise_mul");
+
+  auto* elemul_out = pattern->NewNode(elemul_out_repr())
+                         ->assert_is_op_output("elementwise_mul", "Out")
+                         ->assert_var_not_persistable();
+
+  sigmoid_op->LinksFrom({sigmoid_x}).LinksTo({sigmoid_out});
+  elemul_op->LinksFrom({sigmoid_x, sigmoid_out}).LinksTo({elemul_out});
+}
+
+}  // namespace patterns
+
+SigmoidElementmulFusePass::SigmoidElementmulFusePass() {}
+
+void SigmoidElementmulFusePass::ApplyImpl(ir::Graph* graph) const {
+  PADDLE_ENFORCE_NOT_NULL(
+      graph, platform::errors::PreconditionNotMet("graph should not be null."));
+  Init(name_scope_, graph);
+
+  GraphPatternDetector gpd;
+  patterns::SigmoidElementmulFusePattern pattern(gpd.mutable_pattern(),
+                                                 name_scope_);
+
+  int found_subgraph_count = 0;
+  auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph,
+                     Graph* graph) {
+    VLOG(4) << "handle SigmoidElementmulFusePass fuse";
+#define GET_IR_NODE(node_) GET_IR_NODE_FROM_SUBGRAPH(node_, node_, pattern)
+    GET_IR_NODE(sigmoid_x);
+    GET_IR_NODE(sigmoid);
+    GET_IR_NODE(sigmoid_out);
+    GET_IR_NODE(elementwise_mul);
+    GET_IR_NODE(elemul_out);
+#undef GET_IR_NODE
+    auto* block = sigmoid->Op()->Block();
+    std::string elemul_out_name = elemul_out->Name();
+
+    // Generate swish op
+    framework::OpDesc swish_op_desc(block);
+    swish_op_desc.SetType("swish");
+    swish_op_desc.SetInput("X", {sigmoid_x->Name()});
+    swish_op_desc.SetAttr("beta", 1.f);
+    swish_op_desc.SetOutput("Out", {elemul_out_name});
+
+    auto* swish = graph->CreateOpNode(&swish_op_desc);
+    IR_NODE_LINK_TO(sigmoid_x, swish);
+    IR_NODE_LINK_TO(swish, elemul_out);
+
+    // delete useless node
+    std::unordered_set<const Node*> delete_nodes;
+    delete_nodes = {sigmoid, sigmoid_out, elementwise_mul};
+    GraphSafeRemoveNodes(graph, delete_nodes);
+    found_subgraph_count++;
+  };
+
+  gpd(graph, handler);
+  AddStatis(found_subgraph_count);
+}
+
+}  // namespace ir
+}  // namespace framework
+}  // namespace paddle
+
+REGISTER_PASS(sigmoid_elementmul_fuse_pass,
+              paddle::framework::ir::SigmoidElementmulFusePass);
+
+REGISTER_PASS_CAPABILITY(sigmoid_elementmul_fuse_pass)
+    .AddCombination(
+        paddle::framework::compatible::OpVersionComparatorCombination().EQ(
+            "swish", 0));
diff --git a/paddle/fluid/framework/ir/sigmoid_elementmul_fuse_pass.h b/paddle/fluid/framework/ir/sigmoid_elementmul_fuse_pass.h
new file mode 100644
index 0000000000000..6d116ad9c8110
--- /dev/null
+++ b/paddle/fluid/framework/ir/sigmoid_elementmul_fuse_pass.h
@@ -0,0 +1,62 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include "paddle/fluid/framework/ir/fuse_pass_base.h"
+
+namespace paddle {
+namespace framework {
+namespace ir {
+
+class Graph;
+
+/*
+1. fuse sigmoid + elementwise_mul into swish
+
+Origin subgraph:
+              input
+            /      \
+            |      |
+            |     sigmoid
+            |      |
+            |      |
+          elementwise_mul
+                |
+                |
+               out
+
+Fused subgraph:
+              input
+                |
+                |
+              swish
+                |
+                |
+               out
+*/
+class SigmoidElementmulFusePass : public FusePassBase {
+ public:
+  SigmoidElementmulFusePass();
+  virtual ~SigmoidElementmulFusePass() {}
+
+ protected:
+  void ApplyImpl(ir::Graph* graph) const override;
+
+ private:
+  const std::string name_scope_{"sigmoid_elementmul_fuse_pass"};
+};
+
+}  // namespace ir
+}  // namespace framework
+}  // namespace paddle
diff --git a/paddle/fluid/inference/api/paddle_pass_builder.cc b/paddle/fluid/inference/api/paddle_pass_builder.cc
index bea0b82ecd494..f9123a111771f 100644
--- a/paddle/fluid/inference/api/paddle_pass_builder.cc
+++ b/paddle/fluid/inference/api/paddle_pass_builder.cc
@@ -522,6 +522,7 @@ XpuPassStrategy::XpuPassStrategy() : PassStrategy({}) {
       "delete_cast_op_pass",
       "stack_fuse_pass",
       "fused_multi_transformer_xpu_pass",
+      "sigmoid_elementmul_fuse_pass",
       "fc_xpu_fuse_pass",
       "conv2d_xpu_fuse_pass",
       "link_xpu_op_max_pass",
diff --git a/test/ir/inference/test_xpu_sigmoid_elementmul_fuse_pass.py b/test/ir/inference/test_xpu_sigmoid_elementmul_fuse_pass.py
new file mode 100644
index 0000000000000..e6a348b30a8c9
--- /dev/null
+++ b/test/ir/inference/test_xpu_sigmoid_elementmul_fuse_pass.py
@@ -0,0 +1,71 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import hypothesis.strategies as st
+import numpy as np
+from auto_scan_test import PassAutoScanTest
+from program_config import OpConfig, ProgramConfig, TensorConfig
+
+
+class TestSigmoidElementmulFusePass(PassAutoScanTest):
+    def sample_predictor_configs(self, program_config):
+        config = self.create_inference_config(use_xpu=True)
+        yield config, ["swish"], (1e-3, 1e-3)
+
+    def sample_program_config(self, draw):
+        # 1. sigmoid
+        sigmoid_x_shape = draw(
+            st.lists(
+                st.integers(min_value=1, max_value=4), min_size=2, max_size=4
+            )
+        )
+
+        sigmoid_op = OpConfig(
+            "sigmoid",
+            inputs={"X": ["sigmoid_x"]},
+            outputs={"Out": ["sigmoid_out"]},
+            trans_x=False,
+            trans_y=False,
+        )
+        mul_op = OpConfig(
+            "elementwise_mul",
+            inputs={"X": ["sigmoid_x"], "Y": ["sigmoid_out"]},
+            outputs={"Out": ["out"]},
+            axis=-1,
+        )
+        ops = [sigmoid_op, mul_op]
+
+        program_config = ProgramConfig(
+            ops=ops,
+            weights={},
+            inputs={
+                "sigmoid_x": TensorConfig(shape=sigmoid_x_shape),
+            },
+            outputs=ops[-1].outputs["Out"],
+        )
+        return program_config
+
+    def test(self):
+        self.run_and_statis(
+            quant=False,
+            max_examples=25,
+            passes=["sigmoid_elementmul_fuse_pass"],
+        )
+
+
+if __name__ == "__main__":
+    np.random.seed(200)
+    unittest.main()

From 69161a96d2cc9ccb067df5548fb57ae6f3e39316 Mon Sep 17 00:00:00 2001
From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com>
Date: Tue, 16 May 2023 16:21:41 +0800
Subject: [PATCH 362/405] =?UTF-8?q?=E3=80=90static=E3=80=91modify=20backwa?=
 =?UTF-8?q?rd=20prune=20logic=20for=20EmptygradOpMaker=20(#53746)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add rules

* modify no kernel yaml parse

* success op generate

* success test_silu_double

* modify bug

* modify static error

* modify silu_grad input

* modify kernel signature

* modify kernel signature

* code style

* code style

* review

* delete opinfo modify

* modify gradOpMaker

* modify gradOpMaker

* modify genarated-j2

* add approve rules

* modify aytograd_functional_static_test
---
 paddle/fluid/framework/op_info.h                    |  2 ++
 .../generator/templates/operator_utils.c.j2         |  1 +
 paddle/fluid/pybind/pybind.cc                       | 13 ++++++++++---
 python/paddle/fluid/backward.py                     |  6 +++---
 test/autograd/test_autograd_functional_static.py    |  2 +-
 tools/check_file_diff_approvals.sh                  |  3 ++-
 6 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/paddle/fluid/framework/op_info.h b/paddle/fluid/framework/op_info.h
index 5ce39d0c3a334..e1bc5be8c64f9 100644
--- a/paddle/fluid/framework/op_info.h
+++ b/paddle/fluid/framework/op_info.h
@@ -97,6 +97,8 @@ class OpInfo {
     return grad_op_maker_ != nullptr && !use_empty_grad_op_desc_maker_;
   }
 
+  bool HasEmptyGradOpMaker() const { return use_empty_grad_op_desc_maker_; }
+
   const DygraphGradOpMakerFN& DygraphGradOpMaker() const {
     // Normally, proto_ should not be null, except some special operators, such
     // as LeaklyReluDoubleGrad op.
diff --git a/paddle/fluid/operators/generator/templates/operator_utils.c.j2 b/paddle/fluid/operators/generator/templates/operator_utils.c.j2
index 16f0ecaa642f3..f762eedf3bffb 100644
--- a/paddle/fluid/operators/generator/templates/operator_utils.c.j2
+++ b/paddle/fluid/operators/generator/templates/operator_utils.c.j2
@@ -477,6 +477,7 @@ REGISTER_OPERATOR({{name}}, ops::{{name | to_pascal_case}}Op,
   {% set backward_name = op["backward"] %}
                   ops::{{backward_name | to_pascal_case}}OpMaker<paddle::framework::OpDesc>,
                   ops::{{backward_name | to_pascal_case}}OpMaker<paddle::imperative::OpBase>,
+{% elif "forward" in op %}
 {% else %}
                   paddle::framework::EmptyGradOpMaker<paddle::framework::OpDesc>,
                   paddle::framework::EmptyGradOpMaker<paddle::imperative::OpBase>,
diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc
index a9b1bf398f84e..34bb746c48ed6 100644
--- a/paddle/fluid/pybind/pybind.cc
+++ b/paddle/fluid/pybind/pybind.cc
@@ -1327,8 +1327,7 @@ All parameter, weight, gradient are variables in Paddle.
           if ((grad_op_maker == nullptr) && (grad_comp_op_maker == nullptr)) {
             // Normally, proto_ should not be null, except some special
             // operators, such as LeaklyReluDoubleGrad op.
-            std::string type =
-                op_info.proto_ ? op_info.proto_->type() : "unknown";
+            std::string type = op_desc.Type();
             PADDLE_THROW(platform::errors::NotFound(
                 "Neither operator %s's GradOpMaker nor CompGradOpMaker has "
                 "been registered.\nPlease check whether (%s) operator has "
@@ -1350,7 +1349,8 @@ All parameter, weight, gradient are variables in Paddle.
           VLOG(3) << "need skip: " << need_skip << std::endl;
           if (paddle::prim::PrimCommonUtils::IsBwdPrimEnabled()) {
             if ((grad_comp_op_maker != nullptr) && (!need_skip)) {
-              VLOG(3) << "Runing composite fun for " << op_desc.Type();
+              VLOG(3) << "Prim Flag Open: Runing composite grad fun for "
+                      << op_desc.Type();
               grad_op_descs = grad_comp_op_maker(op_desc,
                                                  no_grad_set,
                                                  &grad_to_var,
@@ -1362,9 +1362,13 @@ All parameter, weight, gradient are variables in Paddle.
             }
           } else {
             if (grad_op_maker != nullptr) {
+              VLOG(3) << "Prim Flag Close: Runing origin grad fun for "
+                      << op_desc.Type();
               grad_op_descs = grad_op_maker(
                   op_desc, no_grad_set, &grad_to_var, grad_sub_block);
             } else {
+              VLOG(3) << "Prim Flag Close: Runing composite grad fun for "
+                      << op_desc.Type();
               grad_op_descs = grad_comp_op_maker(op_desc,
                                                  no_grad_set,
                                                  &grad_to_var,
@@ -1392,6 +1396,9 @@ All parameter, weight, gradient are variables in Paddle.
         .Get(op_type)
         .HasNonEmptyGradOpMaker();
   });
+  m.def("has_empty_grad_op_maker", [](const std::string op_type) {
+    return framework::OpInfoMap::Instance().Get(op_type).HasEmptyGradOpMaker();
+  });
   m.def("has_infer_inplace", [](const std::string op_type) {
     return framework::OpInfoMap::Instance().Get(op_type).HasInferInplace();
   });
diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py
index 51fe6294a5077..f7b8531aee4d9 100755
--- a/python/paddle/fluid/backward.py
+++ b/python/paddle/fluid/backward.py
@@ -2348,7 +2348,7 @@ def _find_op_path_(
         for i, op in enumerate(block.ops):
             if _some_in_set_(
                 op.desc.input_arg_names(), input_names
-            ) and core.has_non_empty_grad_op_maker(op.type):
+            ) and not core.has_empty_grad_op_maker(op.type):
                 for name in op.desc.output_arg_names():
                     if name not in no_grad_set:
                         input_names.add(name)
@@ -2367,7 +2367,7 @@ def _find_op_path_(
 
         if _some_in_set_(
             op.desc.output_arg_names(), output_names
-        ) and core.has_non_empty_grad_op_maker(op.type):
+        ) and not core.has_empty_grad_op_maker(op.type):
             for name in op.desc.input_arg_names():
                 if name not in no_grad_set:
                     output_names.add(name)
@@ -2382,7 +2382,7 @@ def _find_op_path_(
                 op.desc.output_arg_names(), output_names
             ):
                 relevant_op_flags[i] = True
-                if core.has_non_empty_grad_op_maker(op.type):
+                if not core.has_empty_grad_op_maker(op.type):
                     for name in op.desc.input_arg_names():
                         if name not in no_grad_set:
                             output_names.add(name)
diff --git a/test/autograd/test_autograd_functional_static.py b/test/autograd/test_autograd_functional_static.py
index 38c27c5e2384e..8786515d2699a 100644
--- a/test/autograd/test_autograd_functional_static.py
+++ b/test/autograd/test_autograd_functional_static.py
@@ -466,7 +466,7 @@ def run_test_by_fullmatrix(self, pd_f, inps, np_hess, batch=False):
     def test_square(self):
         def pd_f(x):
             """Input is a square matrix."""
-            return paddle.matmul(x, x.T).flatten().sum()
+            return paddle.matmul(x, x.T).sum()
 
         def np_hess(x):
             dim = x.shape[0]
diff --git a/tools/check_file_diff_approvals.sh b/tools/check_file_diff_approvals.sh
index 3215f9d507222..f2fb9d86cdfda 100644
--- a/tools/check_file_diff_approvals.sh
+++ b/tools/check_file_diff_approvals.sh
@@ -81,6 +81,7 @@ API_FILES=("CMakeLists.txt"
            "paddle/phi/core/kernel_context.h"
            "paddle/phi/core/infermeta_utils.h"
            "paddle/fluid/prim/api/composite_backward/composite_backward_api.h"
+           "paddle/fluid/prim/api/composite_backward/composite_double_backward_api.h"
            "paddle/fluid/prim/api/manual_prim/prim_manual_api.h"
            "paddle/fluid/prim/api/api.yaml"
            "python/paddle/incubate/autograd/composite_rules.py"
@@ -207,7 +208,7 @@ for API_FILE in ${API_FILES[*]}; do
       elif [ "${API_FILE}" == "paddle/phi/api/include/tensor.h" ] || [ "${API_FILE}" == "paddle/phi/core/tensor_base.h" ] || [ "${API_FILE}" == "paddle/phi/core/dense_tensor.h" ] || [ "${API_FILE}" == "paddle/phi/core/meta_tensor.h" ] || [ "${API_FILE}" == "paddle/phi/core/tensor_meta.h" ] || [ "${API_FILE}" == "paddle/phi/core/attribute.h" ] || [ "${API_FILE}" == "paddle/phi/core/device_context.h" ] || [ "${API_FILE}" == "paddle/phi/core/kernel_utils.h" ] || [ "${API_FILE}" == "paddle/phi/core/kernel_registry.h" ] || [ "${API_FILE}" == "paddle/phi/core/kernel_factory.h" ] || [ "${API_FILE}" == "paddle/phi/core/kernel_context.h" ] || [ "${API_FILE}" == "paddle/phi/core/infermeta_utils.h" ]; then
             echo_line="You must have one RD (chenwhql, phlrain, zyfncg, YuanRisheng) approval for changing ${API_FILE} , which manages the underlying code for PaddlePaddle PHI Library.\n"
             check_approval chenwhql phlrain zyfncg YuanRisheng
-      elif [ "${API_FILE}" == "paddle/fluid/prim/api/composite_backward/composite_backward_api.h" ] || [ "${API_FILE}" == "paddle/fluid/prim/api/manual_prim/prim_manual_api.h" ] || [ "${API_FILE}" == "paddle/fluid/prim/api/api.yaml" ]; then
+      elif [ "${API_FILE}" == "paddle/fluid/prim/api/composite_backward/composite_backward_api.h" ] || [ "${API_FILE}" == "paddle/fluid/prim/api/manual_prim/prim_manual_api.h" ] || [ "${API_FILE}" == "paddle/fluid/prim/api/api.yaml" ] || [ "${API_FILE}" == "paddle/fluid/prim/api/composite_backward/composite_double_backward_api.h" ]; then
             echo_line="You must have one RD (JiabinYang, cxxly(chenxiaoxu) , xiaoguoguo626807(wangruting)) approval for changing ${API_FILE} , which manages the code for PaddlePaddle Composite Bacward Prim API.\n"
             check_approval 1 JiabinYang cxxly xiaoguoguo626807
       elif [ "${API_FILE}" == "python/paddle/incubate/autograd/primitives.py" ] || [ "${API_FILE}" == "python/paddle/incubate/autograd/composite_rules.py" ]; then

From c33ba9d4bb3b79a4445df499661edb8bde3df2f4 Mon Sep 17 00:00:00 2001
From: liuzhenhai93 <liuzhenhai93@outlook.com>
Date: Tue, 16 May 2023 17:47:36 +0800
Subject: [PATCH 363/405] Fix some tests for issuse 52842 (#53795)

* polish

* polish
---
 paddle/fluid/pybind/communication.cc          |  4 +--
 .../communication/stream/reduce_scatter.py    |  1 +
 .../collective/collective_allgather_api.py    |  2 ++
 .../collective/collective_allreduce_api.py    |  1 +
 .../collective/collective_broadcast_api.py    |  1 +
 .../collective/collective_reduce_api.py       |  1 +
 .../collective/collective_sendrecv_api.py     |  2 ++
 .../unittests/test_collective_api_base.py     | 27 ++++++++++++++++---
 8 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/paddle/fluid/pybind/communication.cc b/paddle/fluid/pybind/communication.cc
index 31b89d1ca94b9..2379ac15a37f3 100644
--- a/paddle/fluid/pybind/communication.cc
+++ b/paddle/fluid/pybind/communication.cc
@@ -78,8 +78,8 @@ void BindTCPStore(py::module *m) {
                        [](phi::distributed::Store &self,
                           const std::string &key) -> py::bytes {
                          auto data = self.get(key);
-                         return py::bytes(reinterpret_cast<char *>(data.data()),
-                                          data.size());
+                         std::string s(data.begin(), data.end());
+                         return py::bytes(s);
                        },
                        py::arg("key"),
                        py::call_guard<py::gil_scoped_release>())
diff --git a/python/paddle/distributed/communication/stream/reduce_scatter.py b/python/paddle/distributed/communication/stream/reduce_scatter.py
index 79c1749990826..5812ec5d8de5c 100644
--- a/python/paddle/distributed/communication/stream/reduce_scatter.py
+++ b/python/paddle/distributed/communication/stream/reduce_scatter.py
@@ -81,6 +81,7 @@ def _reduce_scatter_in_static_mode(tensor, tensor_or_tensor_list, group):
             'int8',
             'uint8',
             'bool',
+            'uint16',
         ],
         op_type,
     )
diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allgather_api.py b/python/paddle/fluid/tests/unittests/collective/collective_allgather_api.py
index b9289592c2e1e..b9e28c21a0eed 100644
--- a/python/paddle/fluid/tests/unittests/collective/collective_allgather_api.py
+++ b/python/paddle/fluid/tests/unittests/collective/collective_allgather_api.py
@@ -43,6 +43,7 @@ def all_gather_new(tensor_list, tensor, group=None):
                 'bool',
                 'int8',
                 'uint8',
+                'uint16',
             ],
             op_type,
         )
@@ -58,6 +59,7 @@ def all_gather_new(tensor_list, tensor, group=None):
             'bool',
             'int8',
             'uint8',
+            'uint16',
         ],
         op_type,
     )
diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api.py b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api.py
index c4a79e5d7041c..a93718643ed3c 100644
--- a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api.py
+++ b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api.py
@@ -36,6 +36,7 @@ def all_reduce_new(tensor, reduce_type=str(dist.ReduceOp.SUM), group=None):
             'int8',
             'uint8',
             'bool',
+            'uint16',
         ],
         op_type,
     )
diff --git a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api.py b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api.py
index 8bf096c3bd8c3..67002ebffc863 100644
--- a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api.py
+++ b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api.py
@@ -35,6 +35,7 @@ def broadcast_new(tensor, src, group=None, sync_op=True):
             'int8',
             'uint8',
             'bool',
+            'uint16',
         ],
         op_type,
     )
diff --git a/python/paddle/fluid/tests/unittests/collective/collective_reduce_api.py b/python/paddle/fluid/tests/unittests/collective/collective_reduce_api.py
index 71f1efd703a8d..0eb11a282d4dd 100644
--- a/python/paddle/fluid/tests/unittests/collective/collective_reduce_api.py
+++ b/python/paddle/fluid/tests/unittests/collective/collective_reduce_api.py
@@ -36,6 +36,7 @@ def reduce_new(tensor, dst, reduce_type=str(dist.ReduceOp.SUM), group=None):
             'int8',
             'uint8',
             'bool',
+            'uint16',
         ],
         op_type,
     )
diff --git a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api.py b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api.py
index 96fb99feb1583..d59e479bb3f81 100644
--- a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api.py
+++ b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api.py
@@ -35,6 +35,7 @@ def send_new(tensor, dst, group=None, sync_op=True):
             'int8',
             'uint8',
             'bool',
+            'uint16',
         ],
         op_type,
     )
@@ -67,6 +68,7 @@ def recv_new(tensor, src, group=None, sync_op=True, dtype='float32'):
             'int8',
             'uint8',
             'bool',
+            'uint16',
         ],
         op_type,
     )
diff --git a/python/paddle/fluid/tests/unittests/test_collective_api_base.py b/python/paddle/fluid/tests/unittests/test_collective_api_base.py
index d53231bdc31d4..fcd58c25e36e1 100644
--- a/python/paddle/fluid/tests/unittests/test_collective_api_base.py
+++ b/python/paddle/fluid/tests/unittests/test_collective_api_base.py
@@ -22,7 +22,7 @@
 from contextlib import closing
 
 import numpy as np
-from paddle_bfloat import bfloat16
+from eager_op_test import convert_float_to_uint16, convert_uint16_to_float
 
 import paddle
 import paddle.distributed as dist
@@ -45,6 +45,14 @@ def create_float_test_data(shape=None, dtype=None, seed=None):
     return data
 
 
+def create_bfloat16_test_data(shape=None, seed=None):
+    if seed:
+        np.random.seed(seed)
+    data = np.random.uniform(-100.0, 100.0, shape).astype("float32")
+    data = convert_float_to_uint16(data)
+    return data
+
+
 def create_int_test_data(shape=None, dtype=None, seed=None):
     if seed:
         np.random.seed(seed)
@@ -76,8 +84,9 @@ def create_test_data(shape=None, dtype=None, seed=None):
     if dtype == "float32" or dtype == "float16" or dtype == "float64":
         return create_float_test_data(shape=shape, dtype=dtype, seed=seed)
     elif dtype == "bfloat16":
+        return create_bfloat16_test_data(shape=shape, seed=seed)
         # since numpy does not support bfloat16 yet, use `paddle_bfloat` to replace
-        return create_float_test_data(shape=shape, dtype=bfloat16, seed=seed)
+        # return create_float_test_data(shape=shape, dtype=bfloat16, seed=seed)
     elif dtype == "bool":
         return create_bool_test_data(shape=shape, seed=seed)
     elif (
@@ -344,8 +353,18 @@ def check_with_place(
         input2 = create_test_data(shape=(10, 1000), dtype=dtype, seed=pid1)
         # cast bfloat16 to float32 for numeric comparison
         if dtype == "bfloat16":
-            input1 = input1.astype("float32")
-            input2 = input2.astype("float32")
+
+            def convertbf16(origin):
+                if origin.dtype == np.uint16:
+                    return convert_uint16_to_float(origin)
+                else:
+                    return origin.astype("float32")
+
+            input1 = convertbf16(input1)
+            input2 = convertbf16(input2)
+            tr0_out = [convertbf16(e) for e in tr0_out]
+            tr1_out = [convertbf16(e) for e in tr1_out]
+
         if col_type == "allgather":
             need_result = np.vstack((input1, input2))
             tr_out0 = np.vstack((tr0_out[0], tr0_out[1]))

From 0ab7f9496ea2c3f2adf677c07f662654a0899b6b Mon Sep 17 00:00:00 2001
From: Yuang Liu <liuyuang@baidu.com>
Date: Tue, 16 May 2023 18:15:25 +0800
Subject: [PATCH 364/405] add timer to pp (#53831)

---
 .../framework/distributed_strategy.proto      |   1 +
 .../fleet/meta_parallel/pipeline_parallel.py  |  45 ++++++-
 .../pp_utils/p2p_communication.py             |  56 ++++++++-
 .../distributed/fleet/utils/timer_helper.py   | 113 ++++++++++++++++++
 ...allel_pp_transformer_with_virtual_stage.py |   3 +
 .../unittests/hybrid_parallel_pp_alexnet.py   |   1 +
 6 files changed, 215 insertions(+), 4 deletions(-)
 create mode 100644 python/paddle/distributed/fleet/utils/timer_helper.py

diff --git a/paddle/fluid/framework/distributed_strategy.proto b/paddle/fluid/framework/distributed_strategy.proto
index 0435199ecaa48..4082bf2ec5585 100755
--- a/paddle/fluid/framework/distributed_strategy.proto
+++ b/paddle/fluid/framework/distributed_strategy.proto
@@ -61,6 +61,7 @@ message MpConfig {
 message PpConfig {
     optional bool dp_comm_overlap = 1 [ default = false ];
     optional bool delay_scale_loss = 2 [ default = false ];
+    optional bool enable_timer = 3 [ default = false ];
 }
 
 message HybridConfig {
diff --git a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py
index 2a7dd4d0bb710..5ff01d35ec5ff 100755
--- a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py
+++ b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py
@@ -15,6 +15,7 @@
 from paddle import framework
 
 from ..meta_optimizers.dygraph_optimizer import HybridParallelOptimizer
+from ..utils import timer_helper as timer
 from ..utils.hybrid_parallel_util import (
     broadcast_dp_parameters,
     broadcast_mp_parameters,
@@ -73,13 +74,24 @@ def __init__(self, layers, hcg, strategy):
         self._dp_comm_overlap = self._strategy.hybrid_configs[
             "pp_configs"
         ].dp_comm_overlap
+        self._enable_timer = self._strategy.hybrid_configs[
+            "pp_configs"
+        ].enable_timer
         self._dp_comm_buffers = []
 
         if self._dp_comm_overlap:
             assert self.use_data_parallel and self.num_stages > 1
 
+        if self._enable_timer:
+            if not timer.is_timer_initialized():
+                timer.set_timers()
+            self.timers = timer.get_timers()
+
         p2p.initialize_p2p_groups(
-            hcg, self._using_cache, self._enable_partial_send_recv
+            hcg,
+            self._using_cache,
+            self._enable_partial_send_recv,
+            self._enable_timer,
         )
 
         self.global_rank = self._hcg.get_global_rank()
@@ -153,6 +165,12 @@ def register_allreduce_overlap_hook(self, model, comm_group, acc_steps):
             for param in parameters:
                 param._register_backward_hook(self.bw_hook_func(buffer, param))
 
+    def timer_printer(self):
+        if not self._enable_timer:
+            return
+        all_flag_names = self.timers.timers.keys()
+        self.timers.log(all_flag_names)
+
     def forward_backward_pipeline(self, data, scaler=None):
         # use the 1f1b scheduling strategy.
         # this strategy is inspired by:
@@ -236,9 +254,17 @@ def forward_backward_pipeline(self, data, scaler=None):
             for buffer in self._dp_comm_buffers:
                 buffer.scale_and_split_grads()
 
+        if self._enable_timer:
+            self.timers("allreduce_shared_weight_gradients").start()
         self._layers.allreduce_shared_weight_gradients()
+        if self._enable_timer:
+            self.timers("allreduce_shared_weight_gradients").stop()
+            self.timers("broadcast_final_loss").start()
         with paddle.amp.auto_cast(enable=False):
             train_loss = self._broadcast_final_loss()
+        if self._enable_timer:
+            self.timers("broadcast_final_loss").stop()
+        self.timer_printer()
         return train_loss
 
     def _prepare_training(self, data, optimizer, lr_scheduler):
@@ -334,6 +360,8 @@ def eval_batch(self, data, compute_loss=False):
         return self.train_loss
 
     def _forward_step(self, input_tensor, chunk_id=None):
+        if self._enable_timer:
+            self.timers("forward_step").start()
         if self.is_pipeline_first_stage():
             input_tensor = self._load_micro_batch(self.micro_batch_id)
 
@@ -365,9 +393,13 @@ def _forward_step(self, input_tensor, chunk_id=None):
             # Only increase micro batch id at virtual first/last pp stage.
             # The micro batch id is used to load data, therefore, only increase it when load data.
             self.micro_batch_id += 1
+        if self._enable_timer:
+            self.timers("forward_step").stop()
         return output_tensor
 
     def _backward_step(self, input_tensor, output_tensor, output_tensor_grad):
+        if self._enable_timer:
+            self.timers("backward_step").start()
         with paddle.amp.auto_cast(enable=False):
             if self.is_pipeline_last_stage():
                 assert output_tensor_grad is None
@@ -397,6 +429,8 @@ def _backward_step(self, input_tensor, output_tensor, output_tensor_grad):
                     )
                 else:
                     input_tensor_grad = input_tensor.grad
+            if self._enable_timer:
+                self.timers("backward_step").stop()
             return input_tensor_grad
 
     def _check_data_vaild(self, data):
@@ -807,16 +841,25 @@ def forward_backward_pipeline(
                 for buffer in self._dp_comm_buffers:
                     buffer.scale_and_split_grads()
 
+            if self._enable_timer:
+                self.timers("allreduce_shared_weight_gradients").start()
             self._layers.allreduce_shared_weight_gradients()
+            if self._enable_timer:
+                self.timers("allreduce_shared_weight_gradients").stop()
 
         if compute_loss:
             # return loss if compute loss
+            if self._enable_timer:
+                self.timers("broadcast_final_loss").start()
             with paddle.amp.auto_cast(enable=False):
                 train_loss = self._broadcast_final_loss()
+            if self._enable_timer:
+                self.timers("broadcast_final_loss").stop()
         else:
             # else just return all intermediate output tensor for all micro steps
             train_loss = self.output_tensors
 
+        self.timer_printer()
         return train_loss
 
     def train_batch(self, data, optimizer, lr_scheduler=None, scaler=None):
diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
index fa4b937ba56b7..1da86c83640b5 100644
--- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
+++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
@@ -19,12 +19,14 @@
 import paddle
 from paddle import framework
 
+from ...utils import timer_helper as timer
 from ...utils.log_util import logger
 from .utils import number_2_dtype, paddle_2_number
 
 _hcg = None
 _use_cache = False
 _enable_partial_send_recv = True
+_timers = None
 
 _xpu_comm_group_started = False
 
@@ -50,11 +52,15 @@ def _xpu_comm_group_end():
         _xpu_comm_group_started = False
 
 
-def initialize_p2p_groups(hcg, use_cache=True, enable_partial_send_recv=True):
-    global _hcg, _use_cache, _enable_partial_send_recv
+def initialize_p2p_groups(
+    hcg, use_cache=True, enable_partial_send_recv=True, enable_timer=False
+):
+    global _hcg, _use_cache, _enable_partial_send_recv, _timers
     _hcg = hcg
     _use_cache = use_cache
     _enable_partial_send_recv = enable_partial_send_recv
+    if enable_timer:
+        _timers = timer.get_timers()
     (
         send_next_group,
         send_prev_group,
@@ -683,6 +689,9 @@ def _p2p_helper(
 
 
 def recv_forward(pp_first_stage, sync_recv=True):
+    global _timers
+    if _timers is not None:
+        _timers("recv_forward").start()
     if pp_first_stage:
         input_tensor = None
     else:
@@ -697,10 +706,15 @@ def recv_forward(pp_first_stage, sync_recv=True):
             recv_next=False,
             sync_recv=sync_recv,
         )
+    if _timers is not None:
+        _timers("recv_forward").stop()
     return input_tensor
 
 
 def recv_backward(pp_last_stage, sync_recv=True):
+    global _timers
+    if _timers is not None:
+        _timers("recv_backward").start()
     if pp_last_stage:
         output_tensor_grad = None
     else:
@@ -711,10 +725,15 @@ def recv_backward(pp_last_stage, sync_recv=True):
             recv_next=True,
             sync_recv=sync_recv,
         )
+    if _timers is not None:
+        _timers("recv_backward").stop()
     return output_tensor_grad
 
 
 def send_forward(output_tensor, pp_last_stage):
+    global _timers
+    if _timers is not None:
+        _timers("send_forward").start()
     if not pp_last_stage:
         if not _send_recv_meta.has_send_meta:
             _send_recv_meta.set_send_message(output_tensor)
@@ -727,9 +746,14 @@ def send_forward(output_tensor, pp_last_stage):
             recv_prev=False,
             recv_next=False,
         )
+    if _timers is not None:
+        _timers("send_forward").stop()
 
 
 def send_backward(input_tensor_grad, pp_first_stage):
+    global _timers
+    if _timers is not None:
+        _timers("send_backward").start()
     if not pp_first_stage:
         _p2p_helper(
             tensor_send_next=None,
@@ -737,9 +761,14 @@ def send_backward(input_tensor_grad, pp_first_stage):
             recv_prev=False,
             recv_next=False,
         )
+    if _timers is not None:
+        _timers("send_backward").stop()
 
 
 def send_forward_recv_backward(output_tensor, pp_last_stage):
+    global _timers
+    if _timers is not None:
+        _timers("send_forward_recv_backward").start()
     if pp_last_stage:
         output_tensor_grad = None
     else:
@@ -749,10 +778,15 @@ def send_forward_recv_backward(output_tensor, pp_last_stage):
             recv_prev=False,
             recv_next=True,
         )
+    if _timers is not None:
+        _timers("send_forward_recv_backward").stop()
     return output_tensor_grad
 
 
 def send_backward_recv_forward(input_tensor_grad, pp_first_stage):
+    global _timers
+    if _timers is not None:
+        _timers("send_backward_recv_forward").start()
     if pp_first_stage:
         input_tensor = None
     else:
@@ -762,6 +796,8 @@ def send_backward_recv_forward(input_tensor_grad, pp_first_stage):
             recv_prev=True,
             recv_next=False,
         )
+    if _timers is not None:
+        _timers("send_backward_recv_forward").stop()
     return input_tensor
 
 
@@ -769,6 +805,9 @@ def send_forward_backward_recv_forward_backward(
     output_tensor, input_tensor_grad, recv_prev, recv_next
 ):
     # always have to send dytpe info to downstream
+    global _timers
+    if _timers is not None:
+        _timers("send_forward_backward_recv_forward_backward").start()
     if not _send_recv_meta.has_send_meta:
         _send_recv_meta.set_send_message(output_tensor)
         _send_recv_meta.send_meta(output_tensor, _hcg.send_next_group)
@@ -783,11 +822,16 @@ def send_forward_backward_recv_forward_backward(
         recv_next=recv_next,
         sync_recv=False,
     )
+    if _timers is not None:
+        _timers("send_forward_backward_recv_forward_backward").stop()
     return input_tensor, output_tensor_grad
 
 
 def send_forward_recv_forward(output_tensor, recv_prev):
     # always have to send dytpe info to downstream
+    global _timers
+    if _timers is not None:
+        _timers("send_forward_recv_forward").start()
     if not _send_recv_meta.has_send_meta:
         _send_recv_meta.set_send_message(output_tensor)
         _send_recv_meta.send_meta(output_tensor, _hcg.send_next_group)
@@ -803,11 +847,15 @@ def send_forward_recv_forward(output_tensor, recv_prev):
         recv_next=False,
         sync_recv=False,
     )
-
+    if _timers is not None:
+        _timers("send_forward_recv_forward").stop()
     return input_tensor
 
 
 def send_backward_recv_backward(input_tensor_grad, recv_next):
+    global _timers
+    if _timers is not None:
+        _timers("send_backward_recv_backward").start()
     _, output_tensor_grad = _p2p_helper(
         tensor_send_next=None,
         tensor_send_prev=input_tensor_grad,
@@ -815,4 +863,6 @@ def send_backward_recv_backward(input_tensor_grad, recv_next):
         recv_next=recv_next,
         sync_recv=False,
     )
+    if _timers is not None:
+        _timers("send_backward_recv_backward").stop()
     return output_tensor_grad
diff --git a/python/paddle/distributed/fleet/utils/timer_helper.py b/python/paddle/distributed/fleet/utils/timer_helper.py
new file mode 100644
index 0000000000000..1c0e737f00526
--- /dev/null
+++ b/python/paddle/distributed/fleet/utils/timer_helper.py
@@ -0,0 +1,113 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+import paddle
+
+_GLOBAL_TIMERS = None
+
+
+def is_timer_initialized():
+    return _GLOBAL_TIMERS is not None
+
+
+def _ensure_var_is_not_initialized(var, name):
+    """Make sure the input variable is not None."""
+    assert var is None, f"{name} has been already initialized."
+
+
+def _ensure_var_is_initialized(var, name):
+    """Make sure the input variable is not None."""
+    assert var is not None, f"{name} is not initialized."
+
+
+def get_timers():
+    _ensure_var_is_initialized(_GLOBAL_TIMERS, "timers")
+    return _GLOBAL_TIMERS
+
+
+def set_timers():
+    """Initialize timers."""
+    global _GLOBAL_TIMERS
+    _ensure_var_is_not_initialized(_GLOBAL_TIMERS, "timers")
+    _GLOBAL_TIMERS = Timers()
+
+
+class _Timer:
+    """Timer."""
+
+    def __init__(self, name):
+        self.name = name
+        self.elapsed_ = 0.0
+        self.started_ = False
+        self.start_time = time.time()
+
+    def start(self):
+        """Start the timer."""
+        assert not self.started_, "timer has already started"
+        paddle.device.cuda.synchronize()
+        self.start_time = time.time()
+        self.started_ = True
+
+    def stop(self):
+        """Stop the timers."""
+        assert self.started_, "timer is not started."
+        paddle.device.cuda.synchronize()
+        self.elapsed_ += time.time() - self.start_time
+        self.started_ = False
+
+    def reset(self):
+        """Reset timer."""
+        self.elapsed_ = 0.0
+        self.started_ = False
+
+    def elapsed(self, reset=True):
+        """Calculate the elapsed time."""
+        started_ = self.started_
+        # If the timing in progress, end it first.
+        if self.started_:
+            self.stop()
+        # Get the elapsed time.
+        elapsed_ = self.elapsed_
+        # Reset the elapsed time
+        if reset:
+            self.reset()
+        # If timing was in progress, set it back.
+        if started_:
+            self.start()
+        return elapsed_
+
+
+class Timers:
+    """Group of timers."""
+
+    def __init__(self):
+        self.timers = {}
+
+    def __call__(self, name):
+        if name not in self.timers:
+            self.timers[name] = _Timer(name)
+        return self.timers[name]
+
+    def log(self, names, normalizer=1.0, reset=True):
+        """Log a group of timers."""
+        assert normalizer > 0.0
+        string = "time (ms)"
+        for name in names:
+            elapsed_time = (
+                self.timers[name].elapsed(reset=reset) * 1000.0 / normalizer
+            )
+            string += f" | {name}: {elapsed_time:.2f}"
+        print(string, flush=True)
diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py
index 9f43fc4c9ef8e..3d7991727ee83 100644
--- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py
+++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py
@@ -147,6 +147,9 @@ def setUp(self):
             "dp_degree": self.data_parallel_size,
             "mp_degree": self.model_parallel_size,
             "pp_degree": self.pipeline_parallel_size,
+            "pp_configs": {
+                "enable_timer": True,
+            },
         }
         strategy.pipeline_configs = {
             "accumulate_steps": batch_size // micro_batch_size,
diff --git a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py
index 1c3eac9cec402..052abbeb594ce 100644
--- a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py
+++ b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py
@@ -145,6 +145,7 @@ def setUp(self):
             "pp_degree": self.pipeline_parallel_size,
             "pp_configs": {
                 "delay_scale_loss": True,
+                "enable_timer": True,
             },
         }
         strategy.pipeline_configs = {

From e592534a5af4204c8cf4f667c824e7cefcc06309 Mon Sep 17 00:00:00 2001
From: Winters Montagne
 <118546135+WintersMontagne10335@users.noreply.github.com>
Date: Tue, 16 May 2023 19:00:01 +0800
Subject: [PATCH 365/405] =?UTF-8?q?=E3=80=90PaddlePaddle=20Hackathon=204?=
 =?UTF-8?q?=20No.34=E3=80=91=E4=B8=BA=20Paddle=20=E4=BC=98=E5=8C=96=20Lerp?=
 =?UTF-8?q?=20OP=20=E5=9C=A8=20GPU=20=E4=B8=8A=E7=9A=84=E6=80=A7=E8=83=BD?=
 =?UTF-8?q?=20(#53154)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* modify lerp_kernel.cu

* pre-commit

* fix some CI issues

* fix some CI issues

* fix some CI issues

* fix some CI issues

* fix some CI issues

* fix some CI issues

* fix some CI issues

* fix some CI issues

* Add files via upload

fix some CI issues
---
 paddle/phi/kernels/gpu/lerp_kernel.cu | 109 +++++++++++++++++++++++++-
 1 file changed, 108 insertions(+), 1 deletion(-)

diff --git a/paddle/phi/kernels/gpu/lerp_kernel.cu b/paddle/phi/kernels/gpu/lerp_kernel.cu
index 25f37bb170476..4304dd38238f7 100644
--- a/paddle/phi/kernels/gpu/lerp_kernel.cu
+++ b/paddle/phi/kernels/gpu/lerp_kernel.cu
@@ -15,8 +15,115 @@
 #include "paddle/phi/kernels/lerp_kernel.h"
 
 #include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/common/amp_type_traits.h"
 #include "paddle/phi/core/kernel_registry.h"
-#include "paddle/phi/kernels/impl/lerp_kernel_impl.h"
+#include "paddle/phi/kernels/empty_kernel.h"
+#include "paddle/phi/kernels/funcs/broadcast_function.h"
+#include "paddle/phi/kernels/funcs/common_shape.h"
+#include "paddle/phi/kernels/funcs/math_function.h"
+
+namespace phi {
+
+template <typename T>
+struct BroadcastMinElementWiseDirectCUDAFunctor {
+  HOSTDEVICE inline T operator()(const T min) const { return min; }
+};
+
+template <typename T>
+struct LerpElementWiseDirectCUDAFunctor {
+  HOSTDEVICE inline T operator()(const T x, const T y, const T weight) const {
+    return x + weight * (y - x);
+  }
+};
+
+template <typename T>
+struct LerpScalarDirectCUDAFunctor {
+  const T *weight_;
+
+  HOSTDEVICE inline LerpScalarDirectCUDAFunctor(const T *weight)
+      : weight_(weight) {}
+
+  HOSTDEVICE inline T operator()(const T x, const T y) const {
+    return x + weight_[0] * (y - x);
+  }
+};
+
+template <typename T, typename Context>
+void LerpKernel(const Context &ctx,
+                const DenseTensor &x,
+                const DenseTensor &y,
+                const DenseTensor &weight,
+                DenseTensor *out) {
+  int rank = out->dims().size();
+  PADDLE_ENFORCE_GE(
+      rank,
+      0,
+      phi::errors::InvalidArgument(
+          "The number of dimensions for LerpOp must be "
+          "greater than or equal to 0, but the value received is %d.",
+          rank));
+
+  ctx.template Alloc<T>(out);
+  std::vector<DenseTensor *> outputs = {out};
+
+  std::vector<const DenseTensor *> inputs;
+  if (weight.numel() == 1) {
+    const T *weight_ptr = weight.data<T>();
+    inputs.reserve(2);
+    inputs.emplace_back(&x);
+    inputs.emplace_back(&y);
+    auto functor = LerpScalarDirectCUDAFunctor<T>(weight_ptr);
+    phi::funcs::BroadcastKernel<T>(ctx, inputs, &outputs, functor);
+  } else {
+    inputs.reserve(3);
+    auto functor = LerpElementWiseDirectCUDAFunctor<T>();
+    DenseTensor b_min = phi::EmptyLike<T>(ctx, *out);
+    if (x.dims().size() != y.dims().size() &&
+        weight.dims().size() != y.dims().size()) {
+      std::vector<const DenseTensor *> broadcast_min_inputs;
+      broadcast_min_inputs.reserve(1);
+      std::vector<DenseTensor *> broadcast_min_outputs = {&b_min};
+      auto broadcast_min_functor =
+          BroadcastMinElementWiseDirectCUDAFunctor<T>();
+      if (x.dims().size() < y.dims().size() &&
+          x.dims().size() < weight.dims().size()) {
+        broadcast_min_inputs.emplace_back(&x);
+        phi::funcs::BroadcastKernel<T>(ctx,
+                                       broadcast_min_inputs,
+                                       &broadcast_min_outputs,
+                                       broadcast_min_functor);
+        inputs.emplace_back(&b_min);
+        inputs.emplace_back(&y);
+        inputs.emplace_back(&weight);
+      } else if (y.dims().size() < weight.dims().size()) {
+        broadcast_min_inputs.emplace_back(&y);
+        phi::funcs::BroadcastKernel<T>(ctx,
+                                       broadcast_min_inputs,
+                                       &broadcast_min_outputs,
+                                       broadcast_min_functor);
+        inputs.emplace_back(&x);
+        inputs.emplace_back(&b_min);
+        inputs.emplace_back(&weight);
+      } else {
+        broadcast_min_inputs.emplace_back(&weight);
+        phi::funcs::BroadcastKernel<T>(ctx,
+                                       broadcast_min_inputs,
+                                       &broadcast_min_outputs,
+                                       broadcast_min_functor);
+        inputs.emplace_back(&x);
+        inputs.emplace_back(&y);
+        inputs.emplace_back(&b_min);
+      }
+    } else {
+      inputs.emplace_back(&x);
+      inputs.emplace_back(&y);
+      inputs.emplace_back(&weight);
+    }
+    phi::funcs::BroadcastKernel<T>(ctx, inputs, &outputs, functor);
+  }
+}
+
+}  // namespace phi
 
 PD_REGISTER_KERNEL(lerp,
                    GPU,

From 10a38b4e5c685066f5a710175983b86e7d5997ab Mon Sep 17 00:00:00 2001
From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com>
Date: Tue, 16 May 2023 19:26:50 +0800
Subject: [PATCH 366/405] remove some [-Wunused-parameter] warning and  fix a
 file to pass cpplint (#53814)

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop

* test,test=develop
---
 .../framework/details/nan_inf_utils_detail.h  |   4 +-
 .../ir/fusion_group/code_generator_helper.h   |   2 +-
 paddle/fluid/imperative/data_loader.cc        |   3 +-
 paddle/fluid/imperative/infer_shape_context.h |  28 ++--
 .../fluid/imperative/infer_var_type_context.h |  40 ++---
 .../fluid/operators/collective/c_concat_op.h  |   2 +-
 .../c_softmax_with_cross_entropy_op.h         |   2 +-
 paddle/fluid/operators/controlflow/feed_op.cc |   2 +-
 .../operators/elementwise/elementwise_op.h    |  10 +-
 .../fused/mkldnn/multi_gru_mkldnn_op.cc       |   2 +-
 paddle/fluid/platform/device_context.cc       |   3 +-
 .../cpu/margin_cross_entropy_kernel.cc        |  26 ++--
 paddle/phi/kernels/cpu/rnn_functor.h          |   2 +-
 paddle/phi/kernels/cpu/sgd_kernel.cc          |  20 +--
 paddle/phi/kernels/cpu/uniform_kernel.cc      |   2 +-
 paddle/phi/kernels/funcs/activation_functor.h |  87 +++++------
 .../phi/kernels/funcs/detection/poly_util.cc  | 138 ------------------
 .../phi/kernels/funcs/detection/poly_util.h   | 103 ++++++++++++-
 .../phi/kernels/legacy/cpu/uniform_kernel.cc  |   2 +-
 19 files changed, 218 insertions(+), 260 deletions(-)
 delete mode 100644 paddle/phi/kernels/funcs/detection/poly_util.cc

diff --git a/paddle/fluid/framework/details/nan_inf_utils_detail.h b/paddle/fluid/framework/details/nan_inf_utils_detail.h
index f4ee2c20b9d2f..ed2fa25a5ae60 100644
--- a/paddle/fluid/framework/details/nan_inf_utils_detail.h
+++ b/paddle/fluid/framework/details/nan_inf_utils_detail.h
@@ -61,7 +61,9 @@ HOSTDEVICE bool NeedPrint(MT max_value, MT min_value, int check_nan_inf_level) {
 template <typename T,
           typename MT,
           std::enable_if_t<!std::is_same<T, float>::value, bool> = true>
-HOSTDEVICE bool NeedPrint(MT max_value, MT min_value, int check_nan_inf_level) {
+HOSTDEVICE bool NeedPrint(MT max_value UNUSED,
+                          MT min_value UNUSED,
+                          int check_nan_inf_level) {
   if (check_nan_inf_level >= 3) {
     return true;
   }
diff --git a/paddle/fluid/framework/ir/fusion_group/code_generator_helper.h b/paddle/fluid/framework/ir/fusion_group/code_generator_helper.h
index 0980b17853cad..e068403181b1b 100644
--- a/paddle/fluid/framework/ir/fusion_group/code_generator_helper.h
+++ b/paddle/fluid/framework/ir/fusion_group/code_generator_helper.h
@@ -95,7 +95,7 @@ class TemplateVariable {
     strings_[identifier] = expression;
   }
 
-  void Remove(std::string identifier, std::string expression) {
+  void Remove(std::string identifier, std::string expression UNUSED) {
     for (auto it = strings_.begin(); it != strings_.end();) {
       if (it->first == identifier) {
         it = strings_.erase(it);
diff --git a/paddle/fluid/imperative/data_loader.cc b/paddle/fluid/imperative/data_loader.cc
index 65f6ef4e53ce6..f225b871b6860 100644
--- a/paddle/fluid/imperative/data_loader.cc
+++ b/paddle/fluid/imperative/data_loader.cc
@@ -73,7 +73,8 @@ void EraseLoadProcessPIDs(int64_t key) {
   } while (0)
 
 #define REGISTER_SIGNAL_HANDLER(SIGNAL, HANDLER_NAME, ERROR_MSG)           \
-  static void HANDLER_NAME(int sig, siginfo_t *info, void *ctx) {          \
+  static void HANDLER_NAME(                                                \
+      int sig UNUSED, siginfo_t *info UNUSED, void *ctx UNUSED) {          \
     auto _w =                                                              \
         write(STDERR_FILENO, ERROR_MSG, sizeof(ERROR_MSG) / sizeof(char)); \
     (void)_w;                                                              \
diff --git a/paddle/fluid/imperative/infer_shape_context.h b/paddle/fluid/imperative/infer_shape_context.h
index 6f1f54de8a95c..80c52ab4ac10f 100644
--- a/paddle/fluid/imperative/infer_shape_context.h
+++ b/paddle/fluid/imperative/infer_shape_context.h
@@ -237,14 +237,14 @@ class DygraphInferShapeContext : public framework::InferShapeContext {
     }
   }
 
-  void ShareAllLoD(const std::string& in,
-                   const std::string& out) const override {
+  void ShareAllLoD(const std::string& in UNUSED,
+                   const std::string& out UNUSED) const override {
     // do nothing
   }
-  void ShareLoD(const std::string& in,
-                const std::string& out,
-                size_t i = 0,
-                size_t j = 0) const override {
+  void ShareLoD(const std::string& in UNUSED,
+                const std::string& out UNUSED,
+                size_t i UNUSED = 0,
+                size_t j UNUSED = 0) const override {
     // do nothing
   }
 
@@ -415,14 +415,15 @@ class DygraphInferShapeContext : public framework::InferShapeContext {
     }
   }
 
-  int32_t GetLoDLevel(const std::string& in, size_t i = 0) const override {
+  int32_t GetLoDLevel(const std::string& in UNUSED,
+                      size_t i UNUSED = 0) const override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "GetLoDLevel function not support in dygraph mode"));
   }
 
-  void SetLoDLevel(const std::string& out,
-                   int32_t lod_level,
-                   size_t j = 0) const override {
+  void SetLoDLevel(const std::string& out UNUSED,
+                   int32_t lod_level UNUSED,
+                   size_t j UNUSED = 0) const override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "SetLoDLevel function not support in dygraph mode"));
   }
@@ -452,7 +453,8 @@ class DygraphInferShapeContext : public framework::InferShapeContext {
     }
   }
 
-  std::vector<DDim> GetRepeatedDims(const std::string& name) const override {
+  std::vector<DDim> GetRepeatedDims(
+      const std::string& name UNUSED) const override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "GetRepeatedDims not support in dygraph runtime"));
   }
@@ -486,8 +488,8 @@ class DygraphInferShapeContext : public framework::InferShapeContext {
     }
   }
 
-  void SetRepeatedDims(const std::string& name,
-                       const std::vector<DDim>& dims) override {
+  void SetRepeatedDims(const std::string& name UNUSED,
+                       const std::vector<DDim>& dims UNUSED) override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "SetRepeatedDims not support in dygraph runtime"));
   }
diff --git a/paddle/fluid/imperative/infer_var_type_context.h b/paddle/fluid/imperative/infer_var_type_context.h
index 36558366e1d2f..fcf9c1d3ab4c9 100644
--- a/paddle/fluid/imperative/infer_var_type_context.h
+++ b/paddle/fluid/imperative/infer_var_type_context.h
@@ -145,8 +145,8 @@ class RuntimeInferVarTypeContext : public framework::InferVarTypeContext {
   }
 
   void SetOutputDataType(const std::string& name,
-                         framework::proto::VarType::Type type,
-                         int index = 0) override {
+                         framework::proto::VarType::Type type UNUSED,
+                         int index UNUSED = 0) override {
     VLOG(10) << "Set data type in infer var type of Eager mode is meaning less "
                 "for var: "
              << name;
@@ -155,77 +155,79 @@ class RuntimeInferVarTypeContext : public framework::InferVarTypeContext {
   bool IsDygraph() const override { return true; }
 
  protected:
-  bool HasVar(const std::string& name) const override {
+  bool HasVar(const std::string& name UNUSED) const override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "HasVar is not supported in runtime InferVarType"));
   }
 
   const std::vector<std::string>& InputVars(
-      const std::string& name) const override {
+      const std::string& name UNUSED) const override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "InputVars is not supported in runtime InferVarType"));
   }
 
   const std::vector<std::string>& OutputVars(
-      const std::string& name) const override {
+      const std::string& name UNUSED) const override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "OutputVars is not supported in runtime InferVarType"));
   }
 
   framework::proto::VarType::Type GetVarType(
-      const std::string& name) const override {
+      const std::string& name UNUSED) const override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "Do not manipulate var in runtime InferVarType"));
   }
 
-  void SetVarType(const std::string& name,
-                  framework::proto::VarType::Type type) override {
+  void SetVarType(const std::string& name UNUSED,
+                  framework::proto::VarType::Type type UNUSED) override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "Do not manipulate var in runtime InferVarType"));
   }
 
   framework::proto::VarType::Type GetVarDataType(
-      const std::string& name) const override {
+      const std::string& name UNUSED) const override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "Do not manipulate var in runtime InferVarType"));
   }
 
-  void SetVarDataType(const std::string& name,
-                      framework::proto::VarType::Type type) override {
+  void SetVarDataType(const std::string& name UNUSED,
+                      framework::proto::VarType::Type type UNUSED) override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "Do not manipulate var in runtime InferVarType"));
   }
 
   std::vector<framework::proto::VarType::Type> GetVarDataTypes(
-      const std::string& name) const override {
+      const std::string& name UNUSED) const override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "GetVarDataTypes is not supported in runtime InferVarType"));
   }
 
-  void SetVarDataTypes(const std::string& name,
+  void SetVarDataTypes(const std::string& name UNUSED,
                        const std::vector<framework::proto::VarType::Type>&
-                           multiple_data_type) override {
+                           multiple_data_type UNUSED) override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "SetVarDataTypes is not supported in runtime InferVarType"));
   }
 
-  std::vector<int64_t> GetVarShape(const std::string& name) const override {
+  std::vector<int64_t> GetVarShape(
+      const std::string& name UNUSED) const override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "Do not handle Shape in runtime InferVarType"));
   }
 
-  void SetVarShape(const std::string& name,
-                   const std::vector<int64_t>& dims) override {
+  void SetVarShape(const std::string& name UNUSED,
+                   const std::vector<int64_t>& dims UNUSED) override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "Do not handle Shape in runtime InferVarType"));
   }
 
-  int32_t GetVarLoDLevel(const std::string& name) const override {
+  int32_t GetVarLoDLevel(const std::string& name UNUSED) const override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "Do not handle LoDLevel in runtime InferVarType"));
   }
 
-  void SetVarLoDLevel(const std::string& name, int32_t lod_level) override {
+  void SetVarLoDLevel(const std::string& name UNUSED,
+                      int32_t lod_level UNUSED) override {
     PADDLE_THROW(platform::errors::PermissionDenied(
         "Do not handle LoDLevel in runtime InferVarType"));
   }
diff --git a/paddle/fluid/operators/collective/c_concat_op.h b/paddle/fluid/operators/collective/c_concat_op.h
index d9fe5c861a4da..39bdc4c2740de 100644
--- a/paddle/fluid/operators/collective/c_concat_op.h
+++ b/paddle/fluid/operators/collective/c_concat_op.h
@@ -28,7 +28,7 @@ namespace operators {
 template <typename T, typename DeviceContext>
 class CConcatOpCPUKernel : public framework::OpKernel<T> {
  public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
+  void Compute(const framework::ExecutionContext& ctx UNUSED) const override {
     PADDLE_THROW(platform::errors::Unavailable(
         "Do not support c_concat for cpu kernel now."));
   }
diff --git a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.h b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.h
index c8311bdab813c..9b6a2c86897cb 100644
--- a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.h
+++ b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.h
@@ -32,7 +32,7 @@ namespace operators {
 template <typename T, typename DeviceContext>
 class CSoftmaxWithCrossEntropyOpCPUKernel : public framework::OpKernel<T> {
  public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
+  void Compute(const framework::ExecutionContext& ctx UNUSED) const override {
     PADDLE_THROW(platform::errors::Unavailable(
         "Do not support c_embedding for cpu kernel now."));
   }
diff --git a/paddle/fluid/operators/controlflow/feed_op.cc b/paddle/fluid/operators/controlflow/feed_op.cc
index 698ca4e02e629..c2deeb4190986 100644
--- a/paddle/fluid/operators/controlflow/feed_op.cc
+++ b/paddle/fluid/operators/controlflow/feed_op.cc
@@ -97,7 +97,7 @@ void FeedSparseCooTensorKernel(const Context& dev_ctx,
 }
 
 template <typename Context>
-void FeedStringsKernel(const Context& dev_ctx,
+void FeedStringsKernel(const Context& dev_ctx UNUSED,
                        const phi::ExtendedTensor& x,
                        int col,
                        phi::ExtendedTensor* out) {
diff --git a/paddle/fluid/operators/elementwise/elementwise_op.h b/paddle/fluid/operators/elementwise/elementwise_op.h
index 4ef896ff01b87..2f8db0e737a21 100644
--- a/paddle/fluid/operators/elementwise/elementwise_op.h
+++ b/paddle/fluid/operators/elementwise/elementwise_op.h
@@ -159,7 +159,7 @@ class ElementwiseOp : public framework::OperatorWithKernel {
   }
 
   phi::KernelKey GetKernelTypeForVar(
-      const std::string &var_name,
+      const std::string &var_name UNUSED,
       const phi::DenseTensor &tensor,
       const phi::KernelKey &expected_kernel_type) const override {
     if (framework::IsComplexType(expected_kernel_type.dtype())) {
@@ -305,7 +305,7 @@ class ElementwiseOpGrad : public framework::OperatorWithKernel {
   }
 
   phi::KernelKey GetKernelTypeForVar(
-      const std::string &var_name,
+      const std::string &var_name UNUSED,
       const phi::DenseTensor &tensor,
       const phi::KernelKey &expected_kernel_type) const override {
     if (framework::IsComplexType(expected_kernel_type.dtype())) {
@@ -346,7 +346,7 @@ class ElementwiseOpDoubleGrad : public framework::OperatorWithKernel {
   }
 
   phi::KernelKey GetKernelTypeForVar(
-      const std::string &var_name,
+      const std::string &var_name UNUSED,
       const phi::DenseTensor &tensor,
       const phi::KernelKey &expected_kernel_type) const override {
     if (framework::IsComplexType(expected_kernel_type.dtype())) {
@@ -394,7 +394,7 @@ class ElementwiseOpDoubleGradWithoutDXDY
   }
 
   phi::KernelKey GetKernelTypeForVar(
-      const std::string &var_name,
+      const std::string &var_name UNUSED,
       const phi::DenseTensor &tensor,
       const phi::KernelKey &expected_kernel_type) const override {
     if (framework::IsComplexType(expected_kernel_type.dtype())) {
@@ -442,7 +442,7 @@ class ElementwiseOpTripleGrad : public framework::OperatorWithKernel {
   }
 
   phi::KernelKey GetKernelTypeForVar(
-      const std::string &var_name,
+      const std::string &var_name UNUSED,
       const phi::DenseTensor &tensor,
       const phi::KernelKey &expected_kernel_type) const override {
     if (framework::IsComplexType(expected_kernel_type.dtype())) {
diff --git a/paddle/fluid/operators/fused/mkldnn/multi_gru_mkldnn_op.cc b/paddle/fluid/operators/fused/mkldnn/multi_gru_mkldnn_op.cc
index dba3087f1a01b..ff6613d8ae94f 100644
--- a/paddle/fluid/operators/fused/mkldnn/multi_gru_mkldnn_op.cc
+++ b/paddle/fluid/operators/fused/mkldnn/multi_gru_mkldnn_op.cc
@@ -598,7 +598,7 @@ class MultiGRUHandler {
   }
 
   template <typename Tout>
-  void reorderOutput(std::shared_ptr<dnnl::memory> mem, int layer) {
+  void reorderOutput(std::shared_ptr<dnnl::memory> mem, int layer UNUSED) {
     auto* data = mem->get_data_handle();
     auto* hidden_data =
         phi::funcs::to_void_cast(hidden_->mutable_data<Tout>(place_));
diff --git a/paddle/fluid/platform/device_context.cc b/paddle/fluid/platform/device_context.cc
index e14ba8b1710cf..456abd55ef68f 100644
--- a/paddle/fluid/platform/device_context.cc
+++ b/paddle/fluid/platform/device_context.cc
@@ -57,7 +57,8 @@ DeviceType Place2DeviceType(const platform::Place& place) {
 template <typename DevCtx>
 typename std::enable_if<!std::is_same<DevCtx, phi::GPUContext>::value,
                         DevCtx*>::type
-ConstructDevCtx(const phi::Place& p, /*unused*/ int stream_priority = 0) {
+ConstructDevCtx(const phi::Place& p,
+                /*unused*/ int stream_priority UNUSED = 0) {
   return new DevCtx(p);
 }
 
diff --git a/paddle/phi/kernels/cpu/margin_cross_entropy_kernel.cc b/paddle/phi/kernels/cpu/margin_cross_entropy_kernel.cc
index 06d74471dd992..b967d7232254c 100644
--- a/paddle/phi/kernels/cpu/margin_cross_entropy_kernel.cc
+++ b/paddle/phi/kernels/cpu/margin_cross_entropy_kernel.cc
@@ -21,19 +21,19 @@
 namespace phi {
 
 template <typename T, typename Context>
-void MarginCrossEntropyKernel(const Context& dev_ctx,
-                              const DenseTensor& logits,
-                              const DenseTensor& labels,
-                              bool return_softmax,
-                              int ring_id,
-                              int rank,
-                              int nranks,
-                              float margin1,
-                              float margin2,
-                              float margin3,
-                              float scale,
-                              DenseTensor* softmax,
-                              DenseTensor* loss) {
+void MarginCrossEntropyKernel(const Context& dev_ctx UNUSED,
+                              const DenseTensor& logits UNUSED,
+                              const DenseTensor& labels UNUSED,
+                              bool return_softmax UNUSED,
+                              int ring_id UNUSED,
+                              int rank UNUSED,
+                              int nranks UNUSED,
+                              float margin1 UNUSED,
+                              float margin2 UNUSED,
+                              float margin3 UNUSED,
+                              float scale UNUSED,
+                              DenseTensor* softmax UNUSED,
+                              DenseTensor* loss UNUSED) {
   PADDLE_THROW(
       errors::Unavailable("Do not support margin_cross_entropy for cpu kernel "
                           "now."));
diff --git a/paddle/phi/kernels/cpu/rnn_functor.h b/paddle/phi/kernels/cpu/rnn_functor.h
index 49b0c7655c6df..511021c00860a 100644
--- a/paddle/phi/kernels/cpu/rnn_functor.h
+++ b/paddle/phi/kernels/cpu/rnn_functor.h
@@ -79,7 +79,7 @@ void CreateMaskMatrix(const CPUContext& dev_ctx,
 template <typename TensorType>
 void ResetParameterVector(const std::vector<TensorType>& raw_params_vec,
                           int num_layers,
-                          int gate_num,
+                          int gate_num UNUSED,
                           bool is_bidirec,
                           std::vector<std::vector<DenseTensor>>* params_vec) {
   // the parameter raw seuquence is [FWhi, FWhh, BWhi, BWhh] * num_layers
diff --git a/paddle/phi/kernels/cpu/sgd_kernel.cc b/paddle/phi/kernels/cpu/sgd_kernel.cc
index e63e2f2637bdc..30a3a13a50d69 100644
--- a/paddle/phi/kernels/cpu/sgd_kernel.cc
+++ b/paddle/phi/kernels/cpu/sgd_kernel.cc
@@ -117,10 +117,10 @@ void SGDDenseKernel(const Context& dev_ctx,
                     const DenseTensor& param,
                     const DenseTensor& learning_rate,
                     const DenseTensor& grad,
-                    const paddle::optional<DenseTensor>& master_param,
-                    bool multi_precision,
+                    const paddle::optional<DenseTensor>& master_param UNUSED,
+                    bool multi_precision UNUSED,
                     DenseTensor* param_out,
-                    DenseTensor* master_param_out) {
+                    DenseTensor* master_param_out UNUSED) {
   dev_ctx.template Alloc<T>(param_out);
   sgd_dense_param_dense_grad_impl<T>(param, learning_rate, grad, param_out);
 }
@@ -131,24 +131,24 @@ void SGDDenseParamSparseGradKernel(
     const DenseTensor& param,
     const DenseTensor& learning_rate,
     const SelectedRows& grad,
-    const paddle::optional<DenseTensor>& master_param,
-    bool multi_precision,
+    const paddle::optional<DenseTensor>& master_param UNUSED,
+    bool multi_precision UNUSED,
     DenseTensor* param_out,
-    DenseTensor* master_param_out) {
+    DenseTensor* master_param_out UNUSED) {
   dev_ctx.template Alloc<T>(param_out);
   sgd_dense_param_sparse_grad_impl<T>(param, learning_rate, grad, param_out);
 }
 
 template <typename T, typename Context>
 void SGDSparseParamSparseGradKernel(
-    const Context& dev_ctx,
+    const Context& dev_ctx UNUSED,
     const SelectedRows& param,
     const DenseTensor& learning_rate,
     const SelectedRows& grad,
-    const paddle::optional<SelectedRows>& master_param,
-    bool multi_precision,
+    const paddle::optional<SelectedRows>& master_param UNUSED,
+    bool multi_precision UNUSED,
     SelectedRows* param_out,
-    SelectedRows* master_param_out) {
+    SelectedRows* master_param_out UNUSED) {
   // for distributed training, a sparse var may be empty,
   // just skip updating.
   if (grad.rows().size() == 0) {
diff --git a/paddle/phi/kernels/cpu/uniform_kernel.cc b/paddle/phi/kernels/cpu/uniform_kernel.cc
index 17ee545918824..d850dc5074e03 100644
--- a/paddle/phi/kernels/cpu/uniform_kernel.cc
+++ b/paddle/phi/kernels/cpu/uniform_kernel.cc
@@ -22,7 +22,7 @@ namespace phi {
 template <typename T, typename Context>
 void UniformKernel(const Context &dev_ctx,
                    const IntArray &shape,
-                   DataType dtype,
+                   DataType dtype UNUSED,
                    const Scalar &min,
                    const Scalar &max,
                    int seed,
diff --git a/paddle/phi/kernels/funcs/activation_functor.h b/paddle/phi/kernels/funcs/activation_functor.h
index df63c22c94600..b73ca44f7cace 100644
--- a/paddle/phi/kernels/funcs/activation_functor.h
+++ b/paddle/phi/kernels/funcs/activation_functor.h
@@ -104,7 +104,7 @@ struct SinGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = dout * x.unaryExpr(Cosine<T>());
   }
 
@@ -277,7 +277,7 @@ struct ReciprocalGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x UNUSED, Out out, dOut dout, dX dx) const {
     dx.device(d) = dout * static_cast<T>(-1) * out * out;
   }
 
@@ -310,7 +310,7 @@ struct CosGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = -dout * x.unaryExpr(Sine<T>());
   }
 
@@ -505,7 +505,7 @@ struct MishGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     auto sp = (x > static_cast<T>(threshold))
                   .select(x, (static_cast<T>(1) + x.exp()).log());
     auto gsp = static_cast<T>(1) - (-sp).exp();
@@ -544,7 +544,7 @@ struct STanhGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     auto a = static_cast<T>(scale_a);
     auto b = static_cast<T>(scale_b);
     auto temp = (a * x).tanh() * (a * x).tanh();
@@ -574,7 +574,7 @@ struct TanGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = dout / x.unaryExpr(Cosine<T>()).square();
   }
 
@@ -620,7 +620,7 @@ struct SqrtGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x UNUSED, Out out, dOut dout, dX dx) const {
     dx.device(d) = static_cast<T>(0.5) * dout / out;
   }
 
@@ -645,7 +645,7 @@ struct RsqrtGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x UNUSED, Out out, dOut dout, dX dx) const {
     dx.device(d) = static_cast<T>(-0.5) * dout * out * out * out;
   }
 
@@ -697,7 +697,7 @@ struct SoftplusGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     auto x_beta = static_cast<T>(beta) * x;
     dx.device(d) =
         (x_beta > static_cast<T>(threshold))
@@ -816,7 +816,7 @@ struct SinhGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = dout * x.unaryExpr(Cosh<T>());
   }
 
@@ -831,7 +831,7 @@ struct CoshGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = dout * x.unaryExpr(Sinh<T>());
   }
 
@@ -867,7 +867,7 @@ struct AcosGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) =
         -dout * static_cast<T>(1) / (static_cast<T>(1) - x.square()).sqrt();
   }
@@ -904,7 +904,7 @@ struct AsinGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) =
         dout * static_cast<T>(1) / (static_cast<T>(1) - x.square()).sqrt();
   }
@@ -941,7 +941,7 @@ struct AtanGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = dout * static_cast<T>(1) / (static_cast<T>(1) + x.square());
   }
 
@@ -989,7 +989,7 @@ struct AcoshGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) =
         dout * static_cast<T>(1) / (x * x - static_cast<T>(1)).sqrt();
   }
@@ -1026,7 +1026,7 @@ struct AsinhGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) =
         dout * static_cast<T>(1) / (x.square() + static_cast<T>(1)).sqrt();
   }
@@ -1063,7 +1063,7 @@ struct AtanhGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = dout * static_cast<T>(1) / (static_cast<T>(1) - x.square());
   }
 
@@ -1160,12 +1160,12 @@ template <typename T>
 struct ReluGradGradFunctor : public BaseActivationFunctor<T> {
   template <typename Device>
   void operator()(const Device& dev,
-                  const DenseTensor* X,
+                  const DenseTensor* X UNUSED,
                   const DenseTensor* Out,
                   const DenseTensor* ddX,
                   DenseTensor* ddOut,
-                  DenseTensor* dOut,
-                  DenseTensor* dX) const {
+                  DenseTensor* dOut UNUSED,
+                  DenseTensor* dX UNUSED) const {
     auto* d = dev.eigen_device();
     auto ddx = EigenVector<T>::Flatten(
         GET_DATA_SAFELY(ddX, "Input", "DDX", "ReluGradGrad"));
@@ -1375,7 +1375,7 @@ struct HardTanhGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) =
         dout * ((x > static_cast<T>(t_min)) * (x < static_cast<T>(t_max)))
                    .template cast<T>();
@@ -1431,11 +1431,11 @@ struct LeakyReluGradGradFunctor : public BaseActivationFunctor<T> {
   template <typename Device>
   void operator()(const Device& dev,
                   const DenseTensor* X,
-                  const DenseTensor* Out,
+                  const DenseTensor* Out UNUSED,
                   const DenseTensor* ddX,
                   DenseTensor* ddOut,
-                  DenseTensor* dOut,
-                  DenseTensor* dX) const {
+                  DenseTensor* dOut UNUSED,
+                  DenseTensor* dX UNUSED) const {
     if (ddOut) {
       auto* d = dev.eigen_device();
       auto ddx = EigenVector<T>::Flatten(
@@ -1479,7 +1479,7 @@ struct ThresholdedReluGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     auto th = static_cast<T>(threshold);
     dx.device(d) = dout * (x > th).template cast<T>();
   }
@@ -1511,7 +1511,7 @@ struct Relu6GradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x UNUSED, Out out, dOut dout, dX dx) const {
     float threshold = 6;
     dx.device(d) =
         dout * ((out > static_cast<T>(0)) * (out < static_cast<T>(threshold)))
@@ -1540,7 +1540,7 @@ struct TanhShrinkGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = dout * (x.tanh() * x.tanh());
   }
 
@@ -1577,7 +1577,7 @@ struct HardShrinkGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     auto temp1 = x < static_cast<T>(threshold * -1.f);
     auto temp2 = x > static_cast<T>(threshold);
     dx.device(d) = dout * (temp1 || temp2).template cast<T>();
@@ -1615,7 +1615,7 @@ struct SoftShrinkGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     auto lambdaT = static_cast<T>(lambda);
     auto temp1 = (x > lambdaT).template cast<T>();
     auto temp2 = (x < -lambdaT).template cast<T>();
@@ -1673,7 +1673,7 @@ struct ELUGradNegativeAlphaFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     // case 2: alpha < 0
     // dx = dout, if x > 0
     // dx = dout * (out + alpha), if x <=0
@@ -1742,7 +1742,7 @@ struct SiluGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     auto temp1 = static_cast<T>(1) + (-x).exp();  // 1+e^(-x)
     auto temp2 = x * (-x).exp();                  // x*e^(-x)
     dx.device(d) = dout * ((static_cast<T>(1) / temp1) *
@@ -1770,7 +1770,7 @@ struct SoftsignGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) =
         dout * (static_cast<T>(1) / (static_cast<T>(1) + x.abs()).square());
   }
@@ -1947,7 +1947,7 @@ struct LogSigmoidGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     auto temp = (-x).cwiseMax(static_cast<T>(0));  // temp = max(-x, 0)
     dx.device(d) =
         dout * ((-x - temp).exp() / ((-temp).exp() + (-x - temp).exp()));
@@ -1984,7 +1984,7 @@ struct HardSigmoidGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x UNUSED, Out out, dOut dout, dX dx) const {
     dx.device(d) = dout *
                    ((out > static_cast<T>(0)) * (out < static_cast<T>(1)))
                        .template cast<T>() *
@@ -2012,7 +2012,7 @@ struct LogGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = dout * (static_cast<T>(1) / x);
   }
 
@@ -2036,7 +2036,7 @@ struct Log2GradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = dout * static_cast<T>(1) / (x * static_cast<T>(log(2)));
   }
 
@@ -2060,7 +2060,7 @@ struct Log10GradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = dout * static_cast<T>(1) / (x * static_cast<T>(log(10)));
   }
 
@@ -2083,7 +2083,7 @@ struct Log1pGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = dout * (static_cast<T>(1) / (x + static_cast<T>(1)));
   }
 
@@ -2157,7 +2157,7 @@ struct HardSwishGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     auto tmp = ((x + static_cast<T>(offset)) < static_cast<T>(threshold))
                    .template cast<T>();
     dx.device(d) =
@@ -2193,7 +2193,7 @@ struct SwishGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out fake_out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out fake_out UNUSED, dOut dout, dX dx) const {
     float beta = 1.0;
     auto temp1 = static_cast<T>(1) /
                  (static_cast<T>(1) + (static_cast<T>(-beta) * x).exp());
@@ -2229,7 +2229,7 @@ struct PowGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     dx.device(d) = dout * static_cast<T>(factor) *
                    x.pow(static_cast<T>(factor) - static_cast<T>(1));
   }
@@ -2279,7 +2279,8 @@ struct ZeroGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(
+      Device d, X x UNUSED, Out out, dOut dout UNUSED, dX dx) const {
     dx.device(d) = static_cast<T>(0) * out;
   }
 
@@ -2384,7 +2385,7 @@ struct CELUGradFunctor : public BaseActivationFunctor<T> {
             typename Out,
             typename dOut,
             typename dX>
-  void operator()(Device d, X x, Out out, dOut dout, dX dx) const {
+  void operator()(Device d, X x, Out out UNUSED, dOut dout, dX dx) const {
     auto temp_a_pos = static_cast<T>(alpha > 0);
     auto temp_a_neg = static_cast<T>(alpha <= 0);
     auto temp_x_pos = (x > static_cast<T>(0)).template cast<T>();
diff --git a/paddle/phi/kernels/funcs/detection/poly_util.cc b/paddle/phi/kernels/funcs/detection/poly_util.cc
deleted file mode 100644
index fd8037a82eabc..0000000000000
--- a/paddle/phi/kernels/funcs/detection/poly_util.cc
+++ /dev/null
@@ -1,138 +0,0 @@
-/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-#ifndef POLY_UTIL_CC_
-#define POLY_UTIL_CC_
-
-#include "paddle/phi/kernels/funcs/detection/poly_util.h"
-
-namespace phi {
-namespace funcs {
-
-using phi::funcs::gpc_free_polygon;
-using phi::funcs::gpc_polygon_clip;
-
-template <class T>
-void Array2PointVec(const T* box,
-                    const size_t box_size,
-                    std::vector<Point_<T>>* vec) {
-  size_t pts_num = box_size / 2;
-  (*vec).resize(pts_num);
-  for (size_t i = 0; i < pts_num; i++) {
-    (*vec).at(i).x = box[2 * i];
-    (*vec).at(i).y = box[2 * i + 1];
-  }
-}
-
-template <class T>
-void Array2Poly(const T* box,
-                const size_t box_size,
-                phi::funcs::gpc_polygon* poly) {
-  size_t pts_num = box_size / 2;
-  (*poly).num_contours = 1;
-  (*poly).hole = reinterpret_cast<int*>(malloc(sizeof(int)));
-  (*poly).hole[0] = 0;
-  (*poly).contour =
-      (phi::funcs::gpc_vertex_list*)malloc(sizeof(phi::funcs::gpc_vertex_list));
-  (*poly).contour->num_vertices = pts_num;
-  (*poly).contour->vertex =
-      (phi::funcs::gpc_vertex*)malloc(sizeof(phi::funcs::gpc_vertex) * pts_num);
-  for (size_t i = 0; i < pts_num; ++i) {
-    (*poly).contour->vertex[i].x = box[2 * i];
-    (*poly).contour->vertex[i].y = box[2 * i + 1];
-  }
-}
-
-template <class T>
-void PointVec2Poly(const std::vector<Point_<T>>& vec,
-                   phi::funcs::gpc_polygon* poly) {
-  int pts_num = vec.size();
-  (*poly).num_contours = 1;
-  (*poly).hole = reinterpret_cast<int*>(malloc(sizeof(int)));
-  (*poly).hole[0] = 0;
-  (*poly).contour =
-      (phi::funcs::gpc_vertex_list*)malloc(sizeof(phi::funcs::gpc_vertex_list));
-  (*poly).contour->num_vertices = pts_num;
-  (*poly).contour->vertex =
-      (phi::funcs::gpc_vertex*)malloc(sizeof(phi::funcs::gpc_vertex) * pts_num);
-  for (size_t i = 0; i < pts_num; ++i) {
-    (*poly).contour->vertex[i].x = vec[i].x;
-    (*poly).contour->vertex[i].y = vec[i].y;
-  }
-}
-
-template <class T>
-void Poly2PointVec(const phi::funcs::gpc_vertex_list& contour,
-                   std::vector<Point_<T>>* vec) {
-  int pts_num = contour.num_vertices;
-  (*vec).resize(pts_num);
-  for (int i = 0; i < pts_num; i++) {
-    (*vec).at(i).x = contour.vertex[i].x;
-    (*vec).at(i).y = contour.vertex[i].y;
-  }
-}
-
-template <class T>
-T GetContourArea(const std::vector<Point_<T>>& vec) {
-  size_t pts_num = vec.size();
-  if (pts_num < 3) return T(0.);
-  T area = T(0.);
-  for (size_t i = 0; i < pts_num; ++i) {
-    area += vec[i].x * vec[(i + 1) % pts_num].y -
-            vec[i].y * vec[(i + 1) % pts_num].x;
-  }
-  return std::fabs(area / 2.0);
-}
-
-template <class T>
-T PolyArea(const T* box, const size_t box_size, const bool normalized) {
-  // If coordinate values are is invalid
-  // if area size <= 0,  return 0.
-  std::vector<Point_<T>> vec;
-  Array2PointVec<T>(box, box_size, &vec);
-  return GetContourArea<T>(vec);
-}
-
-template <class T>
-T PolyOverlapArea(const T* box1,
-                  const T* box2,
-                  const size_t box_size,
-                  const bool normalized) {
-  phi::funcs::gpc_polygon poly1;
-  phi::funcs::gpc_polygon poly2;
-  Array2Poly<T>(box1, box_size, &poly1);
-  Array2Poly<T>(box2, box_size, &poly2);
-  phi::funcs::gpc_polygon respoly;
-  phi::funcs::gpc_op op = phi::funcs::GPC_INT;
-  phi::funcs::gpc_polygon_clip(op, &poly2, &poly1, &respoly);
-
-  T inter_area = T(0.);
-  int contour_num = respoly.num_contours;
-  for (int i = 0; i < contour_num; ++i) {
-    std::vector<Point_<T>> resvec;
-    Poly2PointVec<T>(respoly.contour[i], &resvec);
-    // inter_area += std::fabs(cv::contourArea(resvec)) + 0.5f *
-    // (cv::arcLength(resvec, true));
-    inter_area += GetContourArea<T>(resvec);
-  }
-
-  phi::funcs::gpc_free_polygon(&poly1);
-  phi::funcs::gpc_free_polygon(&poly2);
-  phi::funcs::gpc_free_polygon(&respoly);
-  return inter_area;
-}
-
-}  // namespace funcs
-}  // namespace phi
-
-#endif
diff --git a/paddle/phi/kernels/funcs/detection/poly_util.h b/paddle/phi/kernels/funcs/detection/poly_util.h
index 6d527d2d95f9e..608f373f3d6a3 100644
--- a/paddle/phi/kernels/funcs/detection/poly_util.h
+++ b/paddle/phi/kernels/funcs/detection/poly_util.h
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 #pragma once
 
+#ifndef POLY_UTIL_H_
+#define POLY_UTIL_H_
+
 #include <vector>
 
 #include "paddle/phi/kernels/funcs/gpc.h"
@@ -20,6 +23,9 @@ limitations under the License. */
 namespace phi {
 namespace funcs {
 
+using phi::funcs::gpc_free_polygon;
+using phi::funcs::gpc_polygon_clip;
+
 template <class T>
 class Point_ {
  public:
@@ -43,33 +49,114 @@ class Point_ {
 template <class T>
 void Array2PointVec(const T* box,
                     const size_t box_size,
-                    std::vector<Point_<T>>* vec);
+                    std::vector<Point_<T>>* vec) {
+  size_t pts_num = box_size / 2;
+  (*vec).resize(pts_num);
+  for (size_t i = 0; i < pts_num; i++) {
+    (*vec).at(i).x = box[2 * i];
+    (*vec).at(i).y = box[2 * i + 1];
+  }
+}
 
 template <class T>
 void Array2Poly(const T* box,
                 const size_t box_size,
-                phi::funcs::gpc_polygon* poly);
+                phi::funcs::gpc_polygon* poly) {
+  size_t pts_num = box_size / 2;
+  (*poly).num_contours = 1;
+  (*poly).hole = reinterpret_cast<int*>(malloc(sizeof(int)));
+  (*poly).hole[0] = 0;
+  (*poly).contour =
+      (phi::funcs::gpc_vertex_list*)malloc(sizeof(phi::funcs::gpc_vertex_list));
+  (*poly).contour->num_vertices = pts_num;
+  (*poly).contour->vertex =
+      (phi::funcs::gpc_vertex*)malloc(sizeof(phi::funcs::gpc_vertex) * pts_num);
+  for (size_t i = 0; i < pts_num; ++i) {
+    (*poly).contour->vertex[i].x = box[2 * i];
+    (*poly).contour->vertex[i].y = box[2 * i + 1];
+  }
+}
 
 template <class T>
 void PointVec2Poly(const std::vector<Point_<T>>& vec,
-                   phi::funcs::gpc_polygon* poly);
+                   phi::funcs::gpc_polygon* poly) {
+  int pts_num = vec.size();
+  (*poly).num_contours = 1;
+  (*poly).hole = reinterpret_cast<int*>(malloc(sizeof(int)));
+  (*poly).hole[0] = 0;
+  (*poly).contour =
+      (phi::funcs::gpc_vertex_list*)malloc(sizeof(phi::funcs::gpc_vertex_list));
+  (*poly).contour->num_vertices = pts_num;
+  (*poly).contour->vertex =
+      (phi::funcs::gpc_vertex*)malloc(sizeof(phi::funcs::gpc_vertex) * pts_num);
+  for (size_t i = 0; i < pts_num; ++i) {
+    (*poly).contour->vertex[i].x = vec[i].x;
+    (*poly).contour->vertex[i].y = vec[i].y;
+  }
+}
 
 template <class T>
 void Poly2PointVec(const phi::funcs::gpc_vertex_list& contour,
-                   std::vector<Point_<T>>* vec);
+                   std::vector<Point_<T>>* vec) {
+  int pts_num = contour.num_vertices;
+  (*vec).resize(pts_num);
+  for (int i = 0; i < pts_num; i++) {
+    (*vec).at(i).x = contour.vertex[i].x;
+    (*vec).at(i).y = contour.vertex[i].y;
+  }
+}
 
 template <class T>
-T GetContourArea(const std::vector<Point_<T>>& vec);
+T GetContourArea(const std::vector<Point_<T>>& vec) {
+  size_t pts_num = vec.size();
+  if (pts_num < 3) return T(0.);
+  T area = T(0.);
+  for (size_t i = 0; i < pts_num; ++i) {
+    area += vec[i].x * vec[(i + 1) % pts_num].y -
+            vec[i].y * vec[(i + 1) % pts_num].x;
+  }
+  return std::fabs(area / 2.0);
+}
 
 template <class T>
-T PolyArea(const T* box, const size_t box_size, const bool normalized);
+T PolyArea(const T* box, const size_t box_size, const bool normalized UNUSED) {
+  // If coordinate values are is invalid
+  // if area size <= 0,  return 0.
+  std::vector<Point_<T>> vec;
+  Array2PointVec<T>(box, box_size, &vec);
+  return GetContourArea<T>(vec);
+}
 
 template <class T>
 T PolyOverlapArea(const T* box1,
                   const T* box2,
                   const size_t box_size,
-                  const bool normalized);
+                  const bool normalized UNUSED) {
+  phi::funcs::gpc_polygon poly1;
+  phi::funcs::gpc_polygon poly2;
+  Array2Poly<T>(box1, box_size, &poly1);
+  Array2Poly<T>(box2, box_size, &poly2);
+  phi::funcs::gpc_polygon respoly;
+  phi::funcs::gpc_op op = phi::funcs::GPC_INT;
+  phi::funcs::gpc_polygon_clip(op, &poly2, &poly1, &respoly);
+
+  T inter_area = T(0.);
+  int contour_num = respoly.num_contours;
+  for (int i = 0; i < contour_num; ++i) {
+    std::vector<Point_<T>> resvec;
+    Poly2PointVec<T>(respoly.contour[i], &resvec);
+    // inter_area += std::fabs(cv::contourArea(resvec)) + 0.5f *
+    // (cv::arcLength(resvec, true));
+    inter_area += GetContourArea<T>(resvec);
+  }
+
+  phi::funcs::gpc_free_polygon(&poly1);
+  phi::funcs::gpc_free_polygon(&poly2);
+  phi::funcs::gpc_free_polygon(&respoly);
+  return inter_area;
+}
+
 }  // namespace funcs
 }  // namespace phi
 
-#include "paddle/phi/kernels/funcs/detection/poly_util.cc"
+#endif
diff --git a/paddle/phi/kernels/legacy/cpu/uniform_kernel.cc b/paddle/phi/kernels/legacy/cpu/uniform_kernel.cc
index ecea86874a7a2..3aa697b2409ee 100644
--- a/paddle/phi/kernels/legacy/cpu/uniform_kernel.cc
+++ b/paddle/phi/kernels/legacy/cpu/uniform_kernel.cc
@@ -20,7 +20,7 @@ namespace phi {
 template <typename T, typename Context>
 void UniformRawKernel(const Context &dev_ctx,
                       const IntArray &shape,
-                      DataType dtype,
+                      DataType dtype UNUSED,
                       const Scalar &min,
                       const Scalar &max,
                       int seed,

From 640cff0a5c72e2a5e525d48ea83cd48b3048bbe9 Mon Sep 17 00:00:00 2001
From: Difer <707065510@qq.com>
Date: Tue, 16 May 2023 19:31:07 +0800
Subject: [PATCH 367/405] =?UTF-8?q?=E3=80=90Hackathon=20No57=E3=80=91add?=
 =?UTF-8?q?=20bf16=20for=20mode=20(#53195)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add bf16 for mode

* remove random seed 666

* try to fix op_type error

* test for me

* try to fix op_type

* fix redundancy code

* add fp,bf for lastdim

* fix some error

* simplify code

* fix shape error

* optype error

* fix skipif bf16
---
 paddle/phi/kernels/gpu/mode_grad_kernel.cu    |   6 +-
 paddle/phi/kernels/gpu/mode_kernel.cu         |  14 +-
 .../fluid/tests/unittests/test_mode_op.py     | 133 ++++++++++++++----
 3 files changed, 125 insertions(+), 28 deletions(-)

diff --git a/paddle/phi/kernels/gpu/mode_grad_kernel.cu b/paddle/phi/kernels/gpu/mode_grad_kernel.cu
index e297eb88d0ca0..e0fe5a3d0abef 100644
--- a/paddle/phi/kernels/gpu/mode_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/mode_grad_kernel.cu
@@ -15,6 +15,8 @@
 #include "paddle/phi/kernels/mode_grad_kernel.h"
 
 #include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/common/bfloat16.h"
+#include "paddle/phi/common/float16.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
 #include "paddle/phi/kernels/funcs/mode.h"
@@ -89,4 +91,6 @@ PD_REGISTER_KERNEL(mode_grad,
                    float,
                    double,
                    int32_t,
-                   int64_t) {}
+                   int64_t,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/paddle/phi/kernels/gpu/mode_kernel.cu b/paddle/phi/kernels/gpu/mode_kernel.cu
index c834d87aca9cb..ed598b2e75d84 100644
--- a/paddle/phi/kernels/gpu/mode_kernel.cu
+++ b/paddle/phi/kernels/gpu/mode_kernel.cu
@@ -15,6 +15,8 @@
 #include "paddle/phi/kernels/mode_kernel.h"
 
 #include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/common/bfloat16.h"
+#include "paddle/phi/common/float16.h"
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/funcs/math_function.h"
 #include "paddle/phi/kernels/funcs/mode.h"
@@ -129,7 +131,15 @@ void ModeKernel(const Context& dev_ctx,
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(
-    mode, GPU, ALL_LAYOUT, phi::ModeKernel, float, double, int32_t, int64_t) {
+PD_REGISTER_KERNEL(mode,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::ModeKernel,
+                   float,
+                   double,
+                   int32_t,
+                   int64_t,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {
   kernel->OutputAt(1).SetDataType(phi::DataType::INT64);
 }
diff --git a/python/paddle/fluid/tests/unittests/test_mode_op.py b/python/paddle/fluid/tests/unittests/test_mode_op.py
index eaf9c39631c97..15c376f57abb6 100644
--- a/python/paddle/fluid/tests/unittests/test_mode_op.py
+++ b/python/paddle/fluid/tests/unittests/test_mode_op.py
@@ -15,10 +15,15 @@
 import unittest
 
 import numpy as np
-from eager_op_test import OpTest
+from eager_op_test import (
+    OpTest,
+    convert_float_to_uint16,
+    convert_uint16_to_float,
+)
 
 import paddle
 from paddle import fluid
+from paddle.fluid import core
 
 
 def _mode1D(a):
@@ -34,7 +39,7 @@ def _mode1D(a):
                 mode = sorted_array[i]
                 index = sorted_inds[i]
                 max_freq = cur_freq
-        cur_freq = 0
+            cur_freq = 0
     return mode, index
 
 
@@ -59,51 +64,129 @@ def cal_mode(a, axis, keepdim=False):
 class TestModeOp(OpTest):
     def init_args(self):
         self.axis = 1
+        self.input_shape = (2, 64, 1)
+
+    def init_input_data(self):
+        self.input_data = np.random.rand(*self.input_shape).astype(self.dtype)
+        self.inputs = {'X': self.input_data}
+
+    def init_dtype(self):
+        self.dtype = np.float64
 
     def setUp(self):
         self.op_type = "mode"
         self.python_api = paddle.mode
-        self.dtype = np.float64
-        np.random.seed(666)
-        self.input_data = np.random.rand(2, 64, 1)
+        self.init_dtype()
         self.init_args()
-        self.inputs = {'X': self.input_data}
+        self.init_input_data()
         self.attrs = {'axis': self.axis}
         output, indices = cal_mode(self.input_data, axis=self.axis)
         self.outputs = {'Out': output, 'Indices': indices}
 
+    def init_numeric_grads(self):
+        if self.axis < 0:
+            axis = len(self.input_data.shape) + self.axis
+        else:
+            axis = self.axis
+        if self.dtype == np.float64:
+            dtype = np.float64
+        else:
+            dtype = np.float32
+        grad = np.zeros(self.input_data.shape).astype(dtype)
+        in_dims = list(range(grad.ndim))
+        if axis == len(self.input_data.shape) - 1:
+            a_view = grad
+        else:
+            a_view = np.transpose(
+                grad,
+                in_dims[:axis] + in_dims[axis + 1 :] + [axis],
+            )
+        idx = np.array(self.outputs['Indices']).flatten()
+        inds = np.ndindex(a_view.shape[:-1])
+        for i, ind in enumerate(inds):
+            a_view[ind][idx[i]] = 1 / np.prod(self.outputs['Indices'].shape)
+        if axis == len(self.input_data.shape) - 1:
+            grad = a_view
+        else:
+            grad = np.transpose(
+                a_view,
+                in_dims[:axis] + in_dims[-1:] + in_dims[axis:-1],
+            )
+        return grad
+
     def test_check_output(self):
         paddle.enable_static()
         self.check_output()
 
     def test_check_grad(self):
         paddle.enable_static()
-        self.check_grad({'X'}, 'Out')
+        grad = self.init_numeric_grads()
+        self.check_grad({'X'}, 'Out', user_defined_grads=[grad])
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda(), "core is not compiled with CUDA"
+)
+class TestModeFP16Op(TestModeOp):
+    def init_dtype(self):
+        self.dtype = np.float16
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA and not support the bfloat16",
+)
+class TestModeBF16Op(TestModeOp):
+    def init_dtype(self):
+        self.dtype = np.uint16
+
+    def init_input_data(self):
+        self.input_data = np.random.rand(*self.input_shape).astype(np.float32)
+        self.input_data = convert_uint16_to_float(
+            convert_float_to_uint16(self.input_data)
+        )
+        self.inputs = {'X': convert_float_to_uint16(self.input_data)}
+
+    def test_check_output(self):
+        place = core.CUDAPlace(0)
+        paddle.enable_static()
+        if core.is_bfloat16_supported(place):
+            self.check_output_with_place(place)
+
+    def test_check_grad(self):
+        place = core.CUDAPlace(0)
+        paddle.enable_static()
+        grad = self.init_numeric_grads()
 
+        if core.is_bfloat16_supported(place):
+            self.check_grad_with_place(
+                place, {'X'}, 'Out', user_defined_grads=[grad]
+            )
 
-class TestModeOpLastdim(OpTest):
+
+class TestModeOpLastdim(TestModeOp):
     def init_args(self):
         self.axis = -1
+        self.input_shape = (2, 1, 1, 2, 30)
 
-    def setUp(self):
-        self.op_type = "mode"
-        self.python_api = paddle.mode
-        self.dtype = np.float64
-        np.random.seed(666)
-        self.input_data = np.random.rand(2, 1, 1, 2, 30)
-        self.init_args()
-        self.inputs = {'X': self.input_data}
-        self.attrs = {'axis': self.axis}
-        output, indices = cal_mode(self.input_data, axis=self.axis)
-        self.outputs = {'Out': output, 'Indices': indices}
 
-    def test_check_output(self):
-        paddle.enable_static()
-        self.check_output()
+@unittest.skipIf(
+    not core.is_compiled_with_cuda(), "core is not compiled with CUDA"
+)
+class TestModeFP16OpLastdim(TestModeFP16Op):
+    def init_args(self):
+        self.axis = -1
+        self.input_shape = (2, 1, 1, 2, 30)
 
-    def test_check_grad(self):
-        paddle.enable_static()
-        self.check_grad({'X'}, 'Out')
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda(), "core is not compiled with CUDA"
+)
+class TestModeBF16OpLastdim(TestModeBF16Op):
+    def init_args(self):
+        self.axis = -1
+        self.input_shape = (2, 1, 1, 2, 30)
 
 
 class TestModeOpKernels(unittest.TestCase):

From 50f0acc01879563c3a27ab04331d369acdc2f9ff Mon Sep 17 00:00:00 2001
From: zhwesky2010 <1183042833@qq.com>
Date: Tue, 16 May 2023 19:45:15 +0800
Subject: [PATCH 368/405] [Zero-Dim] update 0d tensor api en doc,
 test=document_fix (#53823)

---
 .../auto_parallel/operators/common.py         |  2 +-
 .../communication/stream/all_gather.py        |  2 +-
 .../communication/stream/all_to_all.py        |  4 +-
 .../communication/stream/scatter.py           |  2 +-
 python/paddle/distribution/bernoulli.py       | 28 +++---
 python/paddle/distribution/beta.py            |  6 +-
 python/paddle/distribution/cauchy.py          | 14 +--
 python/paddle/distribution/dirichlet.py       |  4 +-
 python/paddle/distribution/geometric.py       | 32 +++----
 python/paddle/distribution/gumbel.py          |  2 +-
 python/paddle/distribution/independent.py     |  4 +-
 python/paddle/distribution/kl.py              |  2 +-
 python/paddle/distribution/laplace.py         | 68 +++++++-------
 python/paddle/distribution/lognormal.py       |  4 +-
 python/paddle/distribution/normal.py          |  4 +-
 python/paddle/distribution/transform.py       |  8 +-
 .../distribution/transformed_distribution.py  |  4 +-
 python/paddle/distribution/uniform.py         |  2 +-
 python/paddle/fluid/backward.py               |  2 +-
 .../fluid/dygraph/learning_rate_scheduler.py  |  6 +-
 python/paddle/fluid/dygraph/math_op_patch.py  |  2 +-
 .../fluid/dygraph/tensor_patch_methods.py     | 10 +-
 python/paddle/fluid/framework.py              |  2 +-
 .../fluid/tests/unittests/test_numel_op.py    |  1 -
 .../fluid/tests/unittests/test_size_op.py     |  1 -
 .../paddle/jit/dy2static/convert_operators.py |  2 +-
 python/paddle/metric/metrics.py               |  2 +-
 python/paddle/nn/functional/loss.py           | 92 +++++++++----------
 python/paddle/nn/layer/loss.py                | 88 +++++++++---------
 python/paddle/static/nn/metric.py             |  6 +-
 python/paddle/tensor/creation.py              |  5 +-
 python/paddle/tensor/linalg.py                | 60 ++++++------
 python/paddle/tensor/logic.py                 | 21 ++---
 python/paddle/tensor/manipulation.py          | 48 +++++-----
 python/paddle/tensor/math.py                  | 72 +++++++--------
 python/paddle/tensor/stat.py                  | 19 ++--
 python/paddle/tensor/to_string.py             |  5 +-
 .../utils/cpp_extension/extension_utils.py    |  2 -
 38 files changed, 315 insertions(+), 323 deletions(-)

diff --git a/python/paddle/distributed/auto_parallel/operators/common.py b/python/paddle/distributed/auto_parallel/operators/common.py
index 987c533a012eb..5d7ff78fa2c53 100644
--- a/python/paddle/distributed/auto_parallel/operators/common.py
+++ b/python/paddle/distributed/auto_parallel/operators/common.py
@@ -393,7 +393,7 @@ def get_data_parallel_group(dist_ctx, op, act_grad_names, rank):
 
     for var_name in act_grad_names:
         var_dim_mapping = op_dist_attr.get_input_dims_mapping(var_name)
-        # consider that the variable's shape is [], which is 0D
+        # consider that the variable's shape is [], which is 0-D
         # TODO utilize the batch_dim attr instead of "0" in future
         batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1
 
diff --git a/python/paddle/distributed/communication/stream/all_gather.py b/python/paddle/distributed/communication/stream/all_gather.py
index 69d9c5d52e080..9f4b19fd5cee5 100644
--- a/python/paddle/distributed/communication/stream/all_gather.py
+++ b/python/paddle/distributed/communication/stream/all_gather.py
@@ -108,7 +108,7 @@ def _all_gather_in_static_mode(tensor_list, tensor, group, sync_op):
         },
     )
     tensor_list.clear()
-    # 0D use stack/unstack while others use concat/split
+    # 0-D use stack/unstack while others use concat/split
     if len(tensor.shape) == 0:
         tensor_list.extend(paddle.unstack(out, 0))
     else:
diff --git a/python/paddle/distributed/communication/stream/all_to_all.py b/python/paddle/distributed/communication/stream/all_to_all.py
index 38b1d2fcb3e82..df9c72c1da44e 100644
--- a/python/paddle/distributed/communication/stream/all_to_all.py
+++ b/python/paddle/distributed/communication/stream/all_to_all.py
@@ -78,7 +78,7 @@ def _all_to_all_in_static_mode(
     if isinstance(in_tensor_or_tensor_list, list):
         if len(in_tensor_or_tensor_list) == 0:
             raise RuntimeError("The input tensor_list should not be empty.")
-        # 0D use stack/unstack while others use concat/split
+        # 0-D use stack/unstack while others use concat/split
         if len(in_tensor_or_tensor_list[0].shape) == 0:
             in_tensor = paddle.stack(in_tensor_or_tensor_list, axis=0)
         else:
@@ -115,7 +115,7 @@ def _all_to_all_in_static_mode(
     if isinstance(out_tensor_or_tensor_list, list):
         if not sync_op:
             dist.wait(out_tensor, use_calc_stream=False)
-        # 0D use stack/unstack while others use concat/split
+        # 0-D use stack/unstack while others use concat/split
         if len(in_tensor_or_tensor_list[0].shape) == 0:
             out_tensor_or_tensor_list.extend(paddle.unstack(out_tensor, 0))
         else:
diff --git a/python/paddle/distributed/communication/stream/scatter.py b/python/paddle/distributed/communication/stream/scatter.py
index c112516a1fc10..13f9c3ecf641e 100644
--- a/python/paddle/distributed/communication/stream/scatter.py
+++ b/python/paddle/distributed/communication/stream/scatter.py
@@ -91,7 +91,7 @@ def _scatter_in_static_mode(
                 )
         else:
             tensor_list = [tensor for _ in range(nranks)]
-        # 0D use stack/unstack while others use concat/split
+        # 0-D use stack/unstack while others use concat/split
         if len(tensor_list[0].shape) == 0:
             input_tensor = paddle.stack(tensor_list, axis=0)
         else:
diff --git a/python/paddle/distribution/bernoulli.py b/python/paddle/distribution/bernoulli.py
index d6c6551b0c5ce..e961ee9d01868 100644
--- a/python/paddle/distribution/bernoulli.py
+++ b/python/paddle/distribution/bernoulli.py
@@ -79,16 +79,16 @@ class Bernoulli(exponential_family.ExponentialFamily):
             rv = Bernoulli(probs=0.3)
 
             print(rv.mean)
-            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-            #        [0.30000001])
+            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        0.30000001)
 
             print(rv.variance)
-            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-            #        [0.21000001])
+            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        0.21000001)
 
             print(rv.entropy())
-            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-            #        [0.61086434])
+            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        0.61086434)
     """
 
     def __init__(self, probs, name=None):
@@ -247,12 +247,12 @@ def rsample(self, shape, temperature=1.0):
 
                 # The smaller the `temperature`, the distribution of `rsample` closer to `sample`, with `probs` of 0.3.
                 print(paddle.nn.functional.sigmoid(rv.rsample([1000, ], temperature=1.0)).sum())
-                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                #        [361.06829834])
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        361.06829834)
 
                 print(paddle.nn.functional.sigmoid(rv.rsample([1000, ], temperature=0.1)).sum())
-                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                #        [288.66418457])
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        288.66418457)
         """
         name = self.name + '_rsample'
         if not _non_static_mode():
@@ -420,8 +420,8 @@ def entropy(self):
 
                 rv = Bernoulli(0.3)
                 print(rv.entropy())
-                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                #        [0.61086434])
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        0.61086434)
         """
         name = self.name + '_entropy'
 
@@ -455,8 +455,8 @@ def kl_divergence(self, other):
                 rv_other = Bernoulli(0.7)
 
                 print(rv.kl_divergence(rv_other))
-                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                #        [0.33891910])
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        0.33891910)
         """
         name = self.name + '_kl_divergence'
         if not _non_static_mode():
diff --git a/python/paddle/distribution/beta.py b/python/paddle/distribution/beta.py
index 07cbf9155c701..ebf373bf6b115 100644
--- a/python/paddle/distribution/beta.py
+++ b/python/paddle/distribution/beta.py
@@ -61,13 +61,13 @@ class Beta(exponential_family.ExponentialFamily):
             beta = paddle.distribution.Beta(alpha=0.5, beta=0.5)
             print(beta.mean)
             # Tensor(shape=[], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
-            #        [0.50000000])
+            #        0.50000000)
             print(beta.variance)
             # Tensor(shape=[], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
-            #        [0.12500000])
+            #        0.12500000)
             print(beta.entropy())
             # Tensor(shape=[], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
-            #        [0.12500000])
+            #        0.12500000)
 
             # tensor input with broadcast
             beta = paddle.distribution.Beta(alpha=paddle.to_tensor([0.2, 0.4]), beta=0.6)
diff --git a/python/paddle/distribution/cauchy.py b/python/paddle/distribution/cauchy.py
index dde50d8a2287e..e149bf0fd8939 100644
--- a/python/paddle/distribution/cauchy.py
+++ b/python/paddle/distribution/cauchy.py
@@ -45,7 +45,7 @@ class Cauchy(distribution.Distribution):
             # init Cauchy with float
             rv = Cauchy(loc=0.1, scale=1.2)
             print(rv.entropy())
-            # Tensor(shape=1, dtype=float32, place=Place(cpu), stop_gradient=True,
+            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
             #        2.71334577)
 
             # init Cauchy with N-Dim tensor
@@ -228,8 +228,8 @@ def prob(self, value):
                 # init Cauchy with float
                 rv = Cauchy(loc=0.1, scale=1.2)
                 print(rv.prob(paddle.to_tensor(1.5)))
-                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                #        [0.11234467])
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        0.11234467)
 
                 # broadcast to value
                 rv = Cauchy(loc=0.1, scale=1.2)
@@ -277,8 +277,8 @@ def log_prob(self, value):
                 # init Cauchy with float
                 rv = Cauchy(loc=0.1, scale=1.2)
                 print(rv.log_prob(paddle.to_tensor(1.5)))
-                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                #        [-2.18618369])
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        -2.18618369)
 
                 # broadcast to value
                 rv = Cauchy(loc=0.1, scale=1.2)
@@ -344,8 +344,8 @@ def cdf(self, value):
                 # init Cauchy with float
                 rv = Cauchy(loc=0.1, scale=1.2)
                 print(rv.cdf(paddle.to_tensor(1.5)))
-                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                #        [0.77443725])
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        0.77443725)
 
                 # broadcast to value
                 rv = Cauchy(loc=0.1, scale=1.2)
diff --git a/python/paddle/distribution/dirichlet.py b/python/paddle/distribution/dirichlet.py
index e4773af7109a6..ca55285540d79 100644
--- a/python/paddle/distribution/dirichlet.py
+++ b/python/paddle/distribution/dirichlet.py
@@ -63,10 +63,10 @@ class Dirichlet(exponential_family.ExponentialFamily):
 
             print(dirichlet.entropy())
             # Tensor(shape=[], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
-            #        [-1.24434423])
+            #        -1.24434423)
             print(dirichlet.prob(paddle.to_tensor([.3, .5, .6])))
             # Tensor(shape=[], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
-            #        [10.80000114])
+            #        10.80000114)
 
     """
 
diff --git a/python/paddle/distribution/geometric.py b/python/paddle/distribution/geometric.py
index 599b78c1f9e42..d338ee6f35ac7 100644
--- a/python/paddle/distribution/geometric.py
+++ b/python/paddle/distribution/geometric.py
@@ -55,16 +55,16 @@ class Geometric(distribution.Distribution):
             geom = Geometric(0.5)
 
             geom.mean
-            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-            #        [2.])
+            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        2.)
 
             geom.variance
-            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-            #        [2.])
+            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        2.)
 
             geom.stddev
-            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-            #        [1.41421354])
+            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        1.41421354)
     """
 
     def __init__(self, probs):
@@ -145,8 +145,8 @@ def pmf(self, k):
 
                 geom = Geometric(0.5)
                 geom.pmf(2)
-                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                #        [0.25000000])
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        0.25000000)
         """
         if isinstance(k, (numbers.Integral, framework.Variable)):
             return paddle.pow((1.0 - self.probs), k - 1.0) * self.probs
@@ -176,8 +176,8 @@ def log_pmf(self, k):
 
                 geom = Geometric(0.5)
                 geom.log_pmf(2)
-                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                #        [-1.38629436])
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        -1.38629436)
         """
         if isinstance(k, (numbers.Integral, framework.Variable)):
             return paddle.log(self.pmf(k))
@@ -266,8 +266,8 @@ def entropy(self):
 
                 geom = Geometric(0.5)
                 geom.entropy()
-                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                #        [1.38629436])
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        1.38629436)
         """
         x = (1.0 - self.probs) * paddle.log(1.0 - self.probs)
         y = self.probs * paddle.log(self.probs)
@@ -296,8 +296,8 @@ def cdf(self, k):
 
                 geom = Geometric(0.5)
                 geom.cdf(4)
-                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                #        [0.93750000])
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        0.93750000)
         """
         if isinstance(k, (numbers.Integral, framework.Variable)):
             return 1.0 - paddle.pow((1.0 - self.probs), k)
@@ -329,8 +329,8 @@ def kl_divergence(self, other):
                 geom_p = Geometric(0.5)
                 geom_q = Geometric(0.1)
                 geom_p.kl_divergence(geom_q)
-                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                #        [0.51082563])
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        0.51082563)
         """
         if isinstance(other, Geometric):
             p, q = self.probs, other.probs
diff --git a/python/paddle/distribution/gumbel.py b/python/paddle/distribution/gumbel.py
index dc0ad391685a5..7513046212c56 100644
--- a/python/paddle/distribution/gumbel.py
+++ b/python/paddle/distribution/gumbel.py
@@ -61,7 +61,7 @@ class Gumbel(TransformedDistribution):
           dist.cdf(value)
           # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True, [0.54523915])
           dist.entropy()
-          # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True, [1.57721567])
+          # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True, [1.57721567])
           dist.rsample([2])
           # Tensor(shape=[2, 1], dtype=float32, place=Place(gpu:0), stop_gradient=True, [[0.80463481], [0.91893655]])
 
diff --git a/python/paddle/distribution/independent.py b/python/paddle/distribution/independent.py
index 4119941b94c59..3d180814b4aee 100644
--- a/python/paddle/distribution/independent.py
+++ b/python/paddle/distribution/independent.py
@@ -44,8 +44,8 @@ class Independent(distribution.Distribution):
             print(reinterpreted_beta.batch_shape, reinterpreted_beta.event_shape)
             # () (2,)
             print(reinterpreted_beta.log_prob(paddle.to_tensor([0.2,  0.2])))
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [-0.45687842])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        -0.45687842)
     """
 
     def __init__(self, base, reinterpreted_batch_rank):
diff --git a/python/paddle/distribution/kl.py b/python/paddle/distribution/kl.py
index 2e9b5e878536d..a47eefc7893e1 100644
--- a/python/paddle/distribution/kl.py
+++ b/python/paddle/distribution/kl.py
@@ -60,7 +60,7 @@ def kl_divergence(p, q):
 
             print(paddle.distribution.kl_divergence(p, q))
             # Tensor(shape=[], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
-            #        [0.21193528])
+            #        0.21193528)
 
     """
     return _dispatch(type(p), type(q))(p, q)
diff --git a/python/paddle/distribution/laplace.py b/python/paddle/distribution/laplace.py
index 124da20576a8f..eebf72cc16943 100644
--- a/python/paddle/distribution/laplace.py
+++ b/python/paddle/distribution/laplace.py
@@ -44,12 +44,12 @@ class Laplace(distribution.Distribution):
     Examples:
         .. code-block:: python
 
-                        import paddle
+            import paddle
 
-                        m = paddle.distribution.Laplace(paddle.to_tensor([0.0]), paddle.to_tensor([1.0]))
-                        m.sample()  # Laplace distributed with loc=0, scale=1
-                        # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
-                        # [3.68546247])
+            m = paddle.distribution.Laplace(paddle.to_tensor(0.0), paddle.to_tensor(1.0))
+            m.sample()  # Laplace distributed with loc=0, scale=1
+            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        3.68546247)
 
     """
 
@@ -173,13 +173,13 @@ def log_prob(self, value):
         Examples:
             .. code-block:: python
 
-                            import paddle
+                import paddle
 
-                            m = paddle.distribution.Laplace(paddle.to_tensor([0.0]), paddle.to_tensor([1.0]))
-                            value = paddle.to_tensor([0.1])
-                            m.log_prob(value)
-                            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                            # [-0.79314721])
+                m = paddle.distribution.Laplace(paddle.to_tensor(0.0), paddle.to_tensor(1.0))
+                value = paddle.to_tensor(0.1)
+                m.log_prob(value)
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        -0.79314721)
 
         """
         loc, scale, value = self._validate_value(value)
@@ -205,12 +205,12 @@ def entropy(self):
         Examples:
             .. code-block:: python
 
-                            import paddle
+                import paddle
 
-                            m = paddle.distribution.Laplace(paddle.to_tensor([0.0]), paddle.to_tensor([1.0]))
-                            m.entropy()
-                            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
-                            # [1.69314718])
+                m = paddle.distribution.Laplace(paddle.to_tensor(0.0), paddle.to_tensor(1.0))
+                m.entropy()
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        1.69314718)
         """
         return 1 + paddle.log(2 * self.scale)
 
@@ -236,13 +236,13 @@ def cdf(self, value):
         Examples:
             .. code-block:: python
 
-                            import paddle
+                import paddle
 
-                            m = paddle.distribution.Laplace(paddle.to_tensor([0.0]), paddle.to_tensor([1.0]))
-                            value = paddle.to_tensor([0.1])
-                            m.cdf(value)
-                            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                            # [0.54758132])
+                m = paddle.distribution.Laplace(paddle.to_tensor(0.0), paddle.to_tensor(1.0))
+                value = paddle.to_tensor(0.1)
+                m.cdf(value)
+                # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                #        0.54758132)
         """
         loc, scale, value = self._validate_value(value)
         iterm = (
@@ -277,11 +277,11 @@ def icdf(self, value):
 
                             import paddle
 
-                            m = paddle.distribution.Laplace(paddle.to_tensor([0.0]), paddle.to_tensor([1.0]))
-                            value = paddle.to_tensor([0.1])
+                            m = paddle.distribution.Laplace(paddle.to_tensor(0.0), paddle.to_tensor(1.0))
+                            value = paddle.to_tensor(0.1)
                             m.icdf(value)
-                            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                            # [-1.60943794])
+                            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                            #        -1.60943794)
         """
         loc, scale, value = self._validate_value(value)
         term = value - 0.5
@@ -302,10 +302,10 @@ def sample(self, shape=()):
 
                             import paddle
 
-                            m = paddle.distribution.Laplace(paddle.to_tensor([0.0]), paddle.to_tensor([1.0]))
+                            m = paddle.distribution.Laplace(paddle.to_tensor(0.0), paddle.to_tensor(1.0))
                             m.sample()  # Laplace distributed with loc=0, scale=1
                             # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
-                            # [3.68546247])
+                            #        3.68546247)
         """
         shape = shape if isinstance(shape, tuple) else tuple(shape)
         with paddle.no_grad():
@@ -395,13 +395,13 @@ def kl_divergence(self, other):
         Examples:
             .. code-block:: python
 
-                            import paddle
+                import paddle
 
-                            m1 = paddle.distribution.Laplace(paddle.to_tensor([0.0]), paddle.to_tensor([1.0]))
-                            m2 = paddle.distribution.Laplace(paddle.to_tensor([1.0]), paddle.to_tensor([0.5]))
-                            m1.kl_divergence(m2)
-                            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
-                            # [1.04261160])
+                m1 = paddle.distribution.Laplace(paddle.to_tensor([0.0]), paddle.to_tensor([1.0]))
+                m2 = paddle.distribution.Laplace(paddle.to_tensor([1.0]), paddle.to_tensor([0.5]))
+                m1.kl_divergence(m2)
+                # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
+                # [1.04261160])
         """
 
         var_ratio = other.scale / self.scale
diff --git a/python/paddle/distribution/lognormal.py b/python/paddle/distribution/lognormal.py
index c69a8a6cf9113..f437be3ab8ac5 100644
--- a/python/paddle/distribution/lognormal.py
+++ b/python/paddle/distribution/lognormal.py
@@ -72,13 +72,13 @@ class LogNormal(TransformedDistribution):
           sample = lognormal_a.sample((2, ))
           # a random tensor created by lognormal distribution with shape: [2, 1]
           entropy = lognormal_a.entropy()
-          # [1.4189385] with shape: []
+          # [1.4189385] with shape: [1]
           lp = lognormal_a.log_prob(value_tensor)
           # [-0.72069150] with shape: [1]
           p = lognormal_a.probs(value_tensor)
           # [0.48641577] with shape: [1]
           kl = lognormal_a.kl_divergence(lognormal_b)
-          # [0.34939718] with shape: []
+          # [0.34939718] with shape: [1]
     """
 
     def __init__(self, loc, scale):
diff --git a/python/paddle/distribution/normal.py b/python/paddle/distribution/normal.py
index 5538c41458f04..31a3750a04819 100644
--- a/python/paddle/distribution/normal.py
+++ b/python/paddle/distribution/normal.py
@@ -77,13 +77,13 @@ class Normal(distribution.Distribution):
             sample = normal_a.sample([2])
             # a random tensor created by normal distribution with shape: [2, 1]
             entropy = normal_a.entropy()
-            # [1.4189385] with shape: []
+            # [1.4189385] with shape: [1]
             lp = normal_a.log_prob(value_tensor)
             # [-1.2389386] with shape: [1]
             p = normal_a.probs(value_tensor)
             # [0.28969154] with shape: [1]
             kl = normal_a.kl_divergence(normal_b)
-            # [0.34939718] with shape: []
+            # [0.34939718] with shape: [1]
     """
 
     def __init__(self, loc, scale, name=None):
diff --git a/python/paddle/distribution/transform.py b/python/paddle/distribution/transform.py
index 85575b3c61a15..f1ee702c15b66 100644
--- a/python/paddle/distribution/transform.py
+++ b/python/paddle/distribution/transform.py
@@ -435,8 +435,8 @@ class AffineTransform(Transform):
             # Tensor(shape=[2], dtype=float32, place=Place(gpu:0), stop_gradient=True,
             #        [1., 2.])
             print(affine.forward_log_det_jacobian(x))
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [0.])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        0.)
     """
     _type = Type.BIJECTION
 
@@ -1189,8 +1189,8 @@ class StickBreakingTransform(Transform):
             # Tensor(shape=[3], dtype=float32, place=Place(gpu:0), stop_gradient=True,
             #        [0.99999988, 2.        , 2.99999881])
             print(t.forward_log_det_jacobian(x))
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [-9.10835075])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        -9.10835075)
     """
 
     _type = Type.BIJECTION
diff --git a/python/paddle/distribution/transformed_distribution.py b/python/paddle/distribution/transformed_distribution.py
index 95f7670c563a3..6fa8e2aed482a 100644
--- a/python/paddle/distribution/transformed_distribution.py
+++ b/python/paddle/distribution/transformed_distribution.py
@@ -42,8 +42,8 @@ class TransformedDistribution(distribution.Distribution):
             #        [-0.10697651,  3.33609009, -0.86234951,  5.07457638,  0.75925219,
             #         -4.17087793,  2.22579336, -0.93845034,  0.66054249,  1.50957513])
             print(d.log_prob(paddle.to_tensor(0.5)))
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [-1.64333570])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        -1.64333570)
     """
 
     def __init__(self, base, transforms):
diff --git a/python/paddle/distribution/uniform.py b/python/paddle/distribution/uniform.py
index 1edc9920ac280..8547c6f9900eb 100644
--- a/python/paddle/distribution/uniform.py
+++ b/python/paddle/distribution/uniform.py
@@ -84,7 +84,7 @@ class Uniform(distribution.Distribution):
             sample = uniform.sample([2])
             # a random tensor created by uniform distribution with shape: [2, 1]
             entropy = uniform.entropy()
-            # [0.6931472] with shape: []
+            # [0.6931472] with shape: [1]
             lp = uniform.log_prob(value_tensor)
             # [-0.6931472] with shape: [1]
             p = uniform.probs(value_tensor)
diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py
index f7b8531aee4d9..1635c7d5d211b 100755
--- a/python/paddle/fluid/backward.py
+++ b/python/paddle/fluid/backward.py
@@ -387,7 +387,7 @@ def _create_op_desc_(op_type, inputs, outputs, attrs):
 
 
 def _create_loss_op_desc_(loss):
-    # 0D Tensor or 0-Size Tensor
+    # 0-D Tensor or 0-Size Tensor
     if len(loss.shape) == 0 or 0 in loss.shape:
         create_shape = loss.shape
     else:
diff --git a/python/paddle/fluid/dygraph/learning_rate_scheduler.py b/python/paddle/fluid/dygraph/learning_rate_scheduler.py
index 27840c8b61ad4..9951f5a7c40f4 100644
--- a/python/paddle/fluid/dygraph/learning_rate_scheduler.py
+++ b/python/paddle/fluid/dygraph/learning_rate_scheduler.py
@@ -789,7 +789,7 @@ class ReduceLROnPlateau(LearningRateDecay):
     Reduce learning rate when ``loss`` has stopped descending. Models often benefit from reducing the learning rate
     by 2 to 10 times once model performance has no longer improvement.
 
-    The ``loss`` is the one which has been pass into ``step`` , it must be 1-D Tensor with shape [1]. When ``loss``
+    The ``loss`` is the one which has been pass into ``step`` , it must be 0-D Tensor with shape []. When ``loss``
     stop descending for a ``patience`` number of epochs, the learning rate will be reduced to ``learning_rate * decay_rate`` .
     (Specially, ``mode`` can also be set to ``'max`` , in this case, when ``loss`` stop ascending for a ``patience`` number
     of epochs, the learning rate will be reduced.)
@@ -943,7 +943,7 @@ def step(self, loss):
         Args:
             loss (Variable): A ``Variable`` that will be monitored to determine whether the learning rate will reduce.
                 If it stop descending for a ``patience`` number of epochs, the learning rate will reduce. It should
-                be 1-D Tensor with shape [1].
+                be 0-D Tensor with shape [].
                 Specially, if ``mode`` has been set to ``'max'`` ,  the learning rate will reduce when it stops ascending.
         Returns:
             None
@@ -952,7 +952,7 @@ def step(self, loss):
             Please refer to the example of current LearningRateDecay.
         """
 
-        # loss must be 1-D Tensor with shape [1]
+        # loss.size must be 1
         check_type(loss, 'loss', Variable, 'ReduceLROnPlateau.step')
         assert np.prod(loss.shape) == 1, (
             "The number of elements of loss should be 1, but the current loss.shape is {}, whose number of elements is not 1. "
diff --git a/python/paddle/fluid/dygraph/math_op_patch.py b/python/paddle/fluid/dygraph/math_op_patch.py
index 5d8fa6493ee72..fc26715d7cc4e 100644
--- a/python/paddle/fluid/dygraph/math_op_patch.py
+++ b/python/paddle/fluid/dygraph/math_op_patch.py
@@ -131,7 +131,7 @@ def _int_(var):
         return int(np.array(var).flatten()[0])
 
     def _len_(var):
-        assert var.ndim > 0, "len() of a 0D tensor is wrong"
+        assert var.ndim > 0, "len() of a 0-D tensor is wrong"
         if var.type == core.VarDesc.VarType.VOCAB:
             return len(var.value().get_map_tensor())
         elif var.type == core.VarDesc.VarType.STRINGS:
diff --git a/python/paddle/fluid/dygraph/tensor_patch_methods.py b/python/paddle/fluid/dygraph/tensor_patch_methods.py
index a081c6ca294ee..ad51d8ff31aff 100644
--- a/python/paddle/fluid/dygraph/tensor_patch_methods.py
+++ b/python/paddle/fluid/dygraph/tensor_patch_methods.py
@@ -516,7 +516,7 @@ def grad(self):
                 y = paddle.pow(x, 4.0)
                 y.backward()
                 print("grad of x: {}".format(x.grad))
-                # Tensor(shape=[1], dtype=float32, place=CUDAPlace(0), stop_gradient=False, [500.])
+                # Tensor(shape=[], dtype=float32, place=CUDAPlace(0), stop_gradient=False, 500.)
 
         """
         msg = (
@@ -638,12 +638,12 @@ def __deepcopy__(self, memo):
                 y = copy.deepcopy(x)
 
                 print(x)
-                # Tensor(shape=[1], dtype=float32, place=CPUPlace, stop_gradient=True,
-                #        [2.])
+                # Tensor(shape=[], dtype=float32, place=CPUPlace, stop_gradient=True,
+                #        2.)
 
                 print(y)
-                # Tensor(shape=[1], dtype=float32, place=CPUPlace, stop_gradient=True,
-                #        [2.])
+                # Tensor(shape=[], dtype=float32, place=CPUPlace, stop_gradient=True,
+                #        2.)
 
         """
         if not self.is_leaf:
diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py
index e440cf4c9e623..2ba5b7c0d5ecc 100644
--- a/python/paddle/fluid/framework.py
+++ b/python/paddle/fluid/framework.py
@@ -2476,7 +2476,7 @@ def set_value(self, value, scope=None):
     def size(self):
         """
 
-        Returns the number of elements for current Variable, which is a int64 Variable with shape [1]
+        Returns the number of elements for current Variable, which is a int64 Variable with shape [] .
 
         Returns:
             Variable, the number of elements for current Variable
diff --git a/python/paddle/fluid/tests/unittests/test_numel_op.py b/python/paddle/fluid/tests/unittests/test_numel_op.py
index 81ddbdc45a02f..c841cde6cbb1f 100644
--- a/python/paddle/fluid/tests/unittests/test_numel_op.py
+++ b/python/paddle/fluid/tests/unittests/test_numel_op.py
@@ -120,7 +120,6 @@ def test_numel_static(self):
                 },
                 fetch_list=[out_1, out_2],
             )
-            # TODO(zhouwei): will change shape [1] to [] to support zero-dim
             assert np.array_equal(
                 res_1, np.array(np.size(input_1)).astype("int64")
             )
diff --git a/python/paddle/fluid/tests/unittests/test_size_op.py b/python/paddle/fluid/tests/unittests/test_size_op.py
index 610ee4eaa5f0e..dfff90b742fca 100644
--- a/python/paddle/fluid/tests/unittests/test_size_op.py
+++ b/python/paddle/fluid/tests/unittests/test_size_op.py
@@ -83,7 +83,6 @@ def test_size_static(self):
                 },
                 fetch_list=[out_1, out_2],
             )
-            # TODO(zhouwei): will change shape [1] to [] to support zero-dim
             assert np.array_equal(
                 res_1, np.array(np.size(input_1)).astype("int64")
             )
diff --git a/python/paddle/jit/dy2static/convert_operators.py b/python/paddle/jit/dy2static/convert_operators.py
index 52d6b7cb854d9..b5d5b3da4c3c8 100644
--- a/python/paddle/jit/dy2static/convert_operators.py
+++ b/python/paddle/jit/dy2static/convert_operators.py
@@ -517,7 +517,7 @@ def convert_len(var):
           `shape_op` in var.block.
     """
     if isinstance(var, Variable):
-        assert var.ndim > 0, "len() of a 0D tensor is wrong"
+        assert var.ndim > 0, "len() of a 0-D tensor is wrong"
         if var.type in [
             core.VarDesc.VarType.LOD_TENSOR,
             core.VarDesc.VarType.SELECTED_ROWS,
diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py
index 430b82c07ae0c..bf3f164c33813 100644
--- a/python/paddle/metric/metrics.py
+++ b/python/paddle/metric/metrics.py
@@ -798,7 +798,7 @@ def accuracy(input, label, k=1, correct=None, total=None, name=None):
             predictions = paddle.to_tensor([[0.2, 0.1, 0.4, 0.1, 0.1], [0.2, 0.3, 0.1, 0.15, 0.25]], dtype='float32')
             label = paddle.to_tensor([[2], [0]], dtype="int64")
             result = paddle.metric.accuracy(input=predictions, label=label, k=1)
-            # [0.5]
+            # 0.5
     """
     if label.dtype == paddle.int32:
         label = paddle.cast(label, paddle.int64)
diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py
index f27fa360609af..578a5da900e14 100644
--- a/python/paddle/nn/functional/loss.py
+++ b/python/paddle/nn/functional/loss.py
@@ -61,7 +61,7 @@ def dice_loss(input, label, epsilon=0.00001, name=None):
                              For more information, please refer to :ref:`api_guide_Name`
 
     Returns:
-        Tensor, which shape is [1], data type is the same as `input` .
+        0-D Tensor, which shape is [], data type is the same as `input` .
 
     Example:
         .. code-block:: python
@@ -327,7 +327,7 @@ def npair_loss(anchor, positive, labels, l2_reg=0.002):
 
 
     Returns:
-      A Tensor representing the npair loss, the data type is the same as anchor, the shape is [1].
+      A 0-D Tensor representing the npair loss, the data type is the same as anchor, the shape is [].
 
     Examples:
 
@@ -634,7 +634,7 @@ def binary_cross_entropy(
             input = paddle.to_tensor([0.5, 0.6, 0.7], 'float32')
             label = paddle.to_tensor([1.0, 0.0, 1.0], 'float32')
             output = paddle.nn.functional.binary_cross_entropy(input, label)
-            print(output)  # [0.65537095]
+            print(output)  # 0.65537095
 
     """
     if reduction not in ['sum', 'mean', 'none']:
@@ -774,7 +774,7 @@ def binary_cross_entropy_with_logits(
             logit = paddle.to_tensor([5.0, 1.0, 3.0])
             label = paddle.to_tensor([1.0, 0.0, 1.0])
             output = paddle.nn.functional.binary_cross_entropy_with_logits(logit, label)
-            print(output)  # [0.45618808]
+            print(output)  # 0.45618808
 
     """
     if reduction not in ['sum', 'mean', 'none']:
@@ -1077,7 +1077,7 @@ def smooth_l1_loss(input, label, reduction='mean', delta=1.0, name=None):
             label = paddle.rand([3, 3]).astype('float32')
             output = paddle.nn.functional.smooth_l1_loss(input, label)
             print(output)
-            # [0.068004]
+            # 0.068004
     """
 
     if in_dygraph_mode():
@@ -1147,7 +1147,7 @@ def margin_ranking_loss(
         name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
 
     Returns:
-        Tensor, if :attr:`reduction` is ``'mean'`` or ``'sum'``, the out shape is :math:`[1]`, otherwise the shape is the same as `input` .The same dtype as input tensor.
+        Tensor, if :attr:`reduction` is ``'mean'`` or ``'sum'``, the out shape is :math:`[]`, otherwise the shape is the same as `input` .The same dtype as input tensor.
 
     Examples:
 
@@ -1159,7 +1159,7 @@ def margin_ranking_loss(
             other = paddle.to_tensor([[2, 1], [2, 4]], dtype='float32')
             label = paddle.to_tensor([[1, -1], [-1, -1]], dtype='float32')
             loss = paddle.nn.functional.margin_ranking_loss(input, other, label)
-            print(loss) # [0.75]
+            print(loss) # 0.75
     """
     if reduction not in ['sum', 'mean', 'none']:
         raise ValueError(
@@ -1264,7 +1264,7 @@ def l1_loss(input, label, reduction='mean', name=None):
     Returns:
         Tensor, the L1 Loss of Tensor ``input`` and ``label``.
         If `reduction` is ``'none'``, the shape of output loss is :math:`[N, *]`, the same as ``input`` .
-        If `reduction` is ``'mean'`` or ``'sum'``, the shape of output loss is [1].
+        If `reduction` is ``'mean'`` or ``'sum'``, the shape of output loss is [].
 
     Examples:
         .. code-block:: python
@@ -1276,8 +1276,8 @@ def l1_loss(input, label, reduction='mean', name=None):
 
             l1_loss = paddle.nn.functional.l1_loss(input, label)
             print(l1_loss)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [0.34999999])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        0.34999999)
 
             l1_loss = paddle.nn.functional.l1_loss(input, label, reduction='none')
             print(l1_loss)
@@ -1287,8 +1287,8 @@ def l1_loss(input, label, reduction='mean', name=None):
 
             l1_loss = paddle.nn.functional.l1_loss(input, label, reduction='sum')
             print(l1_loss)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [1.39999998])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        1.39999998)
 
     """
     if reduction not in ['sum', 'mean', 'none']:
@@ -1377,7 +1377,7 @@ def nll_loss(
                 log_out = log_softmax(input)
                 label = paddle.to_tensor([0, 2, 1, 1, 0], "int64")
                 result = nll_loss(log_out, label)
-                print(result) # Tensor(shape=[1], dtype=float32, place=CPUPlace, stop_gradient=True, [1.07202101])
+                print(result) # Tensor(shape=[], dtype=float32, place=CPUPlace, stop_gradient=True, 1.07202101)
     """
     if reduction not in ['sum', 'mean', 'none']:
         raise ValueError(
@@ -1578,9 +1578,9 @@ def kl_div(input, label, reduction='mean', name=None):
 
     If `reduction` is ``'none'``, the output loss is the same shape as the input, and the loss at each point is calculated separately. There is no reduction to the result.
 
-    If `reduction` is ``'mean'``, the output loss is the shape of [1], and the output is the average of all losses.
+    If `reduction` is ``'mean'``, the output loss is the shape of [], and the output is the average of all losses.
 
-    If `reduction` is ``'sum'``, the output loss is the shape of [1], and the output is the sum of all losses.
+    If `reduction` is ``'sum'``, the output loss is the shape of [], and the output is the sum of all losses.
 
     If `reduction` is ``'batchmean'``, the output loss is the shape of [N], N is the batch size, and the output is the sum of all losses divided by the batch size.
 
@@ -1611,17 +1611,17 @@ def kl_div(input, label, reduction='mean', name=None):
             x = paddle.uniform(shape, min=-10, max=10).astype('float32')
             target = paddle.uniform(shape, min=-10, max=10).astype('float32')
 
-            # 'batchmean' reduction, loss shape will be [1]
+            # 'batchmean' reduction, loss shape will be [], who is 0-D Tensor
             pred_loss = F.kl_div(x, target, reduction='batchmean')
-            # shape=[1]
+            # shape=[]
 
-            # 'mean' reduction, loss shape will be [1]
+            # 'mean' reduction, loss shape will be [], who is 0-D Tensor
             pred_loss = F.kl_div(x, target, reduction='mean')
-            # shape=[1]
+            # shape=[]
 
-            # 'sum' reduction, loss shape will be [1]
+            # 'sum' reduction, loss shape will be [], who is 0-D Tensor
             pred_loss = F.kl_div(x, target, reduction='sum')
-            # shape=[1]
+            # shape=[]
 
             # 'none' reduction, loss shape is same with input shape
             pred_loss = F.kl_div(x, target, reduction='none')
@@ -1724,7 +1724,7 @@ def mse_loss(input, label, reduction='mean', name=None):
             label = paddle.to_tensor(1.7)
             output = mse_loss(input, label)
             print(output)
-            # [0.04000002]
+            # 0.04000002
 
     """
 
@@ -1780,7 +1780,7 @@ def ctc_loss(
         norm_by_times (bool, optional): Whether to normalize the gradients by the number of time-step, which is also the sequence's length. There is no need to normalize the gradients if reduction mode is 'mean'. Default: False.
 
     Returns:
-        Tensor, The Connectionist Temporal Classification (CTC) loss between ``log_probs`` and  ``labels``. If attr:`reduction` is ``'none'``, the shape of loss is [batch_size], otherwise, the shape of loss is [1]. Data type is the same as ``log_probs``.
+        Tensor, The Connectionist Temporal Classification (CTC) loss between ``log_probs`` and  ``labels``. If attr:`reduction` is ``'none'``, the shape of loss is [batch_size], otherwise, the shape of loss is []. Data type is the same as ``log_probs``.
 
     Examples:
 
@@ -1834,8 +1834,8 @@ def ctc_loss(
                 blank=0,
                 reduction='mean')
             print(loss)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [1.13760614])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        1.13760614)
 
     """
 
@@ -1929,7 +1929,7 @@ def rnnt_loss(
         name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
 
     Returns:
-        Tensor, The RNN-T loss between ``logprobs`` and  ``labels``. If attr:`reduction` is ``'none'``, the shape of loss is [batch_size], otherwise, the shape of loss is [1]. Data type is the same as ``logprobs``.
+        Tensor, The RNN-T loss between ``logprobs`` and  ``labels``. If attr:`reduction` is ``'none'``, the shape of loss is [batch_size], otherwise, the shape of loss is []. Data type is the same as ``logprobs``.
 
     Examples:
 
@@ -1961,8 +1961,8 @@ def rnnt_loss(
 
             costs = fn(acts, labels, lengths, label_lengths)
             print(costs)
-            # Tensor(shape=[1], dtype=float64, place=Place(gpu:0), stop_gradient=False,
-            #        [4.49566677])
+            # Tensor(shape=[], dtype=float64, place=Place(gpu:0), stop_gradient=False,
+            #        4.49566677)
     """
 
     def warprnnt(
@@ -2078,7 +2078,7 @@ def margin_cross_entropy(
             softmax is shard_softmax when using model parallel, otherwise
             softmax is in the same shape with input logits. If
             ``reduction == None``, the shape of loss is ``[N, 1]``, otherwise
-            the shape is ``[1]``.
+            the shape is ``[]``.
 
     Examples:
 
@@ -2633,8 +2633,8 @@ def cross_entropy(
                                         input,
                                         label)
             print(dy_ret)
-            # Tensor(shape=[1], dtype=float64, place=Place(gpu:0), stop_gradient=True,
-            #        [5.34043430])
+            # Tensor(shape=[], dtype=float64, place=Place(gpu:0), stop_gradient=True,
+            #        5.34043430)
 
         .. code-block:: python
 
@@ -2659,8 +2659,8 @@ def cross_entropy(
                                                                     weight=weight,
                                                                     reduction=reduction)
             print(paddle_loss_mean)
-            # Tensor(shape=[1], dtype=float64, place=Place(gpu:0), stop_gradient=True,
-            #        [1.11043464])
+            # Tensor(shape=[], dtype=float64, place=Place(gpu:0), stop_gradient=True,
+            #        1.11043464)
 
     """
 
@@ -3012,7 +3012,7 @@ def sigmoid_focal_loss(
             For more information, please refer to :ref:`api_guide_Name`.
 
     Returns:
-        Tensor, if :attr:`reduction` is ``'mean'`` or ``'sum'``, the out shape is :math:`[1]`, otherwise the shape is the same as ``logit``. The same dtype as ``logit`` tensor.
+        Tensor, if :attr:`reduction` is ``'mean'`` or ``'sum'``, the out shape is :math:`[]`, otherwise the shape is the same as ``logit``. The same dtype as ``logit`` tensor.
 
     Examples:
 
@@ -3026,7 +3026,7 @@ def sigmoid_focal_loss(
             fg_label = paddle.greater_equal(label, one)
             fg_num = paddle.sum(paddle.cast(fg_label, dtype='float32'))
             output = paddle.nn.functional.sigmoid_focal_loss(logit, label, normalizer=fg_num)
-            print(output)  # [0.65782464]
+            print(output)  # 0.65782464
 
     """
     if reduction not in ['sum', 'mean', 'none']:
@@ -3183,7 +3183,7 @@ def multi_label_soft_margin_loss(
             # Tensor([3.49625897, 0.71111226, 0.43989015])
             loss = F.multi_label_soft_margin_loss(input, label, reduction='mean')
             print(loss)
-            # Tensor([1.54908717])
+            # Tensor(1.54908717)
     """
     if reduction not in ['sum', 'mean', 'none']:
         raise ValueError(
@@ -3307,7 +3307,7 @@ def hinge_embedding_loss(input, label, margin=1.0, reduction='mean', name=None):
 
             loss = F.hinge_embedding_loss(input, label, margin=1.0, reduction='mean')
             print(loss)
-            # Tensor([0.22222222])
+            # Tensor(0.22222222)
     """
 
     if reduction not in ['sum', 'mean', 'none']:
@@ -3377,7 +3377,7 @@ def cosine_embedding_loss(
     Returns:
         Tensor, the cosine embedding Loss of Tensor ``input1`` ``input2`` and ``label``.
             If `reduction` is ``'none'``, the shape of output loss is [N], the same as ``input`` .
-            If `reduction` is ``'mean'`` or ``'sum'``, the shape of output loss is [1].
+            If `reduction` is ``'mean'`` or ``'sum'``, the shape of output loss is [].
 
     Examples:
         .. code-block:: python
@@ -3389,10 +3389,10 @@ def cosine_embedding_loss(
             label = paddle.to_tensor([1, -1], 'int64')
 
             output = paddle.nn.functional.cosine_embedding_loss(input1, input2, label, margin=0.5, reduction='mean')
-            print(output)  # [0.21155193]
+            print(output)  # 0.21155193
 
             output = paddle.nn.functional.cosine_embedding_loss(input1, input2, label, margin=0.5, reduction='sum')
-            print(output)  # [0.42310387]
+            print(output)  # 0.42310387
 
             output = paddle.nn.functional.cosine_embedding_loss(input1, input2, label, margin=0.5, reduction='none')
             print(output)  # [0.42310387, 0.        ]
@@ -3528,7 +3528,7 @@ def triplet_margin_with_distance_loss(
 
             loss = F.triplet_margin_with_distance_loss(input, positive, negative, margin=1.0, reduction='mean')
             print(loss)
-            # Tensor([0.19165580])
+            # Tensor(0.19165580)
 
     """
     if reduction not in ['sum', 'mean', 'none']:
@@ -3678,7 +3678,7 @@ def triplet_margin_loss(
 
             loss = F.triplet_margin_loss(input, positive, negative, margin=1.0, reduction='mean')
             print(loss)
-            # Tensor([0.19165580])
+            # Tensor(0.19165580)
 
     """
     if reduction not in ['sum', 'mean', 'none']:
@@ -3886,7 +3886,7 @@ def soft_margin_loss(input, label, reduction='mean', name=None):
 
     Returns:
 
-        Output (Tensor): If ``reduction`` is ``'none'``, the shape of output is same as ``input`` , else the shape of output is [1].
+        Output (Tensor): If ``reduction`` is ``'none'``, the shape of output is same as ``input`` , else the shape of output is [].
 
     Examples:
         .. code-block:: python
@@ -3897,8 +3897,8 @@ def soft_margin_loss(input, label, reduction='mean', name=None):
             label = paddle.to_tensor([[1.0, -1.0, 1.0],[-1.0, 1.0, 1.0]], 'float32')
             output = paddle.nn.functional.soft_margin_loss(input, label)
             print(output)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [0.64022040])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        0.64022040)
 
             input = paddle.uniform(shape=(5, 5), dtype="float32", min=0.1, max=0.8)
             label = paddle.randint(0, 2, shape=(5, 5), dtype="int64")
@@ -3998,7 +3998,7 @@ def gaussian_nll_loss(
 
     Returns:
 
-        output (Tensor): If ``reduction`` is ``'none'``, the shape of output is same as ``input`` , else the shape of output is [1].
+        output (Tensor): If ``reduction`` is ``'none'``, the shape of output is same as ``input`` , else the shape of output is [].
 
     Examples::
         .. code-block:: python
diff --git a/python/paddle/nn/layer/loss.py b/python/paddle/nn/layer/loss.py
index 967d490897f6e..15dbeb54c0e0a 100644
--- a/python/paddle/nn/layer/loss.py
+++ b/python/paddle/nn/layer/loss.py
@@ -98,8 +98,8 @@ class BCEWithLogitsLoss(Layer):
             bce_logit_loss = paddle.nn.BCEWithLogitsLoss()
             output = bce_logit_loss(logit, label)
             print(output)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [0.45618814])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        0.45618814)
 
     """
 
@@ -319,8 +319,8 @@ class CrossEntropyLoss(Layer):
                                         input,
                                         label)
             print(dy_ret)
-            # Tensor(shape=[1], dtype=float64, place=Place(gpu:0), stop_gradient=True,
-            #        [5.34043430])
+            # Tensor(shape=[], dtype=float64, place=Place(gpu:0), stop_gradient=True,
+            #        5.34043430)
 
         .. code-block:: python
 
@@ -345,8 +345,8 @@ class CrossEntropyLoss(Layer):
                                                                     weight=weight,
                                                                     reduction=reduction)
             print(paddle_loss_mean)
-            # Tensor(shape=[1], dtype=float64, place=Place(gpu:0), stop_gradient=True,
-            #        [1.11043464])
+            # Tensor(shape=[], dtype=float64, place=Place(gpu:0), stop_gradient=True,
+            #        1.11043464)
 
     """
 
@@ -564,7 +564,7 @@ class MSELoss(Layer):
             label = paddle.to_tensor([1.7])
             output = mse_loss(input, label)
             print(output)
-            # [0.04000002]
+            # 0.04000002
 
     """
 
@@ -637,7 +637,7 @@ class L1Loss(Layer):
         - label (Tensor): label. The shapes is ``[N, *]``, same shape as ``input`` . It's data type should be float32, float64, int32, int64.
         - output (Tensor): The L1 Loss of ``input`` and ``label``.
           If `reduction` is ``'none'``, the shape of output loss is ``[N, *]``, the same as ``input`` .
-          If `reduction` is ``'mean'`` or ``'sum'``, the shape of output loss is [1].
+          If `reduction` is ``'mean'`` or ``'sum'``, the shape of output loss is [].
 
     Examples:
         .. code-block:: python
@@ -650,14 +650,14 @@ class L1Loss(Layer):
             l1_loss = paddle.nn.L1Loss()
             output = l1_loss(input, label)
             print(output)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [0.34999999])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        0.34999999)
 
             l1_loss = paddle.nn.L1Loss(reduction='sum')
             output = l1_loss(input, label)
             print(output)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [1.39999998])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        1.39999998)
 
             l1_loss = paddle.nn.L1Loss(reduction='none')
             output = l1_loss(input, label)
@@ -747,8 +747,8 @@ class BCELoss(Layer):
             bce_loss = paddle.nn.BCELoss()
             output = bce_loss(input, label)
             print(output)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [0.65537101])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        0.65537101)
 
     """
 
@@ -835,7 +835,7 @@ class NLLLoss(Layer):
             The data type is int64.
         - output (Tensor): the `negative log likelihood loss` between input `x` and `label`.
             If `reduction` is `'none'`, the shape is `[N, *]`.
-            If `reduction` is `'sum'` or `'mean'`, the shape is `[1]`.
+            If `reduction` is `'sum'` or `'mean'`, the shape is `[]`.
 
     Examples:
         .. code-block:: python
@@ -853,7 +853,7 @@ class NLLLoss(Layer):
                 log_out = log_softmax(input)
                 label = paddle.to_tensor([0, 2, 1, 1, 0], "int64")
                 result = nll_loss(log_out, label)
-                print(result) # Tensor(shape=[1], dtype=float32, place=CPUPlace, stop_gradient=True, [1.07202101])
+                print(result) # Tensor(shape=[], dtype=float32, place=CPUPlace, stop_gradient=True, 1.07202101)
 
     """
 
@@ -991,9 +991,9 @@ class KLDivLoss(Layer):
 
     If `reduction` is ``'none'``, the output loss is the same shape as the input, and the loss at each point is calculated separately. There is no reduction to the result.
 
-    If `reduction` is ``'mean'``, the output loss is the shape of [1], and the output is the average of all losses.
+    If `reduction` is ``'mean'``, the output loss is the shape of [], and the output is the average of all losses.
 
-    If `reduction` is ``'sum'``, the output loss is the shape of [1], and the output is the sum of all losses.
+    If `reduction` is ``'sum'``, the output loss is the shape of [], and the output is the sum of all losses.
 
     If `reduction` is ``'batchmean'``, the output loss is the shape of [N], N is the batch size, and the output is the sum of all losses divided by the batch size.
 
@@ -1012,7 +1012,7 @@ class KLDivLoss(Layer):
 
         label (Tensor): ``(N, *)``, same shape as input.
 
-        output (Tensor): tensor with shape: [1] by default.
+        output (Tensor): tensor with shape: [] by default.
 
     Examples:
         .. code-block:: python
@@ -1024,20 +1024,20 @@ class KLDivLoss(Layer):
             x = paddle.uniform(shape, min=-10, max=10).astype('float32')
             target = paddle.uniform(shape, min=-10, max=10).astype('float32')
 
-            # 'batchmean' reduction, loss shape will be [1]
+            # 'batchmean' reduction, loss shape will be []
             kldiv_criterion = nn.KLDivLoss(reduction='batchmean')
             pred_loss = kldiv_criterion(x, target)
-            # shape=[1]
+            # shape=[]
 
-            # 'mean' reduction, loss shape will be [1]
+            # 'mean' reduction, loss shape will be []
             kldiv_criterion = nn.KLDivLoss(reduction='mean')
             pred_loss = kldiv_criterion(x, target)
-            # shape=[1]
+            # shape=[]
 
-            # 'sum' reduction, loss shape will be [1]
+            # 'sum' reduction, loss shape will be []
             kldiv_criterion = nn.KLDivLoss(reduction='sum')
             pred_loss = kldiv_criterion(x, target)
-            # shape=[1]
+            # shape=[]
 
             # 'none' reduction, loss shape is same with X shape
             kldiv_criterion = nn.KLDivLoss(reduction='none')
@@ -1090,7 +1090,7 @@ class MarginRankingLoss(Layer):
 
         label: N-D Tensor, label have the same shape and dtype as `input`.
 
-        output: If :attr:`reduction` is ``'mean'`` or ``'sum'`` , the out shape is :math:`[1]`, otherwise the shape is the same as `input` .The same dtype as input tensor.
+        output: If :attr:`reduction` is ``'mean'`` or ``'sum'`` , the out shape is :math:`[]`, otherwise the shape is the same as `input` .The same dtype as input tensor.
 
     Returns:
         A callable object of MarginRankingLoss.
@@ -1108,7 +1108,7 @@ class MarginRankingLoss(Layer):
             loss = margin_rank_loss(input, other, label)
 
             print(loss)
-            # [0.75]
+            # 0.75
     """
 
     def __init__(self, margin=0.0, reduction='mean', name=None):
@@ -1149,7 +1149,7 @@ class CTCLoss(Layer):
         - norm_by_times (bool, optional): Whether to normalize the gradients by the number of time-step, which is also the sequence's length. There is no need to normalize the gradients if reduction mode is 'mean'. Default: False.
 
     Returns:
-        Tensor, The Connectionist Temporal Classification (CTC) loss between ``log_probs`` and  ``labels``. If attr:`reduction` is ``'none'``, the shape of loss is [batch_size], otherwise, the shape of loss is [1]. Data type is the same as ``log_probs``.
+        Tensor, The Connectionist Temporal Classification (CTC) loss between ``log_probs`` and  ``labels``. If attr:`reduction` is ``'none'``, the shape of loss is [batch_size], otherwise, the shape of loss is []. Data type is the same as ``log_probs``.
 
     Examples:
 
@@ -1197,8 +1197,8 @@ class CTCLoss(Layer):
                 input_lengths,
                 label_lengths)
             print(loss)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [1.13760614])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        1.13760614)
     """
 
     def __init__(self, blank=0, reduction='mean'):
@@ -1242,7 +1242,7 @@ class RNNTLoss(Layer):
         label_lengths: Tensor of (batch) containing label length of each example
 
     Returns:
-     Tensor, The RNN-T loss between ``logprobs`` and  ``labels``. If attr:`reduction` is ``'none'``, the shape of loss is [batch_size], otherwise, the shape of loss is [1]. Data type is the same as ``logprobs``.
+     Tensor, The RNN-T loss between ``logprobs`` and  ``labels``. If attr:`reduction` is ``'none'``, the shape of loss is [batch_size], otherwise, the shape of loss is []. Data type is the same as ``logprobs``.
 
     Examples:
         .. code-block:: python
@@ -1272,8 +1272,8 @@ class RNNTLoss(Layer):
 
             costs = fn(acts, labels, lengths, label_lengths)
             print(costs)
-            # Tensor(shape=[1], dtype=float64, place=Place(gpu:0), stop_gradient=False,
-            #        [4.49566677])
+            # Tensor(shape=[], dtype=float64, place=Place(gpu:0), stop_gradient=False,
+            #        4.49566677)
     """
 
     def __init__(
@@ -1352,7 +1352,7 @@ class SmoothL1Loss(Layer):
             loss = paddle.nn.SmoothL1Loss()
             output = loss(input, label)
             print(output)
-            # [0.049606]
+            # 0.049606
     """
 
     def __init__(self, reduction='mean', delta=1.0, name=None):
@@ -1428,7 +1428,7 @@ class MultiLabelSoftMarginLoss(Layer):
                 multi_label_soft_margin_loss = nn.MultiLabelSoftMarginLoss(reduction='mean')
                 loss = multi_label_soft_margin_loss(input, label)
                 print(loss)
-                # Tensor([1.54908717])
+                # Tensor(1.54908717)
         """
 
     def __init__(self, weight=None, reduction="mean", name=None):
@@ -1529,7 +1529,7 @@ class HingeEmbeddingLoss(Layer):
             hinge_embedding_loss = nn.HingeEmbeddingLoss(margin=1.0, reduction='mean')
             loss = hinge_embedding_loss(input, label)
             print(loss)
-            # Tensor([0.22222222])
+            # Tensor(0.22222222)
     """
 
     def __init__(self, margin=1.0, reduction="mean", name=None):
@@ -1590,7 +1590,7 @@ class CosineEmbeddingLoss(Layer):
                          Available dtypes are int32, int64, float32, float64.
         output (Tensor): Tensor, the cosine embedding Loss of Tensor ``input1`` ``input2`` and ``label``.
                          If `reduction` is ``'none'``, the shape of output loss is [N], the same as ``input`` .
-                         If `reduction` is ``'mean'`` or ``'sum'``, the shape of output loss is [1].
+                         If `reduction` is ``'mean'`` or ``'sum'``, the shape of output loss is [].
 
     Examples:
         .. code-block:: python
@@ -1603,11 +1603,11 @@ class CosineEmbeddingLoss(Layer):
 
             cosine_embedding_loss = paddle.nn.CosineEmbeddingLoss(margin=0.5, reduction='mean')
             output = cosine_embedding_loss(input1, input2, label)
-            print(output) # [0.21155193]
+            print(output) # 0.21155193
 
             cosine_embedding_loss = paddle.nn.CosineEmbeddingLoss(margin=0.5, reduction='sum')
             output = cosine_embedding_loss(input1, input2, label)
-            print(output) # [0.42310387]
+            print(output) # 0.42310387
 
             cosine_embedding_loss = paddle.nn.CosineEmbeddingLoss(margin=0.5, reduction='none')
             output = cosine_embedding_loss(input1, input2, label)
@@ -1717,7 +1717,7 @@ class TripletMarginWithDistanceLoss(Layer):
             triplet_margin_with_distance_loss = TripletMarginWithDistanceLoss(reduction='mean')
             loss = triplet_margin_with_distance_loss(input, positive, negative,)
             print(loss)
-            # Tensor([0.19165580])
+            # Tensor(0.19165580)
 
     """
 
@@ -1825,7 +1825,7 @@ class TripletMarginLoss(Layer):
             triplet_margin_loss = paddle.nn.TripletMarginLoss(margin=1.0, swap=True, reduction='mean', )
             loss = triplet_margin_loss(input, positive, negative,)
             print(loss)
-            # Tensor([0.19165580])
+            # Tensor(0.19165580)
 
     """
 
@@ -1995,7 +1995,7 @@ class SoftMarginLoss(Layer):
           ``input``. The target labels which values should be numbers -1 or 1.
           Available dtype is int32, int64, float32, float64.
         - Output (Tensor): If ``reduction`` is ``'none'``, the shape of output is
-          same as ``input`` , else the shape of output is [1].
+          same as ``input`` , else the shape of output is [].
 
     Returns:
         A callable object of SoftMarginLoss.
@@ -2010,8 +2010,8 @@ class SoftMarginLoss(Layer):
             soft_margin_loss = paddle.nn.SoftMarginLoss()
             output = soft_margin_loss(input, label)
             print(output)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [0.64022040])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        0.64022040)
 
             input_np = paddle.uniform(shape=(5, 5), min=0.1, max=0.8, dtype="float64")
             label_np = paddle.randint(high=2, shape=(5, 5), dtype="int64")
diff --git a/python/paddle/static/nn/metric.py b/python/paddle/static/nn/metric.py
index ee1163db82825..3c1e0bf385270 100644
--- a/python/paddle/static/nn/metric.py
+++ b/python/paddle/static/nn/metric.py
@@ -66,10 +66,10 @@ def accuracy(input, label, k=1, correct=None, total=None):
             exe.run(static.default_startup_program())
             x = np.random.rand(3, 32, 32).astype("float32")
             y = np.array([[1],[0],[1]])
-            output= exe.run(feed={"input": x,"label": y},
-                        fetch_list=[result[0]])
+            output = exe.run(feed={"input": x,"label": y},
+                             fetch_list=[result])
             print(output)
-            #[array([0.], dtype=float32)]
+            # [array(0.33333334, dtype=float32)]
 
     """
     if _non_static_mode():
diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py
index 59ce9d6b3677c..b6c9cce21ff96 100644
--- a/python/paddle/tensor/creation.py
+++ b/python/paddle/tensor/creation.py
@@ -717,10 +717,10 @@ def to_tensor(data, dtype=None, place=None, stop_gradient=True):
         We use the dtype conversion rules following this:
                 Keep dtype
         np.number ───────────► paddle.Tensor
-                                (0D-Tensor)
+                                (0-D Tensor)
                     default_dtype
         Python Number ───────────────► paddle.Tensor
-                                        (0D-Tensor)
+                                        (0-D Tensor)
                     Keep dtype
         np.ndarray ───────────► paddle.Tensor
 
@@ -753,7 +753,6 @@ def to_tensor(data, dtype=None, place=None, stop_gradient=True):
         #        1)
 
         x = paddle.to_tensor(1, stop_gradient=False)
-        print(x)
         # Tensor(shape=[], dtype=int64, place=CPUPlace, stop_gradient=False,
         #        1)
 
diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py
index 3aebc68dc61bf..3b3c7565ab31f 100644
--- a/python/paddle/tensor/linalg.py
+++ b/python/paddle/tensor/linalg.py
@@ -333,8 +333,8 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None):
 
             # compute inf-order  norm
             out_pnorm = paddle.linalg.norm(x, p=float("inf"))
-            # out_pnorm  = Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-            #                    [12.])
+            # out_pnorm  = Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #                    12.)
 
             out_pnorm = paddle.linalg.norm(x, p=float("inf"), axis=0)
             # out_pnorm: Tensor(shape=[3, 4], dtype=float32, place=Place(cpu), stop_gradient=True,
@@ -344,8 +344,8 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None):
 
             # compute -inf-order  norm
             out_pnorm = paddle.linalg.norm(x, p=-float("inf"))
-            # out_pnorm: Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-            #                  [0.])
+            # out_pnorm: Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #                  0.)
 
             out_pnorm = paddle.linalg.norm(x, p=-float("inf"), axis=0)
             # out_pnorm: Tensor(shape=[3, 4], dtype=float32, place=Place(cpu), stop_gradient=True,
@@ -690,16 +690,16 @@ def dist(x, y, p=2, name=None):
             x = paddle.to_tensor([[3, 3],[3, 3]], dtype="float32")
             y = paddle.to_tensor([[3, 3],[3, 1]], dtype="float32")
             out = paddle.dist(x, y, 0)
-            print(out) # out = [1.]
+            print(out) # out = 1.
 
             out = paddle.dist(x, y, 2)
-            print(out) # out = [2.]
+            print(out) # out = 2.
 
             out = paddle.dist(x, y, float("inf"))
-            print(out) # out = [2.]
+            print(out) # out = 2.
 
             out = paddle.dist(x, y, float("-inf"))
-            print(out) # out = [0.]
+            print(out) # out = 0.
     """
     if in_dygraph_mode():
         return _C_ops.dist(x, y, p)
@@ -745,48 +745,48 @@ def cond(x, p=None, name=None):
 
             # compute conditional number when p is None
             out = paddle.linalg.cond(x)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [1.41421342])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        1.41421342)
 
             # compute conditional number when order of the norm is 'fro'
             out_fro = paddle.linalg.cond(x, p='fro')
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [3.16227770])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        3.16227770)
 
             # compute conditional number when order of the norm is 'nuc'
             out_nuc = paddle.linalg.cond(x, p='nuc')
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [9.24263859])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        9.24263859)
 
             # compute conditional number when order of the norm is 1
             out_1 = paddle.linalg.cond(x, p=1)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [2.])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        2.)
 
             # compute conditional number when order of the norm is -1
             out_minus_1 = paddle.linalg.cond(x, p=-1)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [1.])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        1.)
 
             # compute conditional number when order of the norm is 2
             out_2 = paddle.linalg.cond(x, p=2)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [1.41421342])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        1.41421342)
 
             # compute conditional number when order of the norm is -1
             out_minus_2 = paddle.linalg.cond(x, p=-2)
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [0.70710683])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        0.70710683)
 
             # compute conditional number when order of the norm is inf
             out_inf = paddle.linalg.cond(x, p=float("inf"))
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [2.])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        2.)
 
             # compute conditional number when order of the norm is -inf
             out_minus_inf = paddle.linalg.cond(x, p=-float("inf"))
-            # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-            #        [1.])
+            # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            #        1.)
 
             a = paddle.randn([2, 4, 4])
             # Tensor(shape=[2, 4, 4], dtype=float32, place=Place(gpu:0), stop_gradient=True,
@@ -1095,13 +1095,13 @@ def dot(x, y, name=None):
         x = paddle.to_tensor([1, 2, 3])
         y = paddle.to_tensor([4, 5, 6])
         z = paddle.dot(x, y)
-        print(z)  # [32]
+        print(z)  # 32
 
         # 2-D Tensor * 2-D Tensor
         x = paddle.to_tensor([[1, 2, 3], [2, 4, 6]])
         y = paddle.to_tensor([[4, 5, 6], [4, 5, 6]])
         z = paddle.dot(x, y)
-        print(z)  # [[32], [64]]
+        print(z)  # [32, 64]
 
     """
     if in_dygraph_mode():
@@ -1163,7 +1163,7 @@ def cov(x, rowvar=True, ddof=True, fweights=None, aweights=None, name=None):
 
         import paddle
 
-        xt = paddle.rand((3,4))
+        xt = paddle.rand((3, 4))
         paddle.linalg.cov(xt)
 
         '''
@@ -1485,7 +1485,7 @@ def matrix_rank(x, tol=None, hermitian=False, name=None):
             a = paddle.eye(10)
             b = paddle.linalg.matrix_rank(a)
             print(b)
-            # b = [10]
+            # b = 10
 
             c = paddle.ones(shape=[3, 4, 5, 5])
             d = paddle.linalg.matrix_rank(c, tol=0.01, hermitian=True)
diff --git a/python/paddle/tensor/logic.py b/python/paddle/tensor/logic.py
index 8a69e4fd2a654..cf3c1f5284dd4 100755
--- a/python/paddle/tensor/logic.py
+++ b/python/paddle/tensor/logic.py
@@ -288,13 +288,8 @@ def is_empty(x, name=None):
 
             input = paddle.rand(shape=[4, 32, 32], dtype='float32')
             res = paddle.is_empty(x=input)
-            print("res:", res)
-            # ('res:', Tensor: eager_tmp_1
-            #    - place: CPUPlace
-            #    - shape: [1]
-            #    - layout: NCHW
-            #    - dtype: bool
-            #    - data: [0])
+            # res: Tensor(shape=[], dtype=bool, place=Place(cpu), stop_gradient=True,
+            #        False)
 
     """
     if in_dygraph_mode():
@@ -339,9 +334,9 @@ def equal_all(x, y, name=None):
           y = paddle.to_tensor([1, 2, 3])
           z = paddle.to_tensor([1, 4, 3])
           result1 = paddle.equal_all(x, y)
-          print(result1) # result1 = [True ]
+          print(result1) # result1 = True
           result2 = paddle.equal_all(x, z)
-          print(result2) # result2 = [False ]
+          print(result2) # result2 = False
     """
     if in_dygraph_mode():
         return _C_ops.equal_all(x, y)
@@ -388,21 +383,21 @@ def allclose(x, y, rtol=1e-05, atol=1e-08, equal_nan=False, name=None):
           y = paddle.to_tensor([10000.1, 1e-08])
           result1 = paddle.allclose(x, y, rtol=1e-05, atol=1e-08,
                                   equal_nan=False, name="ignore_nan")
-          # [False]
+          # False
 
           result2 = paddle.allclose(x, y, rtol=1e-05, atol=1e-08,
                                       equal_nan=True, name="equal_nan")
-          # [False]
+          # False
 
           x = paddle.to_tensor([1.0, float('nan')])
           y = paddle.to_tensor([1.0, float('nan')])
           result1 = paddle.allclose(x, y, rtol=1e-05, atol=1e-08,
                                   equal_nan=False, name="ignore_nan")
-          # [False]
+          # False
 
           result2 = paddle.allclose(x, y, rtol=1e-05, atol=1e-08,
                                       equal_nan=True, name="equal_nan")
-          # [True]
+          # True
     """
 
     if in_dygraph_mode():
diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py
index e6bf626cb1a4c..6b94843b65c13 100644
--- a/python/paddle/tensor/manipulation.py
+++ b/python/paddle/tensor/manipulation.py
@@ -268,11 +268,11 @@ def slice(input, axes, starts, ends):
     Args:
         input (Tensor): A ``Tensor`` . The data type is ``float16``, ``float32``, ``float64``, ``int32`` or ``int64``.
         axes (list|tuple): The data type is ``int32`` . Axes that `starts` and `ends` apply to .
-        starts (list|tuple|Tensor): The data type is ``int32`` . If ``starts`` is a list or tuple, the elements of
-                it should be integers or Tensors with shape [1]. If ``starts`` is an Tensor, it should be an 1-D Tensor.
+        starts (list|tuple|Tensor): The data type is ``int32`` . If ``starts`` is a list or tuple, each element of
+                it should be integer or 0-D int Tensor with shape []. If ``starts`` is an Tensor, it should be an 1-D Tensor.
                 It represents starting indices of corresponding axis in ``axes``.
-        ends (list|tuple|Tensor): The data type is ``int32`` . If ``ends`` is a list or tuple, the elements of
-                it should be integers or Tensors with shape [1]. If ``ends`` is an Tensor, it should be an 1-D Tensor .
+        ends (list|tuple|Tensor): The data type is ``int32`` . If ``ends`` is a list or tuple, each element of
+                it should be integer or 0-D int Tensor with shape []. If ``ends`` is an Tensor, it should be an 1-D Tensor .
                 It represents ending indices of corresponding axis in ``axes``.
 
     Returns:
@@ -1065,21 +1065,21 @@ def tolist(x):
             print(expectlist)   #[0, 1, 2, 3, 4]
 
     """
-    # TODO(zhouwei): will remove 0D Tensor.numpy() hack
+    # TODO(zhouwei): will remove 0-D Tensor.numpy() hack
     return x.numpy(False).tolist()
 
 
 def concat(x, axis=0, name=None):
     """
 
-    Concatenates the input along the axis.
+    Concatenates the input along the axis. It doesn't support 0-D Tensor because it requires a certain axis, and 0-D Tensor
+    doesn't have any axis.
 
     Args:
         x (list|tuple): ``x`` is a Tensor list or Tensor tuple which is with data type bool, float16,
             float32, float64, int32, int64, int8, uint8. All the Tensors in ``x`` must have same data type.
         axis (int|Tensor, optional): Specify the axis to operate on the input Tensors.
-            It's a scalar with data type int or a Tensor with shape [1] and data type int32
-            or int64. The effective range is [-R, R), where R is Rank(x). When ``axis < 0``,
+            Tt should be integer or 0-D int Tensor with shape []. The effective range is [-R, R), where R is Rank(x). When ``axis < 0``,
             it works the same way as ``axis+R``. Default is 0.
         name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
 
@@ -1550,11 +1550,11 @@ def flatten(x, start_axis=0, stop_axis=-1, name=None):
     if x_dim == 0:
         if not (isinstance(start_axis, int)) or start_axis not in [0, -1]:
             raise ValueError(
-                "The start_axis should be int, and should be 0 or -1 when the input tensor is a 0D-Tensor"
+                "The start_axis should be int, and should be 0 or -1 when the input tensor is a 0-D-Tensor"
             )
         if not (isinstance(stop_axis, int)) or stop_axis not in [0, -1]:
             raise ValueError(
-                "The stop_axis should be int, and should be 0 or -1 when the input tensor is a 0D-Tensor"
+                "The stop_axis should be int, and should be 0 or -1 when the input tensor is a 0-D-Tensor"
             )
     else:
         if (
@@ -1913,8 +1913,8 @@ def split(x, num_or_sections, axis=0, name=None):
             If ``num_or_sections`` is a list or tuple, the length of it indicates the number of
             sub-Tensors and the elements in it indicate the sizes of sub-Tensors'  dimension orderly.
             The length of the list must not  be larger than the ``x`` 's size of specified ``axis``.
-        axis (int|Tensor, optional): The axis along which to split, it can be a scalar with type
-            ``int`` or a ``Tensor`` with shape [1] and data type  ``int32`` or ``int64``.
+        axis (int|Tensor, optional): The axis along which to split, it can be a integer or a ``0-D Tensor``
+            with shape [] and data type  ``int32`` or ``int64``.
             If :math::`axis < 0`, the axis to split along is :math:`rank(x) + axis`. Default is 0.
         name (str, optional): The default value is None.  Normally there is no need for user to set this property.
             For more information, please refer to :ref:`api_guide_Name` .
@@ -2557,7 +2557,7 @@ def unsqueeze(x, axis, name=None):
     Args:
         x (Tensor): The input Tensor to be unsqueezed. Supported data type: bfloat16, float16, float32, float64, bool, int8, int32, int64.
         axis (int|list|tuple|Tensor): Indicates the dimensions to be inserted. The data type is ``int32`` .
-                                    If ``axis`` is a list or tuple, the elements of it should be integers or Tensors with shape [1].
+                                    If ``axis`` is a list or tuple, each element of it should be integer or 0-D Tensor with shape [].
                                     If ``axis`` is a Tensor, it should be an 1-D Tensor .
                                     If ``axis`` is negative, ``axis = axis + ndim(x) + 1``.
         name (str|None): Name for this layer. Please refer to :ref:`api_guide_Name`, Default None.
@@ -3083,8 +3083,8 @@ def chunk(x, chunks, axis=0, name=None):
     Args:
         x (Tensor): A N-D Tensor. The data type is bool, float16, float32, float64, int32 or int64.
         chunks(int): The number of tensor to be split along the certain axis.
-        axis (int|Tensor, optional): The axis along which to split, it can be a scalar with type
-            ``int`` or a ``Tensor`` with shape [1] and data type  ``int32`` or ``int64``.
+        axis (int|Tensor, optional): The axis along which to split, it can be a integer or a ``0-D Tensor``
+            with shape [] and data type  ``int32`` or ``int64``.
             If :math::`axis < 0`, the axis to split along is :math:`rank(x) + axis`. Default is 0.
         name (str, optional): The default value is None.  Normally there is no need for user to set this property.
             For more information, please refer to :ref:`api_guide_Name` .
@@ -3523,7 +3523,7 @@ def reshape(x, shape, name=None):
     Args:
         x (Tensor): An N-D Tensor. The data type is ``float32``, ``float64``, ``int32``, ``int64`` or ``bool``
         shape (list|tuple|Tensor): Define the target shape. At most one dimension of the target shape can be -1.
-                        The data type is ``int32`` . If ``shape`` is a list or tuple, the elements of it should be integers or Tensors with shape [].
+                        The data type is ``int32`` . If ``shape`` is a list or tuple, each element of it should be integer or Tensor with shape [].
                         If ``shape`` is an Tensor, it should be an 1-D Tensor .
         name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
 
@@ -3843,11 +3843,15 @@ def strided_slice(x, axes, starts, ends, strides, name=None):
         x (Tensor): An N-D ``Tensor``. The data type is ``bool``, ``float16``, ``float32``, ``float64``, ``int32`` or ``int64``.
         axes (list|tuple): The data type is ``int32`` . Axes that `starts` and `ends` apply to.
                             It's optional. If it is not provides, it will be treated as :math:`[0,1,...,len(starts)-1]`.
-        starts (list|tuple|Tensor): The data type is ``int32`` . If ``starts`` is a list or tuple, the elements of                                                                                          it should be integers or Tensors with shape [1]. If ``starts`` is an Tensor, it should be an 1-D Tensor.                                                                                    It represents starting indices of corresponding axis in ``axes``.
-        ends (list|tuple|Tensor): The data type is ``int32`` . If ``ends`` is a list or tuple, the elements of
-                it should be integers or Tensors with shape [1]. If ``ends`` is an Tensor, it should be an 1-D Tensor .                                                                                     It represents ending indices of corresponding axis in ``axes``.
-        strides (list|tuple|Tensor): The data type is ``int32`` . If ``strides`` is a list or tuple, the elements of
-                it should be integers or Tensors with shape [1]. If ``strides`` is an Tensor, it should be an 1-D Tensor .                                                                                  It represents slice step of corresponding axis in ``axes``.
+        starts (list|tuple|Tensor): The data type is ``int32`` . If ``starts`` is a list or tuple, the elements of it should be
+            integers or Tensors with shape []. If ``starts`` is an Tensor, it should be an 1-D Tensor.
+            It represents starting indices of corresponding axis in ``axes``.
+        ends (list|tuple|Tensor): The data type is ``int32`` . If ``ends`` is a list or tuple, the elements of it should be
+            integers or Tensors with shape []. If ``ends`` is an Tensor, it should be an 1-D Tensor.
+            It represents ending indices of corresponding axis in ``axes``.
+        strides (list|tuple|Tensor): The data type is ``int32`` . If ``strides`` is a list or tuple, the elements of it should be
+            integers or Tensors with shape []. If ``strides`` is an Tensor, it should be an 1-D Tensor.
+            It represents slice step of corresponding axis in ``axes``.
         name(str, optional): The default value is None.  Normally there is no need for user to set this property.
                         For more information, please refer to :ref:`api_guide_Name` .
 
@@ -4074,7 +4078,7 @@ def tensordot(x, y, axes=2, name=None):
             y = paddle.arange(10, dtype=data_type)
             z1 = paddle.tensordot(x, y, axes=1)
             z2 = paddle.dot(x, y)
-            # z1 = z2 = [285.]
+            # z1 = z2 = 285.
 
 
             # For two 2-d tensor x and y, the case axes=1 is equivalent to matrix multiplication.
diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py
index cbe0474960de2..debf74e87ac45 100644
--- a/python/paddle/tensor/math.py
+++ b/python/paddle/tensor/math.py
@@ -187,7 +187,7 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None):
 
     Args:
         x (Tensor): Input N-D Tensor of scale operator. Data type can be float32, float64, int8, int16, int32, int64, uint8.
-        scale (float|Tensor): The scale factor of the input, it should be a float number or a Tensor with shape [1] and data type as float32.
+        scale (float|Tensor): The scale factor of the input, it should be a float number or a 0-D Tensor with shape [] and data type as float32.
         bias (float): The bias to be put on the input.
         bias_after_scale (bool): Apply bias addition after or before scaling. It is useful for numeric stability in some circumstances.
         act (str, optional): Activation applied to the output such as tanh, softmax, sigmoid, relu.
@@ -1337,9 +1337,9 @@ def sum(x, axis=None, dtype=None, keepdim=False, name=None):
             # Each example is followed by the corresponding output tensor.
             x = paddle.to_tensor([[0.2, 0.3, 0.5, 0.9],
                                   [0.1, 0.2, 0.6, 0.7]])
-            out1 = paddle.sum(x)  # [3.5]
+            out1 = paddle.sum(x)          # 3.5
             out2 = paddle.sum(x, axis=0)  # [0.3, 0.5, 1.1, 1.6]
-            out3 = paddle.sum(x, axis=-1)  # [1.9, 1.6]
+            out3 = paddle.sum(x, axis=-1) # [1.9, 1.6]
             out4 = paddle.sum(x, axis=1, keepdim=True)  # [[1.9], [1.6]]
 
             # y is a Tensor with shape [2, 2, 2] and elements as below:
@@ -1357,7 +1357,7 @@ def sum(x, axis=None, dtype=None, keepdim=False, name=None):
             # Each example is followed by the corresponding output tensor.
             x = paddle.to_tensor([[True, True, True, True],
                                   [False, False, False, False]])
-            out7 = paddle.sum(x)  # [4]
+            out7 = paddle.sum(x)          # 4
             out8 = paddle.sum(x, axis=0)  # [1, 1, 1, 1]
             out9 = paddle.sum(x, axis=1)  # [4, 0]
     """
@@ -1493,9 +1493,9 @@ def nansum(x, axis=None, dtype=None, keepdim=False, name=None):
             # Each example is followed by the corresponding output tensor.
             x = paddle.to_tensor([[float('nan'), 0.3, 0.5, 0.9],
                             [0.1, 0.2, float('-nan'), 0.7]],dtype="float32")
-            out1 = paddle.nansum(x)  # [2.7]
+            out1 = paddle.nansum(x)          # 2.7
             out2 = paddle.nansum(x, axis=0)  # [0.1, 0.5, 0.5, 1.6]
-            out3 = paddle.nansum(x, axis=-1)  # [1.7, 1.0]
+            out3 = paddle.nansum(x, axis=-1) # [1.7, 1.0]
             out4 = paddle.nansum(x, axis=1, keepdim=True)  # [[1.7], [1.0]]
 
             # y is a Tensor with shape [2, 2, 2] and elements as below:
@@ -1553,7 +1553,7 @@ def nanmean(x, axis=None, keepdim=False, name=None):
             x = paddle.to_tensor([[float('nan'), 0.3, 0.5, 0.9],
                                   [0.1, 0.2, float('-nan'), 0.7]])
             out1 = paddle.nanmean(x)
-            # [0.44999996]
+            # 0.44999996
             out2 = paddle.nanmean(x, axis=0)
             # [0.1, 0.25, 0.5, 0.79999995]
             out3 = paddle.nanmean(x, axis=0, keepdim=True)
@@ -2263,7 +2263,7 @@ def logsumexp(x, axis=None, keepdim=False, name=None):
         import paddle
 
         x = paddle.to_tensor([[-1.5, 0., 2.], [3., 1.2, -2.4]])
-        out1 = paddle.logsumexp(x) # [3.4691226]
+        out1 = paddle.logsumexp(x)    # 3.4691226
         out2 = paddle.logsumexp(x, 1) # [2.15317821, 3.15684602]
 
     """
@@ -2375,7 +2375,7 @@ def max(x, axis=None, keepdim=False, name=None):
             result1 = paddle.max(x)
             result1.backward()
             print(result1, x.grad)
-            #[0.9], [[0., 0., 0., 1.], [0., 0., 0., 0.]]
+            # 0.9, [[0., 0., 0., 1.], [0., 0., 0., 0.]]
 
             x.clear_grad()
             result2 = paddle.max(x, axis=0)
@@ -2476,7 +2476,7 @@ def min(x, axis=None, keepdim=False, name=None):
             result1 = paddle.min(x)
             result1.backward()
             print(result1, x.grad)
-            #[0.1], [[0., 0., 0., 0.], [1., 0., 0., 0.]]
+            # 0.1, [[0., 0., 0., 0.], [1., 0., 0., 0.]]
 
             x.clear_grad()
             result2 = paddle.min(x, axis=0)
@@ -2580,13 +2580,13 @@ def amax(x, axis=None, keepdim=False, name=None):
             result1 = paddle.amax(x)
             result1.backward()
             print(result1, x.grad)
-            #[0.9], [[0., 0.2, 0.2, 0.2], [0.2, 0.2, 0., 0.]]
+            # 0.9, [[0., 0.2, 0.2, 0.2], [0.2, 0.2, 0., 0.]]
 
             x.clear_grad()
             result1_max = paddle.max(x)
             result1_max.backward()
             print(result1_max, x.grad)
-            #[0.9], [[0., 1.0, 1.0, 1.0], [1.0, 1.0, 0., 0.]]
+            # 0.9, [[0., 1.0, 1.0, 1.0], [1.0, 1.0, 0., 0.]]
 
             ###############################
 
@@ -2690,13 +2690,13 @@ def amin(x, axis=None, keepdim=False, name=None):
             result1 = paddle.amin(x)
             result1.backward()
             print(result1, x.grad)
-            #[0.1], [[0., 0.2, 0.2, 0.2], [0.2, 0.2, 0., 0.]]
+            # 0.1, [[0., 0.2, 0.2, 0.2], [0.2, 0.2, 0., 0.]]
 
             x.clear_grad()
             result1_min = paddle.min(x)
             result1_min.backward()
             print(result1_min, x.grad)
-            #[0.1], [[0., 1.0, 1.0, 1.0], [1.0, 1.0, 0., 0.]]
+            # 0.1, [[0., 1.0, 1.0, 1.0], [1.0, 1.0, 0., 0.]]
 
             ###############################
 
@@ -2907,10 +2907,10 @@ def clip(x, min=None, max=None, name=None):
 
     Args:
         x (Tensor): An N-D Tensor with data type float16, float32, float64, int32 or int64.
-        min (float|int|Tensor, optional): The lower bound with type ``float`` , ``int`` or a ``Tensor``
-            with shape [1] and type ``int32``, ``float16``, ``float32``, ``float64``.
-        max (float|int|Tensor, optional): The upper bound with type ``float``, ``int`` or a ``Tensor``
-            with shape [1] and type ``int32``, ``float16``, ``float32``, ``float64``.
+        min (float|int|Tensor, optional): The lower bound with type ``float`` , ``int`` or a ``0-D Tensor``
+            with shape [] and type ``int32``, ``float16``, ``float32``, ``float64``.
+        max (float|int|Tensor, optional): The upper bound with type ``float``, ``int`` or a ``0-D Tensor``
+            with shape [] and type ``int32``, ``float16``, ``float32``, ``float64``.
         name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
 
     Returns:
@@ -3064,7 +3064,7 @@ def trace(x, offset=0, axis1=0, axis2=1, name=None):
             case1 = paddle.randn([2, 3])
             case2 = paddle.randn([3, 10, 10])
             case3 = paddle.randn([3, 10, 5, 10])
-            data1 = paddle.trace(case1) # data1.shape = [1]
+            data1 = paddle.trace(case1) # data1.shape = []
             data2 = paddle.trace(case2, offset=1, axis1=1, axis2=2) # data2.shape = [3]
             data3 = paddle.trace(case3, offset=-3, axis1=1, axis2=-1) # data2.shape = [3, 5]
     """
@@ -3692,7 +3692,7 @@ def prod(x, axis=None, keepdim=False, dtype=None, name=None):
             x = paddle.to_tensor([[0.2, 0.3, 0.5, 0.9],
                                   [0.1, 0.2, 0.6, 0.7]])
             out1 = paddle.prod(x)
-            # [0.0002268]
+            # 0.0002268
 
             out2 = paddle.prod(x, -1)
             # [0.027  0.0084]
@@ -3898,8 +3898,8 @@ def all(x, axis=None, keepdim=False, name=None):
             print(x)
             x = paddle.cast(x, 'bool')
 
-            # out1 should be [False]
-            out1 = paddle.all(x)  # [False]
+            # out1 should be False
+            out1 = paddle.all(x)          # False
             print(out1)
 
             # out2 should be [True, False]
@@ -3907,7 +3907,7 @@ def all(x, axis=None, keepdim=False, name=None):
             print(out2)
 
             # keepdim=False, out3 should be [False, True], out.shape should be (2,)
-            out3 = paddle.all(x, axis=-1)  # [False, True]
+            out3 = paddle.all(x, axis=-1) # [False, True]
             print(out3)
 
             # keepdim=True, out4 should be [[False], [True]], out.shape should be (2,1)
@@ -3972,12 +3972,12 @@ def any(x, axis=None, keepdim=False, name=None):
             #    [[True, False]
             #     [True, True]]
 
-            # out1 should be [True]
-            out1 = paddle.any(x)  # [True]
+            # out1 should be True
+            out1 = paddle.any(x)           # True
             print(out1)
 
             # out2 should be [True, True]
-            out2 = paddle.any(x, axis=0)  # [True, True]
+            out2 = paddle.any(x, axis=0)   # [True, True]
             print(out2)
 
             # keepdim=False, out3 should be [True, True], out.shape should be (2,)
@@ -4481,8 +4481,8 @@ def rad2deg(x, name=None):
             x2 = paddle.to_tensor(math.pi/2)
             result2 = paddle.rad2deg(x2)
             print(result2)
-            # Tensor(shape=[1], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
-            #         [90.])
+            # Tensor(shape=[], dtype=float32, place=CUDAPlace(0), stop_gradient=True,
+            #         90.)
 
             x3 = paddle.to_tensor(1)
             result3 = paddle.rad2deg(x3)
@@ -5382,27 +5382,27 @@ def trapezoid(y, x=None, dx=None, axis=-1, name=None):
             y = paddle.to_tensor([4, 5, 6], dtype='float32')
 
             print(paddle.trapezoid(y))
-            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-            #        [10.])
+            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        10.)
 
             print(paddle.trapezoid(y, dx=2.))
-            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-            #        [20.])
+            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        20.)
 
             y = paddle.to_tensor([4, 5, 6], dtype='float32')
             x = paddle.to_tensor([1, 2, 3], dtype='float32')
 
             print(paddle.trapezoid(y, x))
-            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-            #        [10.])
+            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        10.)
 
 
             y = paddle.to_tensor([1, 2, 3], dtype='float64')
             x = paddle.to_tensor([8, 6, 4], dtype='float64')
 
             print(paddle.trapezoid(y, x))
-            # Tensor(shape=[1], dtype=float64, place=Place(cpu), stop_gradient=True,
-            #        [-8.])
+            # Tensor(shape=[], dtype=float64, place=Place(cpu), stop_gradient=True,
+            #        -8.)
             y = paddle.arange(6).reshape((2, 3)).astype('float32')
 
             print(paddle.trapezoid(y, axis=0))
diff --git a/python/paddle/tensor/stat.py b/python/paddle/tensor/stat.py
index 7677f71bcac29..c2332da9ea7d9 100644
--- a/python/paddle/tensor/stat.py
+++ b/python/paddle/tensor/stat.py
@@ -65,7 +65,7 @@ def mean(x, axis=None, keepdim=False, name=None):
                                    [17., 18., 19., 20.],
                                    [21., 22., 23., 24.]]])
             out1 = paddle.mean(x)
-            # [12.5]
+            # 12.5
             out2 = paddle.mean(x, axis=-1)
             # [[ 2.5  6.5 10.5]
             #  [14.5 18.5 22.5]]
@@ -140,7 +140,7 @@ def var(x, axis=None, unbiased=True, keepdim=False, name=None):
 
             x = paddle.to_tensor([[1.0, 2.0, 3.0], [1.0, 4.0, 5.0]])
             out1 = paddle.var(x)
-            # [2.66666667]
+            # 2.66666667
             out2 = paddle.var(x, axis=1)
             # [1.         4.33333333]
     """
@@ -205,9 +205,9 @@ def std(x, axis=None, unbiased=True, keepdim=False, name=None):
 
             x = paddle.to_tensor([[1.0, 2.0, 3.0], [1.0, 4.0, 5.0]])
             out1 = paddle.std(x)
-            # [1.63299316]
+            # 1.63299316
             out2 = paddle.std(x, unbiased=False)
-            # [1.49071205]
+            # 1.49071205
             out3 = paddle.std(x, axis=1)
             # [1.       2.081666]
 
@@ -222,8 +222,7 @@ def std(x, axis=None, unbiased=True, keepdim=False, name=None):
 
 def numel(x, name=None):
     """
-    Returns the number of elements for a tensor, which is a int64 Tensor with shape [1] in static graph mode
-    or a scalar value in imperative mode.
+    Returns the number of elements for a tensor, which is a 0-D int64 Tensor with shape [].
 
     Args:
         x (Tensor): The input Tensor, it's data type can be bool, float16, float32, float64, int32, int64.
@@ -231,7 +230,7 @@ def numel(x, name=None):
             For more information, please refer to :ref:`api_guide_Name`.
 
     Returns:
-        Tensor: The number of elements for the input Tensor.
+        Tensor: The number of elements for the input Tensor, whose shape is [].
 
     Examples:
         .. code-block:: python
@@ -387,8 +386,8 @@ def median(x, axis=None, keepdim=False, name=None):
             #         [8 , 9 , 10, 11]])
 
             y1 = paddle.median(x)
-            # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-            #        [5.50000000])
+            # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+            #        5.50000000)
 
             y2 = paddle.median(x, axis=0)
             # Tensor(shape=[4], dtype=float32, place=Place(cpu), stop_gradient=True,
@@ -416,7 +415,7 @@ def median(x, axis=None, keepdim=False, name=None):
             -1,
             0,
             None,
-        ], 'when input 0D, axis can only be [-1, 0] or default None'
+        ], 'when input 0-D, axis can only be [-1, 0] or default None'
         is_flatten = True
 
     if axis is None:
diff --git a/python/paddle/tensor/to_string.py b/python/paddle/tensor/to_string.py
index 897735333ace8..0d8cd9a6b8196 100644
--- a/python/paddle/tensor/to_string.py
+++ b/python/paddle/tensor/to_string.py
@@ -181,8 +181,7 @@ def _format_tensor(var, summary, indent=0, max_width=0, signed=False):
     linewidth = DEFAULT_PRINT_OPTIONS.linewidth
 
     if len(var.shape) == 0:
-        # currently, shape = [], i.e., scaler tensor is not supported.
-        # If it is supported, it should be formatted like this.
+        # 0-D Tensor, whose shape = [], should be formatted like this.
         return _format_item(var, max_width, signed)
     elif len(var.shape) == 1:
         item_length = max_width + 2
@@ -291,7 +290,7 @@ def _format_dense_tensor(tensor, indent):
     if tensor.dtype == core.VarDesc.VarType.BF16:
         tensor = tensor.astype('float32')
 
-    # TODO(zhouwei): will remove 0D Tensor.numpy() hack
+    # TODO(zhouwei): will remove 0-D Tensor.numpy() hack
     np_tensor = tensor.numpy(False)
 
     if len(tensor.shape) == 0:
diff --git a/python/paddle/utils/cpp_extension/extension_utils.py b/python/paddle/utils/cpp_extension/extension_utils.py
index 7da4d1f1ba122..f5fbcc35d46de 100644
--- a/python/paddle/utils/cpp_extension/extension_utils.py
+++ b/python/paddle/utils/cpp_extension/extension_utils.py
@@ -562,8 +562,6 @@ def normalize_extension_kwargs(kwargs, use_cuda=False):
                 extra_compile_args[compiler] = []
 
     if IS_WINDOWS:
-        # TODO(zhouwei): may append compile flags in future
-        pass
         # append link flags
         extra_link_args = kwargs.get('extra_link_args', [])
         extra_link_args.extend(MSVC_LINK_FLAGS)

From 74b91bce4b3a8b761a67fe881c56bd19f552fbd1 Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Tue, 16 May 2023 19:59:53 +0800
Subject: [PATCH 369/405] Add huber_loss tests (#53535)

---
 .../phi/kernels/gpu/huber_loss_grad_kernel.cu | 11 ++-
 paddle/phi/kernels/gpu/huber_loss_kernel.cu   | 10 ++-
 .../impl/huber_loss_grad_kernel_impl.h        | 15 ++--
 .../phi/kernels/impl/huber_loss_kernel_impl.h |  3 +-
 .../tests/unittests/test_huber_loss_op.py     | 87 ++++++++++++++++---
 python/paddle/nn/functional/loss.py           | 10 ++-
 6 files changed, 111 insertions(+), 25 deletions(-)

diff --git a/paddle/phi/kernels/gpu/huber_loss_grad_kernel.cu b/paddle/phi/kernels/gpu/huber_loss_grad_kernel.cu
index 9e169cbdcb7d0..77fb5454e4684 100644
--- a/paddle/phi/kernels/gpu/huber_loss_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/huber_loss_grad_kernel.cu
@@ -18,6 +18,11 @@
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/impl/huber_loss_grad_kernel_impl.h"
 
-PD_REGISTER_KERNEL(
-    huber_loss_grad, GPU, ALL_LAYOUT, phi::HuberLossGradKernel, float, double) {
-}
+PD_REGISTER_KERNEL(huber_loss_grad,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::HuberLossGradKernel,
+                   float,
+                   double,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/paddle/phi/kernels/gpu/huber_loss_kernel.cu b/paddle/phi/kernels/gpu/huber_loss_kernel.cu
index efb454832cd7f..badc655f425c8 100644
--- a/paddle/phi/kernels/gpu/huber_loss_kernel.cu
+++ b/paddle/phi/kernels/gpu/huber_loss_kernel.cu
@@ -18,5 +18,11 @@
 #include "paddle/phi/core/kernel_registry.h"
 #include "paddle/phi/kernels/impl/huber_loss_kernel_impl.h"
 
-PD_REGISTER_KERNEL(
-    huber_loss, GPU, ALL_LAYOUT, phi::HuberLossKernel, float, double) {}
+PD_REGISTER_KERNEL(huber_loss,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::HuberLossKernel,
+                   float,
+                   double,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/paddle/phi/kernels/impl/huber_loss_grad_kernel_impl.h b/paddle/phi/kernels/impl/huber_loss_grad_kernel_impl.h
index b93578abba2b7..6addaab129bda 100644
--- a/paddle/phi/kernels/impl/huber_loss_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/huber_loss_grad_kernel_impl.h
@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include "paddle/phi/common/amp_type_traits.h"
 #include "paddle/phi/kernels/funcs/eigen/common.h"
 #include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
 #include "paddle/phi/kernels/huber_loss_grad_kernel.h"
@@ -26,14 +27,14 @@ struct HuberLossBackward {
       : sign(sign), delta(delta) {}
 
   HOSTDEVICE T operator()(const T& val) const {
-    T abs_val = std::abs(val);
+    T abs_val = abs(val);
     if (abs_val <= delta) {
       return sign * val;
     } else {
-      if (val > 0) {
+      if (val > static_cast<T>(0)) {
         return sign * delta;
       } else {
-        return -1 * sign * delta;
+        return static_cast<T>(-1) * sign * delta;
       }
     }
   }
@@ -58,16 +59,16 @@ void HuberLossGradKernel(const Context& dev_ctx,
   if (input_grad) {
     dev_ctx.template Alloc<T>(input_grad);
     auto eigen_input_grad = EigenVector<T>::Flatten(*input_grad);
-    eigen_input_grad.device(place) =
-        eigen_residual.unaryExpr(HuberLossBackward<T>(delta_, -1.0));
+    eigen_input_grad.device(place) = eigen_residual.unaryExpr(
+        HuberLossBackward<T>(delta_, static_cast<T>(-1.0)));
     eigen_input_grad.device(place) = eigen_out_grad * eigen_input_grad;
   }
 
   if (label_grad) {
     dev_ctx.template Alloc<T>(label_grad);
     auto eigen_label_grad = EigenVector<T>::Flatten(*label_grad);
-    eigen_label_grad.device(place) =
-        eigen_residual.unaryExpr(HuberLossBackward<T>(delta_, 1.0));
+    eigen_label_grad.device(place) = eigen_residual.unaryExpr(
+        HuberLossBackward<T>(delta_, static_cast<T>(1.0)));
     eigen_label_grad.device(place) = eigen_out_grad * eigen_label_grad;
   }
 }
diff --git a/paddle/phi/kernels/impl/huber_loss_kernel_impl.h b/paddle/phi/kernels/impl/huber_loss_kernel_impl.h
index 7fbdc80c3829b..517fffb6ef3a5 100644
--- a/paddle/phi/kernels/impl/huber_loss_kernel_impl.h
+++ b/paddle/phi/kernels/impl/huber_loss_kernel_impl.h
@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include "paddle/phi/common/amp_type_traits.h"
 #include "paddle/phi/kernels/funcs/eigen/common.h"
 #include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
 #include "paddle/phi/kernels/huber_loss_kernel.h"
@@ -25,7 +26,7 @@ struct HuberLossForward {
   HOSTDEVICE HuberLossForward(const T& delta) : delta(delta) {}
 
   HOSTDEVICE T operator()(const T& val) const {
-    T abs_val = std::abs(val);
+    T abs_val = abs(val);
     if (abs_val <= delta) {
       return static_cast<T>(0.5) * val * val;
     } else {
diff --git a/python/paddle/fluid/tests/unittests/test_huber_loss_op.py b/python/paddle/fluid/tests/unittests/test_huber_loss_op.py
index cc873e40b8d76..318e2bf68fc32 100644
--- a/python/paddle/fluid/tests/unittests/test_huber_loss_op.py
+++ b/python/paddle/fluid/tests/unittests/test_huber_loss_op.py
@@ -15,9 +15,10 @@
 import unittest
 
 import numpy as np
-from eager_op_test import OpTest
+from eager_op_test import OpTest, convert_float_to_uint16
 
 import paddle
+from paddle.fluid import core
 
 
 def huber_loss_forward(val, delta):
@@ -40,20 +41,24 @@ def setUp(self):
         self.python_api = huber_loss_wraper
 
         self.delta = 1.0
+        self.init_dtype()
         self.init_input()
         shape = self.set_shape()
         residual = self.inputs['Y'] - self.inputs['X']
         loss = np.vectorize(huber_loss_forward)(residual, self.delta).astype(
-            'float32'
+            self.dtype
         )
         self.attrs = {'delta': self.delta}
         self.outputs = {'Residual': residual, 'Out': loss.reshape(shape)}
 
+    def init_dtype(self):
+        self.dtype = np.float32
+
     def init_input(self):
         shape = self.set_shape()
         self.inputs = {
-            'X': np.random.uniform(0, 1.0, shape).astype('float32'),
-            'Y': np.random.uniform(0, 1.0, shape).astype('float32'),
+            'X': np.random.uniform(0, 1.0, shape).astype(self.dtype),
+            'Y': np.random.uniform(0, 1.0, shape).astype(self.dtype),
         }
 
     def set_shape(self):
@@ -66,14 +71,10 @@ def test_check_grad_normal(self):
         self.check_grad(['X', 'Y'], 'Out')
 
     def test_check_grad_ingore_x(self):
-        self.check_grad(
-            ['Y'], 'Out', max_relative_error=0.008, no_grad_set=set("residual")
-        )
+        self.check_grad(['Y'], 'Out', no_grad_set=set("residual"))
 
     def test_check_grad_ingore_y(self):
-        self.check_grad(
-            ['X'], 'Out', max_relative_error=0.008, no_grad_set=set('residual')
-        )
+        self.check_grad(['X'], 'Out', no_grad_set=set('residual'))
 
 
 def TestHuberLossOp1(TestHuberLossOp):
@@ -91,6 +92,72 @@ def set_shape(self):
         return (6, 6, 1)
 
 
+class TestHuberLossFP16Op(TestHuberLossOp):
+    def init_dtype(self):
+        self.dtype = np.float16
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support bfloat16",
+)
+class TestHuberLossBF16Op(OpTest):
+    def setUp(self):
+        self.op_type = 'huber_loss'
+        self.python_out_sig = ["Out"]
+        self.python_api = huber_loss_wraper
+
+        self.delta = 1.0
+        self.init_dtype()
+        self.init_input()
+        shape = self.set_shape()
+        residual = self.inputs['Y'] - self.inputs['X']
+        loss = np.vectorize(huber_loss_forward)(residual, self.delta).astype(
+            self.np_dtype
+        )
+        self.attrs = {'delta': self.delta}
+        self.outputs = {'Residual': residual, 'Out': loss.reshape(shape)}
+
+        self.place = core.CUDAPlace(0)
+        self.inputs['X'] = convert_float_to_uint16(self.inputs['X'])
+        self.inputs['Y'] = convert_float_to_uint16(self.inputs['Y'])
+        self.outputs['Residual'] = convert_float_to_uint16(
+            self.outputs['Residual']
+        )
+        self.outputs['Out'] = convert_float_to_uint16(self.outputs['Out'])
+
+    def init_dtype(self):
+        self.dtype = np.uint16
+        self.np_dtype = np.float32
+
+    def init_input(self):
+        shape = self.set_shape()
+        self.inputs = {
+            'X': np.random.uniform(0, 1.0, shape).astype(self.np_dtype),
+            'Y': np.random.uniform(0, 1.0, shape).astype(self.np_dtype),
+        }
+
+    def set_shape(self):
+        return (100, 1)
+
+    def test_check_output(self):
+        self.check_output_with_place(self.place)
+
+    def test_check_grad_normal(self):
+        self.check_grad_with_place(self.place, ['X', 'Y'], 'Out')
+
+    def test_check_grad_ingore_x(self):
+        self.check_grad_with_place(
+            self.place, ['Y'], 'Out', no_grad_set=set("residual")
+        )
+
+    def test_check_grad_ingore_y(self):
+        self.check_grad_with_place(
+            self.place, ['X'], 'Out', no_grad_set=set('residual')
+        )
+
+
 if __name__ == '__main__':
     paddle.enable_static()
     unittest.main()
diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py
index 578a5da900e14..7d71178d4a3ee 100644
--- a/python/paddle/nn/functional/loss.py
+++ b/python/paddle/nn/functional/loss.py
@@ -1084,10 +1084,16 @@ def smooth_l1_loss(input, label, reduction='mean', delta=1.0, name=None):
         out = _C_ops.huber_loss(input, label, delta)
     else:
         check_variable_and_dtype(
-            input, 'input', ['float32', 'float64'], 'smooth_l1_loss'
+            input,
+            'input',
+            ['float16', 'float32', 'float64', 'uint16'],
+            'smooth_l1_loss',
         )
         check_variable_and_dtype(
-            label, 'label', ['float32', 'float64'], 'smooth_l1_loss'
+            label,
+            'label',
+            ['float16', 'float32', 'float64', 'uint16'],
+            'smooth_l1_loss',
         )
         helper = LayerHelper('huber_loss', **locals())
         residual = helper.create_variable_for_type_inference(

From d6d3de7900516a1c1e90dc98ba0c29de701b648f Mon Sep 17 00:00:00 2001
From: lzydev <lizhiyu02@baidu.com>
Date: Wed, 17 May 2023 10:57:04 +0800
Subject: [PATCH 370/405] fix bug of unpool in the module 'enet' of paddleseg
 (#53840)

---
 paddle/phi/api/yaml/backward.yaml        | 11 -----------
 paddle/phi/api/yaml/legacy_backward.yaml | 11 +++++++++++
 paddle/phi/api/yaml/legacy_ops.yaml      | 10 ++++++++++
 paddle/phi/api/yaml/ops.yaml             | 10 ----------
 paddle/phi/api/yaml/static_backward.yaml | 11 +++++++++++
 paddle/phi/api/yaml/static_ops.yaml      | 12 ++++++++++++
 6 files changed, 44 insertions(+), 21 deletions(-)

diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index 18738c1af8beb..1c38f5ba51c69 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -2173,14 +2173,3 @@
   kernel:
     func: unpool3d_grad
     data_type: x
-
-- backward_op: unpool_grad
-  forward: unpool (Tensor x, Tensor indices, int[] ksize, int[] strides = {1,1}, int[] paddings ={0,0} ,IntArray output_size = {0,0}, str data_format="NCHW") -> Tensor(out)
-  args: (Tensor x, Tensor indices, Tensor out, Tensor out_grad, int[] ksize, int[] strides, int[] paddings, IntArray output_size, str data_format)
-  output: Tensor(x_grad)
-  infer_meta:
-    func: UnchangedInferMeta
-    param : [x]
-  kernel:
-    func: unpool_grad
-    data_type: x
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index 87ed42986d45c..3b39db07136ed 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -1019,3 +1019,14 @@
     param : [out_grad]
   kernel :
     func : triu_grad
+
+- backward_op: unpool_grad
+  forward: unpool (Tensor x, Tensor indices, int[] ksize, int[] strides, int[] padding,  IntArray output_size, str data_format) -> Tensor(out)
+  args: (Tensor x, Tensor indices, Tensor out, Tensor out_grad, int[] ksize, int[] strides, int[] padding, IntArray output_size, str data_format)
+  output: Tensor(x_grad)
+  infer_meta:
+    func: UnchangedInferMeta
+    param : [x]
+  kernel:
+    func: unpool_grad
+    data_type: x
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index 7c113a65dcb43..0dff2ba6c24e3 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -1177,6 +1177,16 @@
     func : unique
     data_type : x
 
+- op : unpool
+  args: (Tensor x, Tensor indices, int[] ksize, int[] strides, int[] padding, IntArray output_size, str data_format)
+  output: Tensor(out)
+  infer_meta:
+    func: UnpoolInferMeta
+  kernel:
+    func: unpool
+    data_type: x
+  backward: unpool_grad
+
 - op : zeros
   args : (IntArray shape, DataType dtype=DataType::FLOAT32, Place place=CPUPlace())
   output : Tensor(out)
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index b7b2521a6a394..39799238364c2 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -2213,16 +2213,6 @@
     data_type : x
   optional : index, counts
 
-- op : unpool
-  args: (Tensor x, Tensor indices, int[] ksize, int[] strides = {1,1}, int[] paddings ={0,0} ,IntArray output_size = {0,0}, str data_format="NCHW")
-  output: Tensor(out)
-  infer_meta:
-    func: UnpoolInferMeta
-  kernel:
-    func: unpool
-    data_type: x
-  backward: unpool_grad
-
 - op : unpool3d
   args: (Tensor x, Tensor indices, int[] ksize, int[] strides={1,1,1}, int[] paddings={0,0,0}, int[] output_size={0,0,0}, str data_format="NCDHW")
   output: Tensor(out)
diff --git a/paddle/phi/api/yaml/static_backward.yaml b/paddle/phi/api/yaml/static_backward.yaml
index 5d48977ebabf9..3ca5260a03663 100755
--- a/paddle/phi/api/yaml/static_backward.yaml
+++ b/paddle/phi/api/yaml/static_backward.yaml
@@ -64,3 +64,14 @@
   kernel :
     func : softmax_grad
   composite : softmax_grad(out, out_grad, axis, x_grad)
+
+- backward_op: unpool_grad
+  forward: unpool (Tensor x, Tensor indices, int[] ksize, str unpooling_type, int[] strides = {1,1}, int[] paddings ={0,0} ,IntArray output_size = {0,0}, str data_format="NCHW") -> Tensor(out)
+  args: (Tensor x, Tensor indices, Tensor out, Tensor out_grad, int[] ksize, int[] strides, int[] paddings, IntArray output_size, str data_format)
+  output: Tensor(x_grad)
+  infer_meta:
+    func: UnchangedInferMeta
+    param : [x]
+  kernel:
+    func: unpool_grad
+    data_type: x
diff --git a/paddle/phi/api/yaml/static_ops.yaml b/paddle/phi/api/yaml/static_ops.yaml
index f481cedbc1c78..4576633c24866 100755
--- a/paddle/phi/api/yaml/static_ops.yaml
+++ b/paddle/phi/api/yaml/static_ops.yaml
@@ -398,3 +398,15 @@
   kernel :
     func : unique
     data_type : x
+
+- op : unpool
+  args: (Tensor x, Tensor indices, int[] ksize, str unpooling_type, int[] strides = {1,1}, int[] paddings ={0,0} ,IntArray output_size = {0,0}, str data_format="NCHW")
+  output: Tensor(out)
+  infer_meta:
+    func: UnpoolInferMeta
+    param : [x, indices, ksize, strides, paddings,output_size, data_format]
+  kernel:
+    func: unpool
+    data_type: x
+    param : [x, indices, ksize, strides, paddings,output_size, data_format]
+  backward: unpool_grad

From 89653668536140d0da5226306e841f30bef35500 Mon Sep 17 00:00:00 2001
From: Wilber <jiweibo@baidu.com>
Date: Wed, 17 May 2023 11:00:03 +0800
Subject: [PATCH 371/405] update openblas version (#53748)

* update openblas version

* update
---
 cmake/external/openblas.cmake | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake
index 2899119cde9a8..4f0767fb7253b 100644
--- a/cmake/external/openblas.cmake
+++ b/cmake/external/openblas.cmake
@@ -19,12 +19,14 @@ set(CBLAS_INSTALL_DIR ${THIRD_PARTY_PATH}/install/openblas)
 set(CBLAS_REPOSITORY ${GIT_URL}/xianyi/OpenBLAS.git)
 set(CBLAS_TAG v0.3.7)
 
-# Please refer to
-# https://github.com/PaddlePaddle/Paddle/pull/52983
-if(UNIX
-   AND NOT APPLE
-   AND NOT WITH_ROCM
-   AND NOT WITH_XPU)
+# Why use v0.3.18?  The IDG business line encountered a random openblas error,
+# which can be resolved after upgrading openblas.
+# And why compile when gcc>8.2? Please refer to
+# https://github.com/spack/spack/issues/19932#issuecomment-733452619
+# v0.3.18 only support gcc>=8.3 or gcc>=7.4
+if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND CMAKE_CXX_COMPILER_VERSION
+                                              VERSION_GREATER 8.2)
+  # We only compile with openblas 0.3.18 when gcc >= 8.3
   set(CBLAS_TAG v0.3.18)
 endif()
 

From a63fb4c881a7662c60fbd997b46fd5f35ac8466d Mon Sep 17 00:00:00 2001
From: LyndonKong <kongrui.chn@outlook.com>
Date: Wed, 17 May 2023 11:04:54 +0800
Subject: [PATCH 372/405] =?UTF-8?q?=E3=80=90Hackathon=204=20No.21=E3=80=91?=
 =?UTF-8?q?Add=20i1=20/=20i1e=20to=20paddle=20(#53210)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add i1 and i1e op

* resolve merge conflicts
---
 paddle/phi/api/yaml/backward.yaml             |  20 +++
 paddle/phi/api/yaml/ops.yaml                  |  18 +++
 paddle/phi/kernels/cpu/i1_grad_kernel.cc      |  44 +++++
 paddle/phi/kernels/cpu/i1_kernel.cc           |  37 +++++
 paddle/phi/kernels/cpu/i1e_grad_kernel.cc     |  44 +++++
 paddle/phi/kernels/cpu/i1e_kernel.cc          |  37 +++++
 paddle/phi/kernels/gpu/i1_grad_kernel.cu      |  37 +++++
 paddle/phi/kernels/gpu/i1_kernel.cu           |  32 ++++
 paddle/phi/kernels/gpu/i1e_grad_kernel.cu     |  37 +++++
 paddle/phi/kernels/gpu/i1e_kernel.cu          |  32 ++++
 paddle/phi/kernels/i1_grad_kernel.h           |  38 +++++
 paddle/phi/kernels/i1_kernel.h                |  32 ++++
 paddle/phi/kernels/i1e_grad_kernel.h          |  29 ++++
 paddle/phi/kernels/i1e_kernel.h               |  30 ++++
 .../impl/bessel_grad_kernel_cuda_impl.h       |  76 +++++++++
 .../kernels/impl/bessel_grad_kernel_impl.h    |  99 ++++++++++++
 .../kernels/impl/bessel_kernel_cuda_impl.h    |  48 ++++++
 paddle/phi/kernels/impl/bessel_kernel_impl.h  |  61 +++++++
 python/paddle/__init__.py                     |   4 +
 .../fluid/tests/unittests/test_i1_op.py       | 151 ++++++++++++++++++
 .../fluid/tests/unittests/test_i1e_op.py      | 151 ++++++++++++++++++
 python/paddle/tensor/__init__.py              |   4 +
 python/paddle/tensor/math.py                  |  67 ++++++++
 23 files changed, 1128 insertions(+)
 create mode 100644 paddle/phi/kernels/cpu/i1_grad_kernel.cc
 create mode 100644 paddle/phi/kernels/cpu/i1_kernel.cc
 create mode 100644 paddle/phi/kernels/cpu/i1e_grad_kernel.cc
 create mode 100644 paddle/phi/kernels/cpu/i1e_kernel.cc
 create mode 100644 paddle/phi/kernels/gpu/i1_grad_kernel.cu
 create mode 100644 paddle/phi/kernels/gpu/i1_kernel.cu
 create mode 100644 paddle/phi/kernels/gpu/i1e_grad_kernel.cu
 create mode 100644 paddle/phi/kernels/gpu/i1e_kernel.cu
 create mode 100644 paddle/phi/kernels/i1_grad_kernel.h
 create mode 100644 paddle/phi/kernels/i1_kernel.h
 create mode 100644 paddle/phi/kernels/i1e_grad_kernel.h
 create mode 100644 paddle/phi/kernels/i1e_kernel.h
 create mode 100644 python/paddle/fluid/tests/unittests/test_i1_op.py
 create mode 100644 python/paddle/fluid/tests/unittests/test_i1e_op.py

diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index 1c38f5ba51c69..895f0ccb112af 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -821,6 +821,26 @@
   kernel :
     func : i0e_grad
 
+- backward_op : i1_grad
+  forward : i1 (Tensor x) -> Tensor(out)
+  args : (Tensor x, Tensor out, Tensor out_grad)
+  output : Tensor(x_grad)
+  infer_meta :
+    func : UnchangedInferMeta
+    param : [x]
+  kernel :
+    func : i1_grad
+
+- backward_op : i1e_grad
+  forward : i1e (Tensor x) -> Tensor(out)
+  args : (Tensor x, Tensor out, Tensor out_grad)
+  output : Tensor(x_grad)
+  infer_meta :
+    func : UnchangedInferMeta
+    param : [x]
+  kernel :
+    func : i1e_grad
+
 - backward_op : imag_grad
   forward : imag (Tensor x) -> Tensor(out)
   args : (Tensor out_grad)
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index 39799238364c2..8928a825e4636 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -948,6 +948,24 @@
     func : i0e
   backward : i0e_grad
 
+- op : i1
+  args : (Tensor x)
+  output : Tensor(out)
+  infer_meta :
+    func : UnchangedInferMeta
+  kernel :
+    func : i1
+  backward : i1_grad
+
+- op : i1e
+  args : (Tensor x)
+  output : Tensor(out)
+  infer_meta :
+    func : UnchangedInferMeta
+  kernel :
+    func : i1e
+  backward : i1e_grad
+
 - op : imag
   args : (Tensor x)
   output : Tensor (out)
diff --git a/paddle/phi/kernels/cpu/i1_grad_kernel.cc b/paddle/phi/kernels/cpu/i1_grad_kernel.cc
new file mode 100644
index 0000000000000..3a4f140e19039
--- /dev/null
+++ b/paddle/phi/kernels/cpu/i1_grad_kernel.cc
@@ -0,0 +1,44 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/i1_grad_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
+#include "paddle/phi/kernels/impl/bessel_grad_kernel_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I1GradKernel(const Context& ctx,
+                  const DenseTensor& x,
+                  const DenseTensor& out,
+                  const DenseTensor& out_grad,
+                  DenseTensor* x_grad) {
+  const int64_t size = x.numel();
+  const T* x_data = x.data<T>();
+  const T* out_data = out.data<T>();
+  const T* out_grad_data = out_grad.data<T>();
+  T* x_grad_data = ctx.template Alloc<T>(x_grad);
+
+  phi::funcs::ForRange<Context> for_range(ctx, size);
+  I1GradFunctor<T> functor(x_data, out_data, out_grad_data, x_grad_data, size);
+  for_range(functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(i1_grad, CPU, ALL_LAYOUT, phi::I1GradKernel, float, double) {
+}
diff --git a/paddle/phi/kernels/cpu/i1_kernel.cc b/paddle/phi/kernels/cpu/i1_kernel.cc
new file mode 100644
index 0000000000000..18388159a9b6f
--- /dev/null
+++ b/paddle/phi/kernels/cpu/i1_kernel.cc
@@ -0,0 +1,37 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/i1_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
+#include "paddle/phi/kernels/impl/bessel_kernel_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I1Kernel(const Context& ctx, const DenseTensor& x, DenseTensor* out) {
+  const int64_t size = x.numel();
+  const T* x_data = x.data<T>();
+  T* out_data = ctx.template Alloc<T>(out);
+
+  phi::funcs::ForRange<Context> for_range(ctx, size);
+  I1Functor<T> functor(x_data, out_data, size);
+  for_range(functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(i1, CPU, ALL_LAYOUT, phi::I1Kernel, float, double) {}
diff --git a/paddle/phi/kernels/cpu/i1e_grad_kernel.cc b/paddle/phi/kernels/cpu/i1e_grad_kernel.cc
new file mode 100644
index 0000000000000..e97d12c4c1256
--- /dev/null
+++ b/paddle/phi/kernels/cpu/i1e_grad_kernel.cc
@@ -0,0 +1,44 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/i1e_grad_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
+#include "paddle/phi/kernels/impl/bessel_grad_kernel_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I1eGradKernel(const Context& ctx,
+                   const DenseTensor& x,
+                   const DenseTensor& out,
+                   const DenseTensor& out_grad,
+                   DenseTensor* x_grad) {
+  const int64_t size = x.numel();
+  const T* x_data = x.data<T>();
+  const T* out_data = out.data<T>();
+  const T* out_grad_data = out_grad.data<T>();
+  T* x_grad_data = ctx.template Alloc<T>(x_grad);
+
+  phi::funcs::ForRange<Context> for_range(ctx, size);
+  I1eGradFunctor<T> functor(x_data, out_data, out_grad_data, x_grad_data, size);
+  for_range(functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(
+    i1e_grad, CPU, ALL_LAYOUT, phi::I1eGradKernel, float, double) {}
diff --git a/paddle/phi/kernels/cpu/i1e_kernel.cc b/paddle/phi/kernels/cpu/i1e_kernel.cc
new file mode 100644
index 0000000000000..5f50d723e9a50
--- /dev/null
+++ b/paddle/phi/kernels/cpu/i1e_kernel.cc
@@ -0,0 +1,37 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/i1e_kernel.h"
+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/for_range.h"
+#include "paddle/phi/kernels/impl/bessel_kernel_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I1eKernel(const Context& ctx, const DenseTensor& x, DenseTensor* out) {
+  const int64_t size = x.numel();
+  const T* x_data = x.data<T>();
+  T* out_data = ctx.template Alloc<T>(out);
+
+  phi::funcs::ForRange<Context> for_range(ctx, size);
+  I1eFunctor<T> functor(x_data, out_data, size);
+  for_range(functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(i1e, CPU, ALL_LAYOUT, phi::I1eKernel, float, double) {}
diff --git a/paddle/phi/kernels/gpu/i1_grad_kernel.cu b/paddle/phi/kernels/gpu/i1_grad_kernel.cu
new file mode 100644
index 0000000000000..35ab69a176ee2
--- /dev/null
+++ b/paddle/phi/kernels/gpu/i1_grad_kernel.cu
@@ -0,0 +1,37 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/i1_grad_kernel.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/elementwise_base.h"
+#include "paddle/phi/kernels/impl/bessel_grad_kernel_cuda_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I1GradKernel(const Context& ctx,
+                  const DenseTensor& x,
+                  const DenseTensor& out,
+                  const DenseTensor& out_grad,
+                  DenseTensor* x_grad) {
+  ctx.template Alloc<T>(x_grad);
+  std::vector<const DenseTensor*> ins = {&x, &out, &out_grad};
+  std::vector<DenseTensor*> outs = {x_grad};
+  auto functor = CudaI1GradFunctor<T>();
+  phi::funcs::ElementwiseKernel<T>(ctx, ins, &outs, functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(i1_grad, GPU, ALL_LAYOUT, phi::I1GradKernel, float, double) {
+}
diff --git a/paddle/phi/kernels/gpu/i1_kernel.cu b/paddle/phi/kernels/gpu/i1_kernel.cu
new file mode 100644
index 0000000000000..c0387f451e598
--- /dev/null
+++ b/paddle/phi/kernels/gpu/i1_kernel.cu
@@ -0,0 +1,32 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/i1_kernel.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/elementwise_base.h"
+#include "paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I1Kernel(const Context& ctx, const DenseTensor& x, DenseTensor* out) {
+  ctx.template Alloc<T>(out);
+  std::vector<const DenseTensor*> ins = {&x};
+  std::vector<DenseTensor*> outs = {out};
+  auto functor = CudaI1Functor<T>();
+  phi::funcs::ElementwiseKernel<T>(ctx, ins, &outs, functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(i1, GPU, ALL_LAYOUT, phi::I1Kernel, float, double) {}
diff --git a/paddle/phi/kernels/gpu/i1e_grad_kernel.cu b/paddle/phi/kernels/gpu/i1e_grad_kernel.cu
new file mode 100644
index 0000000000000..a01a3ab08f496
--- /dev/null
+++ b/paddle/phi/kernels/gpu/i1e_grad_kernel.cu
@@ -0,0 +1,37 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/i1e_grad_kernel.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/elementwise_base.h"
+#include "paddle/phi/kernels/impl/bessel_grad_kernel_cuda_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I1eGradKernel(const Context& ctx,
+                   const DenseTensor& x,
+                   const DenseTensor& out,
+                   const DenseTensor& out_grad,
+                   DenseTensor* x_grad) {
+  ctx.template Alloc<T>(x_grad);
+  std::vector<const DenseTensor*> ins = {&x, &out, &out_grad};
+  std::vector<DenseTensor*> outs = {x_grad};
+  auto functor = CudaI1eGradFunctor<T>();
+  phi::funcs::ElementwiseKernel<T>(ctx, ins, &outs, functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(
+    i1e_grad, GPU, ALL_LAYOUT, phi::I1eGradKernel, float, double) {}
diff --git a/paddle/phi/kernels/gpu/i1e_kernel.cu b/paddle/phi/kernels/gpu/i1e_kernel.cu
new file mode 100644
index 0000000000000..8dff432a628ef
--- /dev/null
+++ b/paddle/phi/kernels/gpu/i1e_kernel.cu
@@ -0,0 +1,32 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/i1e_kernel.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/elementwise_base.h"
+#include "paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I1eKernel(const Context& ctx, const DenseTensor& x, DenseTensor* out) {
+  ctx.template Alloc<T>(out);
+  std::vector<const DenseTensor*> ins = {&x};
+  std::vector<DenseTensor*> outs = {out};
+  auto functor = CudaI1eFunctor<T>();
+  phi::funcs::ElementwiseKernel<T>(ctx, ins, &outs, functor);
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(i1e, GPU, ALL_LAYOUT, phi::I1eKernel, float, double) {}
diff --git a/paddle/phi/kernels/i1_grad_kernel.h b/paddle/phi/kernels/i1_grad_kernel.h
new file mode 100644
index 0000000000000..1d95c10b6c94c
--- /dev/null
+++ b/paddle/phi/kernels/i1_grad_kernel.h
@@ -0,0 +1,38 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/device_context.h"
+
+namespace phi {
+
+/**
+ * @brief This kernel calculate gradient of Modified Bessel function of order 1.
+ * @param  ctx   device context
+ * @param  x
+ * @param  out
+ * @param  out_grad
+ * @param  x_grad
+ */
+
+template <typename T, typename Context>
+void I1GradKernel(const Context& ctx,
+                  const DenseTensor& x,
+                  const DenseTensor& out,
+                  const DenseTensor& out_grad,
+                  DenseTensor* x_grad);
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/i1_kernel.h b/paddle/phi/kernels/i1_kernel.h
new file mode 100644
index 0000000000000..8865c573c9a19
--- /dev/null
+++ b/paddle/phi/kernels/i1_kernel.h
@@ -0,0 +1,32 @@
+// Copyright (c) 2023PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+
+/**
+ * @brief This kernel calculate Modified Bessel function of order 1.
+ * @param  ctx   device context
+ * @param  x     The input tensor of i1
+ * @param  out   The output tensor of i1 kernel, it has the same shape and
+ *               dtype with input. Each element corresponds to input tensor
+ */
+
+template <typename T, typename Context>
+void I1Kernel(const Context& ctx, const DenseTensor& x, DenseTensor* out);
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/i1e_grad_kernel.h b/paddle/phi/kernels/i1e_grad_kernel.h
new file mode 100644
index 0000000000000..492305022d0a3
--- /dev/null
+++ b/paddle/phi/kernels/i1e_grad_kernel.h
@@ -0,0 +1,29 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/device_context.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void I1eGradKernel(const Context& ctx,
+                   const DenseTensor& x,
+                   const DenseTensor& out,
+                   const DenseTensor& out_grad,
+                   DenseTensor* x_grad);
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/i1e_kernel.h b/paddle/phi/kernels/i1e_kernel.h
new file mode 100644
index 0000000000000..6ca3d9f767fe2
--- /dev/null
+++ b/paddle/phi/kernels/i1e_kernel.h
@@ -0,0 +1,30 @@
+// Copyright (c) 2023PaddlePaddle Authors. All Rights Reserved.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace phi {
+
+/**
+ * @brief This kernel calculate Exponentially scaled modified Bessel function of
+ * order 1..
+ * @param  ctx   device context
+ * @param  x     The input tensor of i1e
+ * @param  out   The output tensor of i1e kernel, it has the same shape and
+ *               dtype with input. Each element corresponds to input tensor
+ */
+
+template <typename T, typename Context>
+void I1eKernel(const Context& ctx, const DenseTensor& x, DenseTensor* out);
+
+}  // namespace phi
diff --git a/paddle/phi/kernels/impl/bessel_grad_kernel_cuda_impl.h b/paddle/phi/kernels/impl/bessel_grad_kernel_cuda_impl.h
index 7789e3d8e9312..b3e272800b482 100644
--- a/paddle/phi/kernels/impl/bessel_grad_kernel_cuda_impl.h
+++ b/paddle/phi/kernels/impl/bessel_grad_kernel_cuda_impl.h
@@ -87,4 +87,80 @@ struct CudaI0eGradFunctor {
   }
 };
 
+template <typename T>
+struct CudaI1GradFunctor {
+  __device__ __forceinline__ T operator()(const T _x,
+                                          const T _out,
+                                          const T _out_grad) const {
+    using MT = typename phi::dtype::MPTypeTrait<T>::Type;
+    const MT mp_x = static_cast<MT>(_x);
+    const MT mp_out = static_cast<MT>(_out);
+    const MT mp_out_grad = static_cast<MT>(_out_grad);
+    MT x = std::abs(mp_x);
+    if (x <= MT{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI0e_A<MT>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      MT y = (x / MT{2.0}) - MT{2.0};
+      MT eps = static_cast<MT>(std::numeric_limits<T>::epsilon());
+
+      if (x <= eps) {
+        MT out = (MT{0.5}) * mp_out_grad;
+        return static_cast<T>(out);
+      } else {
+        return static_cast<T>(
+            (std::exp(x) * Chbevl<MT>(y, A, len) - mp_out / mp_x) *
+            mp_out_grad);
+      }
+    }
+    auto coeff_pair_B = ChebyshevCoefficientsI0e_B<MT>();
+    auto B = std::get<0>(coeff_pair_B);
+    auto len = std::get<1>(coeff_pair_B);
+    MT y = (MT{32.0} / x) - MT{2.0};
+
+    return static_cast<T>(
+        (std::exp(x) * Chbevl<MT>(y, B, len) / std::sqrt(x) - mp_out / mp_x) *
+        mp_out_grad);
+  }
+};
+
+template <typename T>
+struct CudaI1eGradFunctor {
+  __device__ __forceinline__ T operator()(const T _x,
+                                          const T _out,
+                                          const T _out_grad) const {
+    using MT = typename phi::dtype::MPTypeTrait<T>::Type;
+    const MT mp_x = static_cast<MT>(_x);
+    const MT mp_out = static_cast<MT>(_out);
+    const MT mp_out_grad = static_cast<MT>(_out_grad);
+    MT x = std::abs(mp_x);
+    if (x <= MT{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI0e_A<MT>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      MT y = (x / MT{2.0}) - MT{2.0};
+      MT eps = static_cast<MT>(std::numeric_limits<T>::epsilon());
+
+      if (x <= eps) {
+        MT out = (MT{0.5}) * mp_out_grad;
+        return static_cast<T>(out);
+      } else {
+        MT out = (Chbevl<MT>(y, A, len) -
+                  mp_out * (std::copysign(MT{1.0}, mp_x) + (MT{1.0}) / mp_x)) *
+                 mp_out_grad;
+        return static_cast<T>(out);
+      }
+    }
+    auto coeff_pair_B = ChebyshevCoefficientsI0e_B<MT>();
+    auto B = std::get<0>(coeff_pair_B);
+    auto len = std::get<1>(coeff_pair_B);
+    MT y = (MT{32.0} / x) - MT{2.0};
+
+    return static_cast<T>(
+        (Chbevl<T>(y, B, len) / std::sqrt(x) -
+         mp_out * (std::copysign(MT{1.0}, mp_x) + (MT{1.0}) / mp_x)) *
+        mp_out_grad);
+  }
+};
+
 }  // namespace phi
diff --git a/paddle/phi/kernels/impl/bessel_grad_kernel_impl.h b/paddle/phi/kernels/impl/bessel_grad_kernel_impl.h
index 1de9a9fcbfc4b..e48c0c8ec115f 100644
--- a/paddle/phi/kernels/impl/bessel_grad_kernel_impl.h
+++ b/paddle/phi/kernels/impl/bessel_grad_kernel_impl.h
@@ -109,4 +109,103 @@ struct I0eGradFunctor {
   int64_t numel_;
 };
 
+template <typename T>
+struct I1GradFunctor {
+  I1GradFunctor(
+      const T* x, const T* out, const T* out_grad, T* x_grad, int64_t numel)
+      : input_x_(x),
+        input_out_(out),
+        input_out_grad_(out_grad),
+        output_x_grad_(x_grad),
+        numel_(numel) {}
+
+  HOSTDEVICE void operator()(int64_t idx) const {
+    T x = std::abs(input_x_[idx]);
+    T x_ = input_x_[idx];
+    T out_ = input_out_[idx];
+    T out_grad_ = input_out_grad_[idx];
+    if (x <= T{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI0e_A<T>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      T y = (x / T{2.0}) - T{2.0};
+      T eps = std::numeric_limits<T>::epsilon();
+
+      if (x <= eps) {
+        output_x_grad_[idx] = static_cast<T>(T{0.5} * out_grad_);
+      } else {
+        output_x_grad_[idx] = static_cast<T>(
+            (std::exp(x) * Chbevl<T>(y, A, len) - out_ / x_) * out_grad_);
+      }
+    } else {
+      auto coeff_pair_B = ChebyshevCoefficientsI0e_B<T>();
+      auto B = std::get<0>(coeff_pair_B);
+      auto len = std::get<1>(coeff_pair_B);
+      T y = (T{32.0} / x) - T{2.0};
+
+      output_x_grad_[idx] = static_cast<T>(
+          (std::exp(x) * Chbevl<T>(y, B, len) / std::sqrt(x) - out_ / x_) *
+          out_grad_);
+    }
+  }
+
+ private:
+  const T* input_x_;
+  const T* input_out_;
+  const T* input_out_grad_;
+  T* output_x_grad_;
+  int64_t numel_;
+};
+
+template <typename T>
+struct I1eGradFunctor {
+  I1eGradFunctor(
+      const T* x, const T* out, const T* out_grad, T* x_grad, int64_t numel)
+      : input_x_(x),
+        input_out_(out),
+        input_out_grad_(out_grad),
+        output_x_grad_(x_grad),
+        numel_(numel) {}
+
+  HOSTDEVICE void operator()(int64_t idx) const {
+    T x = std::abs(input_x_[idx]);
+    T x_ = input_x_[idx];
+    T out_ = input_out_[idx];
+    T out_grad_ = input_out_grad_[idx];
+    if (x <= T{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI0e_A<T>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      T y = (x / T{2.0}) - T{2.0};
+      T eps = std::numeric_limits<T>::epsilon();
+
+      if (x <= eps) {
+        output_x_grad_[idx] = static_cast<T>(T{0.5} * out_grad_);
+      } else {
+        output_x_grad_[idx] =
+            static_cast<T>((Chbevl<T>(y, A, len) -
+                            out_ * (std::copysign(T{1.0}, x_) + T{1.0} / x_)) *
+                           out_grad_);
+      }
+    } else {
+      auto coeff_pair_B = ChebyshevCoefficientsI0e_B<T>();
+      auto B = std::get<0>(coeff_pair_B);
+      auto len = std::get<1>(coeff_pair_B);
+      T y = (T{32.0} / x) - T{2.0};
+
+      output_x_grad_[idx] =
+          static_cast<T>((Chbevl<T>(y, B, len) / std::sqrt(x) -
+                          out_ * (std::copysign(T{1.0}, x_) + T{1.0} / x_)) *
+                         out_grad_);
+    }
+  }
+
+ private:
+  const T* input_x_;
+  const T* input_out_;
+  const T* input_out_grad_;
+  T* output_x_grad_;
+  int64_t numel_;
+};
+
 }  // namespace phi
diff --git a/paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h b/paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h
index 32649c901906b..63ae22d046a30 100644
--- a/paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h
+++ b/paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h
@@ -228,4 +228,52 @@ ChebyshevCoefficientsI1e_B() {
   return std::make_tuple(coeff, 7);
 }
 
+template <typename T>
+struct CudaI1Functor {
+  __device__ __forceinline__ T operator()(const T _x) const {
+    using MT = typename phi::dtype::MPTypeTrait<T>::Type;
+    const MT mp_x = static_cast<MT>(_x);
+    MT x = std::abs(mp_x);
+    if (x <= MT{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI1e_A<MT>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      MT y = (x / MT{2.0}) - MT{2.0};
+      const T out = std::exp(x) * x * Chbevl<MT>(y, A, len);
+      return (mp_x < MT{0.0}) ? -out : out;
+    }
+    auto coeff_pair_B = ChebyshevCoefficientsI1e_B<MT>();
+    auto B = std::get<0>(coeff_pair_B);
+    auto len = std::get<1>(coeff_pair_B);
+    MT y = (MT{32.0} / x) - MT{2.0};
+    const T out = (std::exp(x) * Chbevl<MT>(y, B, len)) / std::sqrt(x);
+    return (mp_x < MT{0.0}) ? -out : out;
+  }
+};
+
+template <typename T>
+struct CudaI1eFunctor {
+  __device__ __forceinline__ T operator()(const T _x) const {
+    using MT = typename phi::dtype::MPTypeTrait<T>::Type;
+    const MT mp_x = static_cast<MT>(_x);
+    MT x = std::abs(mp_x);
+    if (x <= MT{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI1e_A<MT>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      MT y = (x / MT{2.0}) - MT{2.0};
+
+      const T out = static_cast<T>(Chbevl<T>(y, A, len) * x);
+      return (mp_x < MT{0.0}) ? -out : out;
+    }
+    auto coeff_pair_B = ChebyshevCoefficientsI1e_B<MT>();
+    auto B = std::get<0>(coeff_pair_B);
+    auto len = std::get<1>(coeff_pair_B);
+    MT y = (MT{32.0} / x) - MT{2.0};
+
+    const T out = static_cast<T>(Chbevl<T>(y, B, len) / std::sqrt(x));
+    return (mp_x < MT{0.0}) ? -out : out;
+  }
+};
+
 }  // namespace phi
diff --git a/paddle/phi/kernels/impl/bessel_kernel_impl.h b/paddle/phi/kernels/impl/bessel_kernel_impl.h
index 1df090753930d..12eae74ce96e1 100644
--- a/paddle/phi/kernels/impl/bessel_kernel_impl.h
+++ b/paddle/phi/kernels/impl/bessel_kernel_impl.h
@@ -251,4 +251,65 @@ ChebyshevCoefficientsI1e_B() {
   return std::make_tuple(coeff, 7);
 }
 
+template <typename T>
+struct I1Functor {
+  I1Functor(const T* input, T* output, int64_t numel)
+      : input_(input), output_(output), numel_(numel) {}
+
+  HOSTDEVICE void operator()(int64_t idx) const {
+    T x = std::abs(input_[idx]);
+    if (x <= T{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI1e_A<T>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      T y = (x / T{2.0}) - T{2.0};
+      const T out = std::exp(x) * x * Chbevl(y, A, len);
+      output_[idx] = (input_[idx] < T{0.0}) ? -out : out;
+    } else {
+      auto coeff_pair_B = ChebyshevCoefficientsI1e_B<T>();
+      auto B = std::get<0>(coeff_pair_B);
+      auto len = std::get<1>(coeff_pair_B);
+      T y = (T{32.0} / x) - T{2.0};
+      const T out = (std::exp(x) * Chbevl(y, B, len)) / std::sqrt(x);
+      output_[idx] = (input_[idx] < T{0.0}) ? -out : out;
+    }
+  }
+
+ private:
+  const T* input_;
+  T* output_;
+  int64_t numel_;
+};
+
+template <typename T>
+struct I1eFunctor {
+  I1eFunctor(const T* input, T* output, int64_t numel)
+      : input_(input), output_(output), numel_(numel) {}
+
+  HOSTDEVICE void operator()(int64_t idx) const {
+    T x = std::abs(input_[idx]);
+    if (x <= T{8.0}) {
+      auto coeff_pair_A = ChebyshevCoefficientsI1e_A<T>();
+      auto A = std::get<0>(coeff_pair_A);
+      auto len = std::get<1>(coeff_pair_A);
+      T y = (x / T{2.0}) - T{2.0};
+      const T out = Chbevl<T>(y, A, len) * x;
+      output_[idx] = (input_[idx] < T{0.0}) ? -out : out;
+    } else {
+      auto coeff_pair_B = ChebyshevCoefficientsI1e_B<T>();
+      auto B = std::get<0>(coeff_pair_B);
+      auto len = std::get<1>(coeff_pair_B);
+      T y = (T{32.0} / x) - T{2.0};
+
+      const T out = Chbevl<T>(y, B, len) / std::sqrt(x);
+      output_[idx] = (input_[idx] < T{0.0}) ? -out : out;
+    }
+  }
+
+ private:
+  const T* input_;
+  T* output_;
+  int64_t numel_;
+};
+
 }  // namespace phi
diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py
index 8626e5e9f37ff..f878a6b8d7f7e 100644
--- a/python/paddle/__init__.py
+++ b/python/paddle/__init__.py
@@ -306,6 +306,8 @@
 from .tensor.math import nextafter  # noqa: F401
 from .tensor.math import i0  # noqa: F401
 from .tensor.math import i0e  # noqa: F401
+from .tensor.math import i1  # noqa: F401
+from .tensor.math import i1e  # noqa: F401
 
 from .tensor.random import bernoulli  # noqa: F401
 from .tensor.random import poisson  # noqa: F401
@@ -702,4 +704,6 @@
     'nextafter',
     'i0',
     'i0e',
+    'i1',
+    'i1e',
 ]
diff --git a/python/paddle/fluid/tests/unittests/test_i1_op.py b/python/paddle/fluid/tests/unittests/test_i1_op.py
new file mode 100644
index 0000000000000..b5846d6e15647
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_i1_op.py
@@ -0,0 +1,151 @@
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from eager_op_test import OpTest
+from scipy import special
+
+import paddle
+from paddle.fluid import core
+
+np.random.seed(42)
+paddle.seed(42)
+
+
+def reference_i1(x):
+    return special.i1(x)
+
+
+def reference_i1_grad(x, dout):
+    eps = np.finfo(x.dtype).eps
+    not_tiny = abs(x) > eps
+    safe_x = np.where(not_tiny, x, eps)
+    gradx = special.i0(safe_x) - special.i1(x) / safe_x
+    gradx = np.where(not_tiny, gradx, 0.5)
+    return dout * gradx
+
+
+class Testi1API(unittest.TestCase):
+    DTYPE = "float64"
+    DATA = [0, 1, 2, 3, 4, 5]
+
+    def setUp(self):
+        self.x = np.array(self.DATA).astype(self.DTYPE)
+        self.place = [paddle.CPUPlace()]
+        if core.is_compiled_with_cuda():
+            self.place.append(paddle.CUDAPlace(0))
+
+    def test_api_static(self):
+        def run(place):
+            paddle.enable_static()
+            with paddle.static.program_guard(paddle.static.Program()):
+                x = paddle.static.data(
+                    name="x", shape=self.x.shape, dtype=self.DTYPE
+                )
+                out = paddle.i1(x)
+                exe = paddle.static.Executor(place)
+                res = exe.run(
+                    paddle.static.default_main_program(),
+                    feed={"x": self.x},
+                    fetch_list=[out],
+                )
+                out_ref = reference_i1(self.x)
+                np.testing.assert_allclose(res[0], out_ref, rtol=1e-5)
+            paddle.disable_static()
+
+        for place in self.place:
+            run(place)
+
+    def test_api_dygraph(self):
+        def run(place):
+            paddle.disable_static(place)
+            x = paddle.to_tensor(self.x)
+            out = paddle.i1(x)
+
+            out_ref = reference_i1(self.x)
+            np.testing.assert_allclose(out.numpy(), out_ref, rtol=1e-5)
+            paddle.enable_static()
+
+        for place in self.place:
+            run(place)
+
+    def test_empty_input_error(self):
+        for place in self.place:
+            paddle.disable_static(place)
+            x = None
+            self.assertRaises(ValueError, paddle.i1, x)
+            paddle.enable_static()
+
+
+class Testi1Float32Zero2EightCase(Testi1API):
+    DTYPE = "float32"
+    DATA = [0, 1, 2, 3, 4, 5, 6, 7, 8]
+
+
+class Testi1Float32OverEightCase(Testi1API):
+    DTYPE = "float32"
+    DATA = [9, 10, 11, 12, 13, 14, 15, 16, 17]
+
+
+class Testi1Float64Zero2EightCase(Testi1API):
+    DTYPE = "float64"
+    DATA = [0, 1, 2, 3, 4, 5, 6, 7, 8]
+
+
+class Testi1Float64OverEightCase(Testi1API):
+    DTYPE = "float64"
+    DATA = [9, 10, 11, 12, 13, 14, 15, 16, 17]
+
+
+class TestI1Op(OpTest):
+    # 配置 op 信息以及输入输出等参数
+    def setUp(self):
+        self.op_type = "i1"
+        self.python_api = paddle.i1
+        self.init_config()
+        self.outputs = {'out': self.target}
+
+    # 测试前向输出结果
+    def test_check_output(self):
+        self.check_output()
+
+    # 测试反向梯度输出
+    def test_check_grad(self):
+        self.check_grad(
+            ['x'],
+            'out',
+            user_defined_grads=[
+                reference_i1_grad(
+                    self.case,
+                    1 / self.case.size,
+                )
+            ],
+        )
+
+    def init_config(self):
+        # 生成随机的输入数据
+        zero_case = np.zeros(1).astype('float64')
+        rand_case = np.random.randn(250).astype('float64')
+        over_eight_case = np.random.uniform(low=8, high=9, size=250).astype(
+            'float64'
+        )
+        self.case = np.concatenate([zero_case, rand_case, over_eight_case])
+        self.inputs = {'x': self.case}
+        self.target = reference_i1(self.inputs['x'])
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/python/paddle/fluid/tests/unittests/test_i1e_op.py b/python/paddle/fluid/tests/unittests/test_i1e_op.py
new file mode 100644
index 0000000000000..a4c360ae9a36f
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_i1e_op.py
@@ -0,0 +1,151 @@
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from eager_op_test import OpTest
+from scipy import special
+
+import paddle
+from paddle.fluid import core
+
+np.random.seed(42)
+paddle.seed(42)
+
+
+def reference_i1e(x):
+    return special.i1e(x)
+
+
+def reference_i1e_grad(x, dout):
+    eps = np.finfo(x.dtype).eps
+    not_tiny = abs(x) > eps
+    safe_x = np.where(not_tiny, x, eps)
+    gradx = special.i0e(safe_x) - special.i1e(x) * (np.sign(x) + 1 / safe_x)
+    gradx = np.where(not_tiny, gradx, 0.5)
+    return dout * gradx
+
+
+class TestI1e_API(unittest.TestCase):
+    DTYPE = "float64"
+    DATA = [0, 1, 2, 3, 4, 5]
+
+    def setUp(self):
+        self.x = np.array(self.DATA).astype(self.DTYPE)
+        self.place = [paddle.CPUPlace()]
+        if core.is_compiled_with_cuda():
+            self.place.append(paddle.CUDAPlace(0))
+
+    def test_api_static(self):
+        def run(place):
+            paddle.enable_static()
+            with paddle.static.program_guard(paddle.static.Program()):
+                x = paddle.static.data(
+                    name="x", shape=self.x.shape, dtype=self.DTYPE
+                )
+                y = paddle.i1e(x)
+                exe = paddle.static.Executor(place)
+                res = exe.run(
+                    paddle.static.default_main_program(),
+                    feed={"x": self.x},
+                    fetch_list=[y],
+                )
+                out_ref = reference_i1e(self.x)
+                np.testing.assert_allclose(out_ref, res[0], rtol=1e-5)
+            paddle.disable_static()
+
+        for place in self.place:
+            run(place)
+
+    def test_api_dygraph(self):
+        def run(place):
+            paddle.disable_static(place)
+            x = paddle.to_tensor(self.x)
+            out = paddle.i1e(x)
+
+            out_ref = reference_i1e(self.x)
+            np.testing.assert_allclose(out_ref, out.numpy(), rtol=1e-5)
+            paddle.enable_static()
+
+        for place in self.place:
+            run(place)
+
+    def test_empty_input_error(self):
+        for place in self.place:
+            paddle.disable_static(place)
+            x = None
+            self.assertRaises(ValueError, paddle.i1e, x)
+            paddle.enable_static()
+
+
+class Testi1eFloat32Zero2EightCase(TestI1e_API):
+    DTYPE = "float32"
+    DATA = [0, 1, 2, 3, 4, 5, 6, 7, 8]
+
+
+class Testi1eFloat32OverEightCase(TestI1e_API):
+    DTYPE = "float32"
+    DATA = [9, 10, 11, 12, 13, 14, 15, 16, 17]
+
+
+class Testi1eFloat64Zero2EightCase(TestI1e_API):
+    DTYPE = "float64"
+    DATA = [0, 1, 2, 3, 4, 5, 6, 7, 8]
+
+
+class Testi1eFloat64OverEightCase(TestI1e_API):
+    DTYPE = "float64"
+    DATA = [9, 10, 11, 12, 13, 14, 15, 16, 17]
+
+
+class TestI1eOp(OpTest):
+    # 配置 op 信息以及输入输出等参数
+    def setUp(self):
+        self.op_type = "i1e"
+        self.python_api = paddle.i1e
+        self.init_config()
+        self.outputs = {'out': self.target}
+
+    # 测试前向输出结果
+    def test_check_output(self):
+        self.check_output()
+
+    # 测试反向梯度输出
+    def test_check_grad(self):
+        self.check_grad(
+            ['x'],
+            'out',
+            user_defined_grads=[
+                reference_i1e_grad(
+                    self.case,
+                    1 / self.case.size,
+                )
+            ],
+        )
+
+    # 生成随机的输入数据并计算对应输出
+    def init_config(self):
+        zero_case = np.zeros(1).astype('float64')
+        rand_case = np.random.randn(250).astype('float64')
+        over_eight_case = np.random.uniform(low=8, high=9, size=250).astype(
+            'float64'
+        )
+        self.case = np.concatenate([zero_case, rand_case, over_eight_case])
+        self.inputs = {'x': self.case}
+        self.target = reference_i1e(self.inputs['x'])
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/python/paddle/tensor/__init__.py b/python/paddle/tensor/__init__.py
index 1e26e4ce56d13..ba13034fc56ed 100644
--- a/python/paddle/tensor/__init__.py
+++ b/python/paddle/tensor/__init__.py
@@ -258,6 +258,8 @@
 from .math import nextafter  # noqa: F401
 from .math import i0  # noqa: F401
 from .math import i0e  # noqa: F401
+from .math import i1  # noqa: F401
+from .math import i1e  # noqa: F401
 
 from .random import multinomial  # noqa: F401
 from .random import standard_normal  # noqa: F401
@@ -554,6 +556,8 @@
     'unflatten',
     'i0',
     'i0e',
+    'i1',
+    'i1e',
 ]
 
 # this list used in math_op_patch.py for magic_method bind
diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py
index debf74e87ac45..f3635a69651e1 100644
--- a/python/paddle/tensor/math.py
+++ b/python/paddle/tensor/math.py
@@ -5657,3 +5657,70 @@ def i0e(x, name=None):
         out = helper.create_variable_for_type_inference(dtype=x.dtype)
         helper.append_op(type='i0e', inputs={'x': x}, outputs={'out': out})
     return out
+
+
+def i1(x, name=None):
+    """
+    The function is used to calculate modified bessel function of order 1.
+
+    Args:
+        x (Tensor): The input tensor, it's data type should be float32, float64.
+        name (str, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
+
+    Returns:
+        - out (Tensor), A Tensor. the value of the modified bessel function of order 1 at x.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+
+            x = paddle.to_tensor([0, 1, 2, 3, 4], dtype="float32")
+            print(paddle.i1(x))
+            # (Tensor(shape=[5], dtype=float32, place=Place(cpu), stop_gradient=True, [0., 0.5651591 , 1.59063685 , 3.95337022 , 9.75946515]),
+    """
+    if in_dygraph_mode():
+        return _C_ops.i1(x)
+    else:
+        check_variable_and_dtype(x, "x", ["float32", "float64"], "i1")
+
+        helper = LayerHelper("i1", **locals())
+        out = helper.create_variable_for_type_inference(dtype=x.dtype)
+        helper.append_op(
+            type='i1', inputs={'x': x}, outputs={'out': out}, attrs={}
+        )
+    return out
+
+
+def i1e(x, name=None):
+    """
+    The function is used to calculate exponentially scaled modified Bessel function of order 1.
+
+    Args:
+
+        x (Tensor): The input tensor, it's data type should be float32, float64.
+        name (str, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
+
+    Returns:
+        - out (Tensor), A Tensor. the value of the exponentially scaled modified Bessel function of order 1 at x.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+
+            x = paddle.to_tensor([0, 1, 2, 3, 4], dtype="float32")
+            print(paddle.i1e(x))
+            # (Tensor(shape=[5], dtype=float32, place=Place(cpu), stop_gradient=True, [0., 0.20791042, 0.21526929, 0.24300035, 0.17875084]),
+    """
+    if in_dygraph_mode():
+        return _C_ops.i1e(x)
+    else:
+        check_variable_and_dtype(x, "x", ["float32", "float64"], "i1e")
+
+        helper = LayerHelper("i1e", **locals())
+        out = helper.create_variable_for_type_inference(dtype=x.dtype)
+        helper.append_op(
+            type='i1e', inputs={'x': x}, outputs={'out': out}, attrs={}
+        )
+    return out

From 38e5cd000ec24406927ea4d334e3ad30f6a81d18 Mon Sep 17 00:00:00 2001
From: gouzil <66515297+gouzil@users.noreply.github.com>
Date: Wed, 17 May 2023 13:00:46 +0800
Subject: [PATCH 373/405] [fluid] decoupling abn op (#53826)

---
 paddle/fluid/operators/inplace_abn_op.cc | 254 ++++++++++++++++++++++-
 1 file changed, 244 insertions(+), 10 deletions(-)

diff --git a/paddle/fluid/operators/inplace_abn_op.cc b/paddle/fluid/operators/inplace_abn_op.cc
index 5deffde2b562c..62197787f3bd8 100644
--- a/paddle/fluid/operators/inplace_abn_op.cc
+++ b/paddle/fluid/operators/inplace_abn_op.cc
@@ -17,17 +17,159 @@
 #include <memory>
 #include <string>
 #include <vector>
-
-#include "paddle/fluid/operators/batch_norm_op.h"
 #include "paddle/phi/kernels/batch_norm_grad_kernel.h"
 #include "paddle/phi/kernels/batch_norm_kernel.h"
 
 namespace paddle {
 namespace operators {
 
-class InplaceABNOp : public paddle::operators::BatchNormOp {
+class InplaceABNOp : public framework::OperatorWithKernel {
  public:
-  using paddle::operators::BatchNormOp::BatchNormOp;
+  using framework::OperatorWithKernel::OperatorWithKernel;
+
+  void InferShape(framework::InferShapeContext* ctx) const override {
+    OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "BatchNorm");
+    OP_INOUT_CHECK(ctx->HasInput("Scale"), "Input", "Scale", "BatchNorm");
+    OP_INOUT_CHECK(ctx->HasInput("Bias"), "Input", "Bias", "BatchNorm");
+    OP_INOUT_CHECK(ctx->HasInput("Mean"), "Input", "Mean", "BatchNorm");
+    OP_INOUT_CHECK(ctx->HasInput("Variance"), "Input", "Variance", "BatchNorm");
+    OP_INOUT_CHECK(ctx->HasOutput("Y"), "Output", "Y", "BatchNorm");
+
+    bool is_test = ctx->Attrs().Get<bool>("is_test");
+    bool trainable_stats = ctx->Attrs().Get<bool>("trainable_statistics");
+    bool test_mode = is_test && (!trainable_stats);
+    if (!test_mode) {
+      OP_INOUT_CHECK(
+          ctx->HasOutput("MeanOut"), "Output", "MeanOut", "BatchNorm");
+      OP_INOUT_CHECK(
+          ctx->HasOutput("VarianceOut"), "Output", "VarianceOut", "BatchNorm");
+      OP_INOUT_CHECK(
+          ctx->HasOutput("SavedMean"), "Output", "SavedMean", "BatchNorm");
+      OP_INOUT_CHECK(ctx->HasOutput("SavedVariance"),
+                     "Output",
+                     "SavedVariance",
+                     "BatchNorm");
+    }
+
+    // make sure Mean/MeanOut and Variance/VarianceOut share memory in Python
+    PADDLE_ENFORCE_EQ(ctx->Inputs("Mean")[0],
+                      ctx->Outputs("MeanOut")[0],
+                      platform::errors::InvalidArgument(
+                          "Mean and MeanOut should share the same memory"));
+    PADDLE_ENFORCE_EQ(
+        ctx->Inputs("Variance")[0],
+        ctx->Outputs("VarianceOut")[0],
+        platform::errors::InvalidArgument(
+            "Variance and VarianceOut should share the same memory"));
+
+    const auto x_dims = ctx->GetInputDim("X");
+
+    for (int i = 0; i < x_dims.size(); i++) {
+      PADDLE_ENFORCE_EQ(
+          (x_dims[i] == -1) || (x_dims[i] > 0),
+          true,
+          platform::errors::InvalidArgument(
+              "Each dimension of input tensor is expected to be -1 or a "
+              "positive number, but received %d. Input's shape is [%s].",
+              x_dims[i],
+              x_dims));
+    }
+
+    const DataLayout data_layout =
+        phi::StringToDataLayout(ctx->Attrs().Get<std::string>("data_layout"));
+
+    if (ctx->IsRuntime() && ctx->HasInput("MomentumTensor")) {
+      auto mom = ctx->Inputs("MomentumTensor");
+      PADDLE_ENFORCE_EQ(mom.size(),
+                        1,
+                        platform::errors::InvalidArgument(
+                            "The input tensor MomentumTensor's size must be 1"
+                            "But received: MomentumTensor's size is [%d]",
+                            mom.size()));
+    }
+
+    PADDLE_ENFORCE_GE(x_dims.size(),
+                      2,
+                      platform::errors::InvalidArgument(
+                          "ShapeError: the dimension of input "
+                          "X must greater than or equal to 2. But received: "
+                          "the shape of input "
+                          "X = [%s], the dimension of input X =[%d]",
+                          x_dims,
+                          x_dims.size()));
+    PADDLE_ENFORCE_LE(x_dims.size(),
+                      5,
+                      platform::errors::InvalidArgument(
+                          "ShapeError: the dimension of input X "
+                          "must smaller than or equal to 5. But received: the "
+                          "shape of input X "
+                          "= [%s], the dimension of input X = [%d]",
+                          x_dims,
+                          x_dims.size()));
+    VLOG(4) << ctx->IsRunMKLDNNKernel();
+    VLOG(4) << data_layout;
+    const int64_t C = ((ctx->IsRunMKLDNNKernel() == true) ||
+                               (data_layout == DataLayout::kNCHW)
+                           ? x_dims[1]
+                           : x_dims[x_dims.size() - 1]);
+
+    auto scale_dim = ctx->GetInputDim("Scale");
+    auto bias_dim = ctx->GetInputDim("Bias");
+
+    PADDLE_ENFORCE_EQ(
+        scale_dim.size(),
+        1UL,
+        platform::errors::InvalidArgument(
+            "ShapeError: the dimension of scale must equal to 1."
+            "But received: the shape of scale is [%s], the dimension "
+            "of scale is [%d]",
+            scale_dim,
+            scale_dim.size()));
+    PADDLE_ENFORCE_EQ(
+        bias_dim.size(),
+        1UL,
+        platform::errors::InvalidArgument(
+            "ShapeError: the dimension of bias must equal to 1."
+            "But received: the shape of bias is [%s],the dimension "
+            "of bias is [%d]",
+            bias_dim,
+            bias_dim.size()));
+
+    bool check = true;
+    if ((!ctx->IsRuntime()) &&
+        (phi::product(scale_dim) <= 0 || phi::product(bias_dim) <= 0)) {
+      check = false;
+    }
+
+    if (check) {
+      PADDLE_ENFORCE_EQ(scale_dim[0],
+                        C,
+                        platform::errors::InvalidArgument(
+                            "ShapeError: the shape of scale must equal to [%d]"
+                            "But received: the shape of scale is [%d]",
+                            C,
+                            scale_dim[0]));
+      PADDLE_ENFORCE_EQ(bias_dim[0],
+                        C,
+                        platform::errors::InvalidArgument(
+                            "ShapeError: the shape of bias must equal to [%d]"
+                            "But received: the shape of bias is [%d]",
+                            C,
+                            bias_dim[0]));
+    }
+    ctx->SetOutputDim("Y", x_dims);
+    ctx->ShareLoD("X", "Y");
+    VLOG(4) << x_dims;
+    ctx->SetOutputDim("MeanOut", {C});
+    ctx->SetOutputDim("VarianceOut", {C});
+    if (!test_mode) {
+      ctx->SetOutputDim("SavedMean", {C});
+      ctx->SetOutputDim("SavedVariance", {C});
+    }
+    if (ctx->HasOutput("ReserveSpace")) {
+      ctx->SetOutputDim("ReserveSpace", {-1});
+    }
+  }
 
  protected:
   phi::KernelKey GetExpectedKernelType(
@@ -65,10 +207,9 @@ class InplaceABNOp : public paddle::operators::BatchNormOp {
   }
 };
 
-class InplaceABNGradOp : public paddle::operators::BatchNormGradOp {
+class InplaceABNGradOp : public framework::OperatorWithKernel {
  public:
-  using paddle::operators::BatchNormGradOp::BatchNormGradOp;
-
+  using framework::OperatorWithKernel::OperatorWithKernel;
   void InferShape(framework::InferShapeContext* ctx) const override {
     // check input
     OP_INOUT_CHECK(ctx->HasInput("Scale"), "Input", "Scale", "InplaceABNGrad");
@@ -155,10 +296,82 @@ class InplaceABNGradOp : public paddle::operators::BatchNormGradOp {
   }
 };
 
-class InplaceABNOpMaker : public paddle::operators::BatchNormOpMaker {
+class InplaceABNOpMaker : public framework::OpProtoAndCheckerMaker {
  public:
   void Make() override {
-    BatchNormOpMaker::Make();
+    AddAttr<bool>("is_test",
+                  "(bool, default false) Set to true for inference only, false "
+                  "for training. Some layers may run faster when this is true.")
+        .SetDefault(false);
+    AddAttr<float>("momentum", "").SetDefault(0.9);
+    AddAttr<float>("epsilon", "")
+        .SetDefault(1e-5)
+        .AddCustomChecker([](const float& epsilon) {
+          PADDLE_ENFORCE_GE(
+              epsilon,
+              0.0f,
+              platform::errors::InvalidArgument(
+                  "'epsilon' should be greater or equal than 0.0."));
+          PADDLE_ENFORCE_LE(
+              epsilon,
+              0.001f,
+              platform::errors::InvalidArgument(
+                  "'epsilon' should be less or equal than 0.001."));
+        });
+    AddAttr<std::string>("data_layout", "").SetDefault("NCHW");
+    AddInput("X", "The input tensor");
+    AddInput("Scale",
+             "Scale is a 1-dimensional tensor of size C "
+             "that is applied to the output");
+    AddInput("Bias",
+             "Bias is a 1-dimensional tensor of size C "
+             "that is applied to the output");
+    AddInput("Mean",
+             "The global mean (for training) or "
+             "estimated mean (for testing)");
+    AddInput("Variance",
+             "The global variance (for training) "
+             "or estimated Variance (for testing)");
+    AddInput(
+        "MomentumTensor",
+        "(phi::DenseTensor<float32>, optional) If provided, batch_norm will "
+        "use this as momentum, this has a higher priority than "
+        "attr(momentum), the shape of this tensor MUST BE [1].")
+        .AsDispensable();
+    AddOutput("Y", "result after normalization");
+    AddOutput("MeanOut",
+              "Share memory with Mean. "
+              "Store the global mean when training");
+    AddOutput("VarianceOut",
+              "Share memory with Variance. "
+              "Store the global Variance when training");
+    AddOutput("SavedMean",
+              "Mean of the current mini batch, "
+              "will apply to output when training")
+        .AsIntermediate();
+    AddOutput("SavedVariance",
+              "Variance of the current mini batch, "
+              "will apply to output when training")
+        .AsIntermediate();
+    AddOutput("ReserveSpace",
+              "Reserve GPU space for triggering the new semi-persistent "
+              "NHWC kernel")
+        .AsDispensable()
+        .AsExtra();
+    AddAttr<bool>("use_global_stats",
+                  "(bool, default false) Whether to use global mean and "
+                  "variance. In inference or test mode, set use_global_stats "
+                  "to true or is_test true. the behavior is equivalent. "
+                  "In train mode, when setting use_global_stats True, the "
+                  "global mean and variance are also used during train time, "
+                  "the BN acts as scaling and shiffting.")
+        .SetDefault(false);
+    AddAttr<bool>(
+        "trainable_statistics",
+        "(bool, default false) Whether to calculate mean and variance "
+        "in test mode. If setting true in test mode, mean and variace "
+        "will be calculated by current batch statistics.")
+        .SetDefault(false);
     AddAttr<std::string>(
         "activation",
         "(enum string, default identity, can be identity|elu|leaky-relu) "
@@ -174,6 +387,17 @@ class InplaceABNOpMaker : public paddle::operators::BatchNormOpMaker {
                   "(bool, default false) Whether use synchronize batch "
                   "normalization.")
         .SetDefault(false);
+    AddComment(R"DOC(
+Batch Normalization.
+
+Batch Norm has been implemented as discussed in the paper:
+https://arxiv.org/pdf/1502.03167.pdf
+Can be used as a normalizer function for conv2d and fully_connected operations.
+The required data format for this layer is one of the following:
+1. NHWC `[batch, in_height, in_width, in_channels]`
+2. NCHW `[batch, in_channels, in_height, in_width]`
+
+)DOC");
   }
 };
 
@@ -358,6 +582,16 @@ class InplaceABNGradKernel : public framework::OpKernel<T> {
   }
 };
 
+class InplaceABNOpInferVarType
+    : public framework::PassInDtypeAndVarTypeToOutput {
+ protected:
+  std::unordered_map<std::string, std::string>& GetInputOutputWithSameType()
+      const override {
+    static std::unordered_map<std::string, std::string> m{{"X", /*->*/ "Y"}};
+    return m;
+  }
+};
+
 }  // namespace operators
 }  // namespace paddle
 
@@ -367,7 +601,7 @@ DECLARE_INPLACE_OP_INFERER(InplaceAbnOpInplaceInferer, {"X", "Y"});
 REGISTER_OPERATOR(inplace_abn,
                   ops::InplaceABNOp,
                   ops::InplaceABNOpMaker,
-                  ops::BatchNormOpInferVarType,
+                  ops::InplaceABNOpInferVarType,
                   ops::InplaceABNOpGradMaker<paddle::framework::OpDesc>,
                   ops::InplaceABNOpGradMaker<paddle::imperative::OpBase>,
                   InplaceAbnOpInplaceInferer)

From 56e8affefce63220f8faf6ff0beb35c25ab4f98d Mon Sep 17 00:00:00 2001
From: 6clc <569519574@qq.com>
Date: Wed, 17 May 2023 13:37:19 +0800
Subject: [PATCH 374/405] test(cinn): fix resnet50 precision (#53649)

---
 test/prim/model/test_resnet_prim_cinn.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/test/prim/model/test_resnet_prim_cinn.py b/test/prim/model/test_resnet_prim_cinn.py
index 75238a8192c08..76295fbf9b4c8 100644
--- a/test/prim/model/test_resnet_prim_cinn.py
+++ b/test/prim/model/test_resnet_prim_cinn.py
@@ -46,15 +46,15 @@
 # The results in ci as as follows:
 DY2ST_PRIM_CINN_GT = [
     5.828786849975586,
-    8.332868576049805,
-    5.038548469543457,
-    8.554015159606934,
-    8.106254577636719,
-    7.493070125579834,
-    9.479158401489258,
-    8.270158767700195,
-    8.324719429016113,
-    10.140411376953125,
+    8.332858085632324,
+    5.026939868927002,
+    8.475804328918457,
+    8.017110824584961,
+    7.8353095054626465,
+    9.731267929077148,
+    8.193124771118164,
+    8.155317306518555,
+    10.185102462768555,
 ]
 
 if core.is_compiled_with_cuda():

From 78967ad28cf55fa65b95b6f9660da52a7dfba604 Mon Sep 17 00:00:00 2001
From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com>
Date: Wed, 17 May 2023 14:00:13 +0800
Subject: [PATCH 375/405] [IR] Program & Parameter & PaddleDialect (#53557)

* add program parameter dialect_interface

* fix op create bug

* add ir parameter convert pd variable methods

* refine code

* fix bug

* refine by ut

* refine ut

* delete unused code

* refine code

* refine code by comment

* reset WITH_NEW_IR

* refine op attribute map

* refine program and op create

* refine program and op create
---
 paddle/fluid/CMakeLists.txt                   |   3 +
 paddle/fluid/dialect/CMakeLists.txt           |   9 +
 paddle/fluid/dialect/pd_dialect.cc            |  97 +++++++++
 paddle/fluid/dialect/pd_dialect.h             |  47 ++++
 .../dialect/pd_type.cc}                       |  19 +-
 paddle/fluid/dialect/pd_type.h                |  46 ++++
 .../dialect/pd_type_storage.h}                |   6 +-
 paddle/fluid/dialect/utils.h                  | 136 ++++++++++++
 paddle/ir/builtin_attribute.cc                |  20 --
 paddle/ir/builtin_attribute.h                 |  50 +----
 paddle/ir/builtin_attribute_storage.cc        |  77 -------
 paddle/ir/builtin_attribute_storage.h         |  32 ---
 paddle/ir/builtin_dialect.cc                  |   2 +
 paddle/ir/builtin_op.cc                       |  22 ++
 paddle/ir/builtin_op.h                        |  57 +++++
 paddle/ir/builtin_type.h                      |  23 +-
 paddle/ir/dialect.cc                          |  16 ++
 paddle/ir/dialect.h                           |  45 +++-
 paddle/ir/dialect_interface.h                 |  67 ++++++
 paddle/ir/op_info_impl.h                      |   6 +
 paddle/ir/operation.cc                        |  38 +++-
 paddle/ir/operation.h                         |  28 ++-
 paddle/ir/parameter.h                         |  70 ++++++
 paddle/ir/program.cc                          |  42 ++++
 paddle/ir/program.h                           |  59 +++++
 paddle/ir/type.h                              |   2 +-
 paddle/ir/type_base.h                         |   2 +-
 paddle/ir/value.cc                            |   4 +
 paddle/ir/value_impl.h                        |   4 +
 test/cpp/ir/CMakeLists.txt                    |   9 +
 test/cpp/ir/ir_attribute_test.cc              |  31 ---
 test/cpp/ir/ir_op_test.cc                     |  15 +-
 test/cpp/ir/ir_program_test.cc                | 206 ++++++++++++++++++
 test/cpp/ir/ir_value_test.cc                  |  22 +-
 test/cpp/ir/type_test.cc                      |  38 +---
 35 files changed, 1040 insertions(+), 310 deletions(-)
 create mode 100644 paddle/fluid/dialect/CMakeLists.txt
 create mode 100644 paddle/fluid/dialect/pd_dialect.cc
 create mode 100644 paddle/fluid/dialect/pd_dialect.h
 rename paddle/{ir/builtin_type.cc => fluid/dialect/pd_type.cc} (69%)
 create mode 100644 paddle/fluid/dialect/pd_type.h
 rename paddle/{ir/builtin_type_storage.h => fluid/dialect/pd_type_storage.h} (98%)
 create mode 100644 paddle/fluid/dialect/utils.h
 delete mode 100644 paddle/ir/builtin_attribute_storage.cc
 create mode 100644 paddle/ir/builtin_op.cc
 create mode 100644 paddle/ir/builtin_op.h
 create mode 100644 paddle/ir/dialect_interface.h
 create mode 100644 paddle/ir/parameter.h
 create mode 100644 paddle/ir/program.cc
 create mode 100644 paddle/ir/program.h
 create mode 100644 test/cpp/ir/ir_program_test.cc

diff --git a/paddle/fluid/CMakeLists.txt b/paddle/fluid/CMakeLists.txt
index e281c8c48f504..8248dcd36398b 100644
--- a/paddle/fluid/CMakeLists.txt
+++ b/paddle/fluid/CMakeLists.txt
@@ -8,5 +8,8 @@ add_subdirectory(pybind)
 add_subdirectory(eager)
 add_subdirectory(prim)
 add_subdirectory(jit)
+if(WITH_NEWIR)
+  add_subdirectory(dialect)
+endif()
 # NOTE: please add subdirectory inference at last.
 add_subdirectory(inference)
diff --git a/paddle/fluid/dialect/CMakeLists.txt b/paddle/fluid/dialect/CMakeLists.txt
new file mode 100644
index 0000000000000..644b991379425
--- /dev/null
+++ b/paddle/fluid/dialect/CMakeLists.txt
@@ -0,0 +1,9 @@
+set(PD_DIALECT_SOURCE_DIR "${PADDLE_SOURCE_DIR}/paddle/fluid/dialect")
+set(PD_DIALECT_BINARY_DIR "${PADDLE_BINARY_DIR}/paddle/fluid/dialect")
+
+file(GLOB PD_DIALECT_SRCS "*.cc")
+
+cc_library(
+  pd_dialect
+  SRCS ${PD_DIALECT_SRCS}
+  DEPS new_ir framework_proto dense_tensor)
diff --git a/paddle/fluid/dialect/pd_dialect.cc b/paddle/fluid/dialect/pd_dialect.cc
new file mode 100644
index 0000000000000..df98a9978803a
--- /dev/null
+++ b/paddle/fluid/dialect/pd_dialect.cc
@@ -0,0 +1,97 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/dialect/pd_dialect.h"
+#include "paddle/fluid/dialect/pd_type.h"
+#include "paddle/fluid/dialect/utils.h"
+#include "paddle/fluid/framework/convert_utils.h"
+#include "paddle/fluid/framework/data_type.h"
+#include "paddle/ir/builtin_type.h"
+#include "paddle/ir/dialect_interface.h"
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace paddle {
+namespace dialect {
+std::shared_ptr<paddle::framework::Variable>
+ParameterConvertInterface::ParameterToVariable(ir::Parameter* parameter) {
+  if (parameter->type().isa<DenseTensorType>()) {
+    VLOG(4) << "Convert a DenseTensor Parameter to a variable.";
+    std::shared_ptr<paddle::framework::Variable> var =
+        std::make_shared<paddle::framework::Variable>();
+    phi::DenseTensor* tensor = var->GetMutable<phi::DenseTensor>();
+    // Init DenseTensor
+    auto dim = parameter->type().dyn_cast<DenseTensorType>().dim();
+    phi::DenseTensorMeta meta(
+        TransToPhiDataType(
+            parameter->type().dyn_cast<DenseTensorType>().dtype()),
+        phi::DDim(dim.data(), dim.size()),
+        TransToPhiDataLayout(
+            parameter->type().dyn_cast<DenseTensorType>().data_layout()),
+        parameter->type().dyn_cast<DenseTensorType>().lod(),
+        parameter->type().dyn_cast<DenseTensorType>().offset());
+    tensor->set_meta(meta);
+    paddle::platform::DeviceContext* dev_ctx =
+        paddle::platform::DeviceContextPool::Instance().Get(
+            paddle::platform::CPUPlace());
+    dev_ctx->Alloc(tensor,
+                   TransToPhiDataType(
+                       parameter->type().dyn_cast<DenseTensorType>().dtype()));
+    memcpy(tensor->data(),
+           parameter->data(),
+           tensor->numel() * phi::SizeOf(tensor->dtype()));
+    return var;
+  } else {
+    return nullptr;
+  }
+}
+
+std::unique_ptr<ir::Parameter> ParameterConvertInterface::VariableToParameter(
+    paddle::framework::Variable* var) {
+  if (var->IsType<phi::DenseTensor>()) {
+    phi::DenseTensor* tensor = var->GetMutable<phi::DenseTensor>();
+    // Get Meta
+    ir::IrContext* ctx = ir::IrContext::Instance();
+    ir::Type data_type = TransToIrDataType(tensor->dtype(), ctx);
+    DenseTensorTypeStorage::Dim dims(tensor->dims().size());
+    std::copy(tensor->dims().Get(),
+              tensor->dims().Get() + tensor->dims().size(),
+              dims.data());
+    DenseTensorTypeStorage::DataLayout data_layout =
+        TransToIrDataLayout(tensor->layout());
+    DenseTensorTypeStorage::LoD lod = tensor->lod();
+    size_t offset = tensor->meta().offset;
+    void* data = tensor->data();
+    ir::Type dense_tensor_type =
+        DenseTensorType::get(ctx, data_type, dims, data_layout, lod, offset);
+    return std::make_unique<ir::Parameter>(
+        data,
+        tensor->numel() * phi::SizeOf(tensor->dtype()),
+        dense_tensor_type);
+  } else {
+    return nullptr;
+  }
+}
+
+PaddleDialect::PaddleDialect(ir::IrContext* context)
+    : ir::Dialect(name(), context, ir::TypeId::get<PaddleDialect>()) {
+  initialize();
+}
+
+void PaddleDialect::initialize() {
+  RegisterTypes<GET_PADDLE_TYPE_LIST>();
+  RegisterInterfaces<ParameterConvertInterface>();
+}
+
+}  // namespace dialect
+}  // namespace paddle
diff --git a/paddle/fluid/dialect/pd_dialect.h b/paddle/fluid/dialect/pd_dialect.h
new file mode 100644
index 0000000000000..a81ff7cd48ceb
--- /dev/null
+++ b/paddle/fluid/dialect/pd_dialect.h
@@ -0,0 +1,47 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/fluid/framework/variable.h"
+#include "paddle/ir/dialect.h"
+#include "paddle/ir/parameter.h"
+
+namespace paddle {
+namespace dialect {
+class ParameterConvertInterface
+    : public ir::DialectInterface::Base<ParameterConvertInterface> {
+ public:
+  explicit ParameterConvertInterface(ir::Dialect* dialect) : Base(dialect) {}
+
+  // NOTE(zhangbo): Only support new a CPU Variable.
+  std::shared_ptr<paddle::framework::Variable> ParameterToVariable(
+      ir::Parameter* parameter);
+
+  std::unique_ptr<ir::Parameter> VariableToParameter(
+      paddle::framework::Variable* var);
+};
+
+class PaddleDialect : public ir::Dialect {
+ public:
+  explicit PaddleDialect(ir::IrContext* context);
+
+  static const char* name() { return "pd"; }
+
+ private:
+  void initialize();
+};
+
+}  // namespace dialect
+}  // namespace paddle
diff --git a/paddle/ir/builtin_type.cc b/paddle/fluid/dialect/pd_type.cc
similarity index 69%
rename from paddle/ir/builtin_type.cc
rename to paddle/fluid/dialect/pd_type.cc
index 5e18b945016c7..017af3ffd5aea 100644
--- a/paddle/ir/builtin_type.cc
+++ b/paddle/fluid/dialect/pd_type.cc
@@ -12,23 +12,28 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/ir/builtin_type.h"
+#include "paddle/fluid/dialect/pd_type.h"
 
-namespace ir {
+namespace paddle {
+namespace dialect {
 const ir::Type& DenseTensorType::dtype() const { return storage()->dtype_; }
 
-const ir::DenseTensorTypeStorage::Dim& DenseTensorType::dim() const {
+const paddle::dialect::DenseTensorTypeStorage::Dim& DenseTensorType::dim()
+    const {
   return storage()->dims_;
 }
 
-const ir::DenseTensorTypeStorage::DataLayout& DenseTensorType::data_layout()
-    const {
+const paddle::dialect::DenseTensorTypeStorage::DataLayout&
+DenseTensorType::data_layout() const {
   return storage()->layout_;
 }
 
-const ir::DenseTensorTypeStorage::LoD& DenseTensorType::lod() const {
+const paddle::dialect::DenseTensorTypeStorage::LoD& DenseTensorType::lod()
+    const {
   return storage()->lod_;
 }
 
 const size_t& DenseTensorType::offset() const { return storage()->offset_; }
-}  // namespace ir
+
+}  // namespace dialect
+}  // namespace paddle
diff --git a/paddle/fluid/dialect/pd_type.h b/paddle/fluid/dialect/pd_type.h
new file mode 100644
index 0000000000000..2040828a146ac
--- /dev/null
+++ b/paddle/fluid/dialect/pd_type.h
@@ -0,0 +1,46 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/fluid/dialect/pd_type_storage.h"
+#include "paddle/ir/type.h"
+
+namespace paddle {
+namespace dialect {
+#define GET_PADDLE_TYPE_LIST paddle::dialect::DenseTensorType
+
+///
+/// \brief Define built-in parameteric types.
+///
+class DenseTensorType : public ir::Type {
+ public:
+  using Type::Type;
+
+  DECLARE_TYPE_UTILITY_FUNCTOR(DenseTensorType, DenseTensorTypeStorage);
+
+  const ir::Type &dtype() const;
+
+  const paddle::dialect::DenseTensorTypeStorage::Dim &dim() const;
+
+  const paddle::dialect::DenseTensorTypeStorage::DataLayout &data_layout()
+      const;
+
+  const paddle::dialect::DenseTensorTypeStorage::LoD &lod() const;
+
+  const size_t &offset() const;
+};
+
+}  // namespace dialect
+}  // namespace paddle
diff --git a/paddle/ir/builtin_type_storage.h b/paddle/fluid/dialect/pd_type_storage.h
similarity index 98%
rename from paddle/ir/builtin_type_storage.h
rename to paddle/fluid/dialect/pd_type_storage.h
index 132a1656a7975..1ea44eccc7347 100644
--- a/paddle/ir/builtin_type_storage.h
+++ b/paddle/fluid/dialect/pd_type_storage.h
@@ -36,7 +36,8 @@ struct hash<std::vector<T>> {
 
 }  // namespace std
 
-namespace ir {
+namespace paddle {
+namespace dialect {
 ///
 /// \brief Define Parameteric TypeStorage for DenseTensorType.
 ///
@@ -151,4 +152,5 @@ struct DenseTensorTypeStorage : public ir::TypeStorage {
   size_t offset_;
 };
 
-}  // namespace ir
+}  // namespace dialect
+}  // namespace paddle
diff --git a/paddle/fluid/dialect/utils.h b/paddle/fluid/dialect/utils.h
new file mode 100644
index 0000000000000..69af6fd1ddce8
--- /dev/null
+++ b/paddle/fluid/dialect/utils.h
@@ -0,0 +1,136 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/fluid/dialect/pd_type_storage.h"
+#include "paddle/fluid/framework/convert_utils.h"
+#include "paddle/fluid/framework/data_type.h"
+#include "paddle/ir/builtin_type.h"
+#include "paddle/phi/core/dense_tensor.h"
+
+namespace paddle {
+namespace dialect {
+// TODO(zhangbo): The builtin type needs to cover all data types of
+// phi::DataType.
+inline phi::DataType TransToPhiDataType(ir::Type dtype) {
+  if (dtype.isa<ir::Float16Type>()) {
+    return phi::DataType::FLOAT16;
+  } else if (dtype.isa<ir::Float32Type>()) {
+    return phi::DataType::FLOAT32;
+  } else if (dtype.isa<ir::Float64Type>()) {
+    return phi::DataType::FLOAT64;
+  } else if (dtype.isa<ir::Int16Type>()) {
+    return phi::DataType::INT16;
+  } else if (dtype.isa<ir::Int32Type>()) {
+    return phi::DataType::INT32;
+  } else if (dtype.isa<ir::Int64Type>()) {
+    return phi::DataType::INT64;
+  } else {
+    PADDLE_THROW(phi::errors::Unimplemented(
+        "Unsupported ir data type when casting it into "
+        "phi data type."));
+  }
+}
+
+inline ir::Type TransToIrDataType(phi::DataType dtype,
+                                  ir::IrContext *ctx = nullptr) {
+  if (ctx == nullptr) {
+    ctx = ir::IrContext::Instance();
+  }
+  switch (dtype) {
+    case phi::DataType::FLOAT16:
+      return ir::Float16Type::get(ctx);
+    case phi::DataType::FLOAT32:
+      return ir::Float32Type::get(ctx);
+    case phi::DataType::FLOAT64:
+      return ir::Float64Type::get(ctx);
+    case phi::DataType::INT16:
+      return ir::Int16Type::get(ctx);
+    case phi::DataType::INT32:
+      return ir::Int32Type::get(ctx);
+    case phi::DataType::INT64:
+      return ir::Int64Type::get(ctx);
+    default:
+      PADDLE_THROW(phi::errors::Unimplemented(
+          "Unsupported phi data type `%s` when casting it into "
+          "ir data type.",
+          dtype));
+  }
+}
+
+inline phi::DataLayout TransToPhiDataLayout(
+    DenseTensorTypeStorage::DataLayout data_layout) {
+  switch (data_layout) {
+    case DenseTensorTypeStorage::DataLayout::NHWC:
+      return phi::DataLayout::NHWC;
+    case DenseTensorTypeStorage::DataLayout::NCHW:
+      return phi::DataLayout::NCHW;
+    case DenseTensorTypeStorage::DataLayout::NCDHW:
+      return phi::DataLayout::NCDHW;
+    case DenseTensorTypeStorage::DataLayout::NDHWC:
+      return phi::DataLayout::NDHWC;
+    case DenseTensorTypeStorage::DataLayout::ONEDNN:
+      return phi::DataLayout::ONEDNN;
+    case DenseTensorTypeStorage::DataLayout::SPARSE_COO:
+      return phi::DataLayout::SPARSE_COO;
+    case DenseTensorTypeStorage::DataLayout::SPARSE_CSR:
+      return phi::DataLayout::SPARSE_CSR;
+    case DenseTensorTypeStorage::DataLayout::PSTRING_UNION:
+      return phi::DataLayout::PSTRING_UNION;
+    case DenseTensorTypeStorage::DataLayout::NUM_DATA_LAYOUTS:
+      return phi::DataLayout::NUM_DATA_LAYOUTS;
+    case DenseTensorTypeStorage::DataLayout::ALL_LAYOUT:
+      return phi::DataLayout::ALL_LAYOUT;
+    default:
+      PADDLE_THROW(phi::errors::Unimplemented(
+          "Unsupported ir data layout `%s` when casting it into "
+          "phi data type.",
+          static_cast<int>(data_layout)));
+  }
+}
+
+inline DenseTensorTypeStorage::DataLayout TransToIrDataLayout(
+    phi::DataLayout data_layout) {
+  switch (data_layout) {
+    case phi::DataLayout::NHWC:
+      return DenseTensorTypeStorage::DataLayout::NHWC;
+    case phi::DataLayout::NCHW:
+      return DenseTensorTypeStorage::DataLayout::NCHW;
+    case phi::DataLayout::NCDHW:
+      return DenseTensorTypeStorage::DataLayout::NCDHW;
+    case phi::DataLayout::NDHWC:
+      return DenseTensorTypeStorage::DataLayout::NDHWC;
+    case phi::DataLayout::ONEDNN:
+      return DenseTensorTypeStorage::DataLayout::ONEDNN;
+    case phi::DataLayout::SPARSE_COO:
+      return DenseTensorTypeStorage::DataLayout::SPARSE_COO;
+    case phi::DataLayout::SPARSE_CSR:
+      return DenseTensorTypeStorage::DataLayout::SPARSE_CSR;
+    case phi::DataLayout::PSTRING_UNION:
+      return DenseTensorTypeStorage::DataLayout::PSTRING_UNION;
+    case phi::DataLayout::NUM_DATA_LAYOUTS:
+      return DenseTensorTypeStorage::DataLayout::NUM_DATA_LAYOUTS;
+    case phi::DataLayout::ALL_LAYOUT:
+      return DenseTensorTypeStorage::DataLayout::ALL_LAYOUT;
+    default:
+      PADDLE_THROW(phi::errors::Unimplemented(
+          "Unsupported phi data layout `%s` when casting it into "
+          "ir data type.",
+          static_cast<int>(data_layout)));
+  }
+}
+
+}  // namespace dialect
+}  // namespace paddle
diff --git a/paddle/ir/builtin_attribute.cc b/paddle/ir/builtin_attribute.cc
index 3a1b2acfb716b..40c0204ce9cfd 100644
--- a/paddle/ir/builtin_attribute.cc
+++ b/paddle/ir/builtin_attribute.cc
@@ -19,24 +19,4 @@ std::string StrAttribute::data() const { return storage()->GetAsKey(); }
 
 uint32_t StrAttribute::size() const { return storage()->GetAsKey().size(); }
 
-NamedAttribute::NamedAttribute(StrAttribute name, Attribute value)
-    : name_(name), value_(value) {}
-
-bool NamedAttribute::operator<(const NamedAttribute &right) const {
-  return name() < right.name();
-}
-
-bool NamedAttribute::operator==(const NamedAttribute &right) const {
-  return name() == right.name() && value() == right.value();
-}
-
-bool NamedAttribute::operator!=(const NamedAttribute &right) const {
-  return !(*this == right);
-}
-
-Attribute DictionaryAttribute::GetValue(const StrAttribute &name) {
-  return storage()->GetValue(name);
-}
-
-uint32_t DictionaryAttribute::size() const { return storage()->size(); }
 }  // namespace ir
diff --git a/paddle/ir/builtin_attribute.h b/paddle/ir/builtin_attribute.h
index 82b5f8eb48aa5..4e2164a1e4d6a 100644
--- a/paddle/ir/builtin_attribute.h
+++ b/paddle/ir/builtin_attribute.h
@@ -22,7 +22,7 @@ namespace ir {
 ///
 /// \brief All built-in attributes.
 ///
-#define GET_BUILT_IN_ATTRIBUTE_LIST ir::StrAttribute, ir::DictionaryAttribute
+#define GET_BUILT_IN_ATTRIBUTE_LIST ir::StrAttribute
 
 class StrAttribute : public ir::Attribute {
  public:
@@ -39,55 +39,13 @@ class StrAttribute : public ir::Attribute {
   uint32_t size() const;
 };
 
-class NamedAttribute {
- public:
-  NamedAttribute(StrAttribute name, Attribute value);
-
-  StrAttribute name() const { return name_; }
-
-  Attribute value() const { return value_; }
-
-  void SetName(StrAttribute name) { name_ = name; }
-
-  void SetValue(Attribute value) { value_ = value; }
-
-  bool operator<(const NamedAttribute &right) const;
-
-  bool operator==(const NamedAttribute &right) const;
-
-  bool operator!=(const NamedAttribute &right) const;
-
-  friend struct std::hash<NamedAttribute>;
-
-  operator std::pair<const StrAttribute, Attribute>() const {
-    return std::make_pair(name_, value_);
-  }
-
- private:
-  StrAttribute name_;
-  Attribute value_;
-};
-
-class DictionaryAttribute : public ir::Attribute {
- public:
-  using Attribute::Attribute;
-
-  DECLARE_ATTRIBUTE_UTILITY_FUNCTOR(DictionaryAttribute,
-                                    DictionaryAttributeStorage);
-
-  Attribute GetValue(const StrAttribute &name);
-
-  uint32_t size() const;
-};
-
 }  // namespace ir
 
 namespace std {
 template <>
-struct hash<ir::NamedAttribute> {
-  std::size_t operator()(const ir::NamedAttribute &obj) const {
-    return ir::hash_combine(std::hash<ir::Attribute>()(obj.name_),
-                            std::hash<ir::Attribute>()(obj.value_));
+struct hash<ir::StrAttribute> {
+  std::size_t operator()(const ir::StrAttribute &obj) const {
+    return std::hash<const ir::StrAttribute::Storage *>()(obj.storage());
   }
 };
 }  // namespace std
diff --git a/paddle/ir/builtin_attribute_storage.cc b/paddle/ir/builtin_attribute_storage.cc
deleted file mode 100644
index 3f785d20c9b92..0000000000000
--- a/paddle/ir/builtin_attribute_storage.cc
+++ /dev/null
@@ -1,77 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/ir/builtin_attribute_storage.h"
-#include "paddle/ir/builtin_attribute.h"
-#include "paddle/ir/utils.h"
-
-namespace ir {
-
-DictionaryAttributeStorage::DictionaryAttributeStorage(const ParamKey &key) {
-  size_ = key.size();
-  data_ = reinterpret_cast<NamedAttribute *>(
-      malloc(sizeof(NamedAttribute) * size_));
-  uint32_t idx = 0;
-  for (auto iter = key.begin(); iter != key.end(); ++iter) {
-    data_[idx].SetName(iter->first);
-    data_[idx].SetValue(iter->second);
-    idx++;
-  }
-}
-
-std::size_t DictionaryAttributeStorage::HashValue(const ParamKey &key) {
-  std::size_t hash_value = key.size();
-  for (auto iter = key.begin(); iter != key.end(); ++iter) {
-    hash_value = ir::hash_combine(
-        hash_value,
-        std::hash<NamedAttribute>()(NamedAttribute(iter->first, iter->second)));
-  }
-  return hash_value;
-}
-
-bool DictionaryAttributeStorage::operator==(const ParamKey &key) const {
-  uint32_t size = key.size();
-  if (size_ != size) return false;
-  uint32_t idx = 0;
-  for (auto iter = key.begin(); iter != key.end(); ++iter) {
-    if (data_[idx] != NamedAttribute(iter->first, iter->second)) {
-      return false;
-    }
-    idx++;
-  }
-  return true;
-}
-
-DictionaryAttributeStorage::ParamKey DictionaryAttributeStorage::GetAsKey()
-    const {
-  return ParamKey(data_, data_ + size_);
-}
-
-Attribute DictionaryAttributeStorage::GetValue(const StrAttribute &name) const {
-  size_t left = 0;
-  size_t right = size_;
-  while (left < right) {
-    size_t mid = left + (right - left) / 2;
-    if (data_[mid].name() == name) {
-      return data_[mid].value();
-    } else if (data_[mid].name() < name) {
-      left = mid + 1;
-    } else {
-      right = mid;
-    }
-  }
-  return nullptr;
-}
-
-}  // namespace ir
diff --git a/paddle/ir/builtin_attribute_storage.h b/paddle/ir/builtin_attribute_storage.h
index a34648fb17e35..f6f97d5e616ed 100644
--- a/paddle/ir/builtin_attribute_storage.h
+++ b/paddle/ir/builtin_attribute_storage.h
@@ -54,36 +54,4 @@ struct StrAttributeStorage : public ir::AttributeStorage {
   uint32_t size_;
 };
 
-///
-/// \brief Define Parameteric AttributeStorage for DictionaryAttributeStorage.
-///
-class StrAttribute;
-class NamedAttribute;
-struct DictionaryAttributeStorage : public AttributeStorage {
-  using ParamKey = std::map<StrAttribute, Attribute>;
-
-  explicit DictionaryAttributeStorage(const ParamKey &key);
-
-  ~DictionaryAttributeStorage() { free(data_); }
-
-  static DictionaryAttributeStorage *Construct(ParamKey key) {
-    return new DictionaryAttributeStorage(key);
-  }
-
-  static std::size_t HashValue(const ParamKey &key);
-
-  bool operator==(const ParamKey &key) const;
-
-  ParamKey GetAsKey() const;
-
-  Attribute GetValue(const StrAttribute &name) const;
-
-  NamedAttribute *data() const { return data_; }
-
-  uint32_t size() const { return size_; }
-
- private:
-  NamedAttribute *data_;
-  uint32_t size_;
-};
 }  // namespace ir
diff --git a/paddle/ir/builtin_dialect.cc b/paddle/ir/builtin_dialect.cc
index 144ee272c4c1b..5e08798828f59 100644
--- a/paddle/ir/builtin_dialect.cc
+++ b/paddle/ir/builtin_dialect.cc
@@ -14,6 +14,7 @@
 
 #include "paddle/ir/builtin_dialect.h"
 #include "paddle/ir/builtin_attribute.h"
+#include "paddle/ir/builtin_op.h"
 #include "paddle/ir/builtin_type.h"
 
 namespace ir {
@@ -26,6 +27,7 @@ void BuiltinDialect::initialize() {
   // Register all built-in types defined in builtin_type.h.
   RegisterTypes<GET_BUILT_IN_TYPE_LIST>();
   RegisterAttributes<GET_BUILT_IN_ATTRIBUTE_LIST>();
+  RegisterOps<GET_BUILT_IN_OP_LIST>();
 }
 
 }  // namespace ir
diff --git a/paddle/ir/builtin_op.cc b/paddle/ir/builtin_op.cc
new file mode 100644
index 0000000000000..528631b0b7998
--- /dev/null
+++ b/paddle/ir/builtin_op.cc
@@ -0,0 +1,22 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/ir/builtin_op.h"
+
+namespace ir {
+const char *GetParameterOp::attributes_name_[] = {"parameter_name"};
+
+const char *SetParameterOp::attributes_name_[] = {"parameter_name"};
+
+}  // namespace ir
diff --git a/paddle/ir/builtin_op.h b/paddle/ir/builtin_op.h
new file mode 100644
index 0000000000000..f5c4296394a9a
--- /dev/null
+++ b/paddle/ir/builtin_op.h
@@ -0,0 +1,57 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/ir/op_base.h"
+
+namespace ir {
+///
+/// \brief This macro is used to get a list of all built-in OPs in this file.
+/// The built-in Dialect will use this macro to quickly register all built-in
+/// OPs.
+///
+#define GET_BUILT_IN_OP_LIST ir::GetParameterOp, ir::SetParameterOp
+
+///
+/// \brief GetParameterOp: OpResult = GetParameterOp({StrAttribute,
+/// StrAttribute})
+///
+class GetParameterOp : public ir::Op<GetParameterOp> {
+ public:
+  using Op::Op;
+
+  static const char* name() { return "GetParameterOp"; }
+
+  static uint32_t attributes_num() { return 1; }
+
+  static const char* attributes_name_[];
+};
+
+///
+/// \brief GetParameterOp: SetParameterOp(OpOperand, {StrAttribute,
+/// StrAttribute})
+///
+class SetParameterOp : public ir::Op<SetParameterOp> {
+ public:
+  using Op::Op;
+
+  static const char* name() { return "SetParameterOp"; }
+
+  static uint32_t attributes_num() { return 1; }
+
+  static const char* attributes_name_[];
+};
+
+}  // namespace ir
diff --git a/paddle/ir/builtin_type.h b/paddle/ir/builtin_type.h
index 0f4db31d9d67d..d195cca2939c3 100644
--- a/paddle/ir/builtin_type.h
+++ b/paddle/ir/builtin_type.h
@@ -14,7 +14,6 @@
 
 #pragma once
 
-#include "paddle/ir/builtin_type_storage.h"
 #include "paddle/ir/type.h"
 
 namespace ir {
@@ -25,7 +24,7 @@ namespace ir {
 ///
 #define GET_BUILT_IN_TYPE_LIST                                      \
   ir::Float16Type, ir::Float32Type, ir::Float64Type, ir::Int16Type, \
-      ir::Int32Type, ir::Int64Type, ir::DenseTensorType
+      ir::Int32Type, ir::Int64Type
 
 ///
 /// \brief Define built-in parameterless types. Please add the necessary
@@ -97,24 +96,4 @@ class Int64Type : public ir::Type {
   static Int64Type get(ir::IrContext *context);
 };
 
-///
-/// \brief Define built-in parameteric types.
-///
-class DenseTensorType : public ir::Type {
- public:
-  using Type::Type;
-
-  DECLARE_TYPE_UTILITY_FUNCTOR(DenseTensorType, DenseTensorTypeStorage);
-
-  const ir::Type &dtype() const;
-
-  const ir::DenseTensorTypeStorage::Dim &dim() const;
-
-  const ir::DenseTensorTypeStorage::DataLayout &data_layout() const;
-
-  const ir::DenseTensorTypeStorage::LoD &lod() const;
-
-  const size_t &offset() const;
-};
-
 }  // namespace ir
diff --git a/paddle/ir/dialect.cc b/paddle/ir/dialect.cc
index cdc171557148a..8d52f3bbc9d78 100644
--- a/paddle/ir/dialect.cc
+++ b/paddle/ir/dialect.cc
@@ -18,6 +18,8 @@ namespace ir {
 Dialect::Dialect(std::string name, ir::IrContext *context, ir::TypeId id)
     : name_(std::move(name)), context_(context), id_(id) {}
 
+Dialect::~Dialect() = default;
+
 void Dialect::RegisterType(ir::AbstractType &&abstract_type) {
   ir::AbstractType *new_abstract_type =
       new ir::AbstractType(std::move(abstract_type));
@@ -35,4 +37,18 @@ void Dialect::RegisterAttribute(ir::AbstractAttribute &&abstract_attribute) {
 void Dialect::RegisterOp(const std::string &name, OpInfoImpl *op_info) {
   this->ir_context()->RegisterOpInfo(name, op_info);
 }
+
+void Dialect::RegisterInterface(std::unique_ptr<DialectInterface> interface) {
+  VLOG(4) << "Register interface into dialect" << std::endl;
+  auto it = registered_interfaces_.emplace(interface->interface_id(),
+                                           std::move(interface));
+  (void)it;
+}
+
+DialectInterface::~DialectInterface() = default;
+
+IrContext *DialectInterface::ir_context() const {
+  return dialect_->ir_context();
+}
+
 }  // namespace ir
diff --git a/paddle/ir/dialect.h b/paddle/ir/dialect.h
index 10adc6b3e2e0b..9a6b42dad5aed 100644
--- a/paddle/ir/dialect.h
+++ b/paddle/ir/dialect.h
@@ -15,11 +15,13 @@
 #pragma once
 
 #include "paddle/ir/attribute_base.h"
+#include "paddle/ir/dialect_interface.h"
 #include "paddle/ir/ir_context.h"
 #include "paddle/ir/op_info_impl.h"
 #include "paddle/ir/type_base.h"
 
 namespace ir {
+class DialectInterface;
 ///
 /// \brief Dialect can basically be understood as a namespace. In Dialect, we
 /// can define a series of types, attributes, operations, etc. An instance of
@@ -31,6 +33,8 @@ class Dialect {
  public:
   Dialect(std::string name, ir::IrContext *context, ir::TypeId id);
 
+  virtual ~Dialect();
+
   const std::string &name() const { return name_; }
 
   ir::IrContext *ir_context() const { return context_; }
@@ -49,8 +53,6 @@ class Dialect {
   template <typename T>
   void RegisterType() {
     VLOG(4) << "Type registered into Dialect. --->";
-    // if (this->ir_context()->registed_abstract_type().count(
-    //         ir::TypeId::get<T>()) == 0) {
     if (this->ir_context()->GetRegisteredAbstractType(ir::TypeId::get<T>()) ==
         nullptr) {
       ir::AbstractType *abstract_type =
@@ -118,11 +120,50 @@ class Dialect {
 
   void RegisterOp(const std::string &name, OpInfoImpl *op_info);
 
+  ///
+  /// \brief Register interface methods.
+  ///
+
+  DialectInterface *GetRegisteredInterface(TypeId id) {
+    auto it = registered_interfaces_.find(id);
+    return it != registered_interfaces_.end() ? it->second.get() : nullptr;
+  }
+
+  template <typename InterfaceT>
+  InterfaceT *GetRegisteredInterface() {
+    return static_cast<InterfaceT *>(
+        GetRegisteredInterface(TypeId::get<InterfaceT>()));
+  }
+
+  /// Register a dialect interface with this dialect instance.
+  void RegisterInterface(std::unique_ptr<DialectInterface> interface);
+
+  /// Register a set of dialect interfaces with this dialect instance.
+  template <typename... Args>
+  void RegisterInterfaces() {
+    (void)std::initializer_list<int>{
+        0, (RegisterInterface(std::make_unique<Args>(this)), 0)...};
+  }
+
+  template <typename InterfaceT, typename... Args>
+  InterfaceT &RegisterInterface(Args &&...args) {
+    InterfaceT *interface = new InterfaceT(this, std::forward<Args>(args)...);
+    RegisterInterface(std::unique_ptr<DialectInterface>(interface));
+    return *interface;
+  }
+
  private:
+  Dialect(const Dialect &) = delete;
+
+  Dialect &operator=(Dialect &) = delete;
+
   std::string name_;
 
   ir::IrContext *context_;  // not owned
 
   ir::TypeId id_;
+
+  std::unordered_map<TypeId, std::unique_ptr<DialectInterface>>
+      registered_interfaces_;
 };
 }  // namespace ir
diff --git a/paddle/ir/dialect_interface.h b/paddle/ir/dialect_interface.h
new file mode 100644
index 0000000000000..dd3ec08af52fd
--- /dev/null
+++ b/paddle/ir/dialect_interface.h
@@ -0,0 +1,67 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/ir/type_id.h"
+
+namespace ir {
+class Dialect;
+class IrContext;
+///
+/// \brief DialectInterface
+///
+template <typename ConcreteType, typename BaseT>
+class DialectInterfaceBase : public BaseT {
+ public:
+  using Base = DialectInterfaceBase<ConcreteType, BaseT>;
+
+  /// Get a unique id for the derived interface type.
+  static TypeId id() { return TypeId::get<ConcreteType>(); }
+
+ protected:
+  explicit DialectInterfaceBase(Dialect *dialect) : BaseT(dialect, id()) {}
+};
+
+class DialectInterface {
+ public:
+  virtual ~DialectInterface();
+
+  /// The base class used for all derived interface types. This class provides
+  /// utilities necessary for registration.
+  template <typename ConcreteType>
+  using Base = DialectInterfaceBase<ConcreteType, DialectInterface>;
+
+  /// Return the dialect that this interface represents.
+  Dialect *dialect() const { return dialect_; }
+
+  /// Return the context that holds the parent dialect of this interface.
+  IrContext *ir_context() const;
+
+  /// Return the derived interface id.
+  TypeId interface_id() const { return interface_id_; }
+
+ protected:
+  DialectInterface(Dialect *dialect, TypeId id)
+      : dialect_(dialect), interface_id_(id) {}
+
+ private:
+  /// The dialect that represents this interface.
+  Dialect *dialect_;
+
+  /// The unique identifier for the derived interface type.
+  TypeId interface_id_;
+};
+
+}  // namespace ir
diff --git a/paddle/ir/op_info_impl.h b/paddle/ir/op_info_impl.h
index 78ca586d9d209..4380866bceb81 100644
--- a/paddle/ir/op_info_impl.h
+++ b/paddle/ir/op_info_impl.h
@@ -236,6 +236,12 @@ class OpInfoImpl {
 
   ir::Dialect *dialect() const { return dialect_; }
 
+  uint32_t AttributeNum() const { return num_attributes_; }
+
+  const char *GetAttributeByIndex(size_t idx) const {
+    return idx < num_attributes_ ? p_attributes_[idx] : nullptr;
+  }
+
  private:
   OpInfoImpl(uint32_t num_interfaces,
              uint32_t num_traits,
diff --git a/paddle/ir/operation.cc b/paddle/ir/operation.cc
index 9222ee7e1afbb..3cfe2b048e472 100644
--- a/paddle/ir/operation.cc
+++ b/paddle/ir/operation.cc
@@ -13,6 +13,8 @@
 // limitations under the License.
 
 #include "paddle/ir/operation.h"
+#include "paddle/ir/dialect.h"
+#include "paddle/ir/program.h"
 #include "paddle/ir/utils.h"
 
 namespace ir {
@@ -21,7 +23,7 @@ namespace ir {
 // OpInlineResult, Operation, Operand.
 Operation *Operation::create(const std::vector<ir::OpResult> &inputs,
                              const std::vector<ir::Type> &output_types,
-                             ir::DictionaryAttribute attribute,
+                             const AttributeMap &attribute,
                              ir::OpInfo op_info) {
   // 1. Calculate the required memory size for OpResults + Operation +
   // OpOperands.
@@ -63,7 +65,7 @@ Operation *Operation::create(const std::vector<ir::OpResult> &inputs,
     new (base_ptr) detail::OpOperandImpl(inputs[idx].impl_, op);
     base_ptr += sizeof(detail::OpOperandImpl);
   }
-  VLOG(4) << "Construct an Operation: " << op->print();
+
   return op;
 }
 
@@ -83,9 +85,18 @@ void Operation::destroy() {
   // 2.1. Deconstruct OpResult.
   char *base_ptr = aligned_ptr;
   for (size_t idx = num_results_; idx > 0; idx--) {
-    if (!reinterpret_cast<detail::OpResultImpl *>(base_ptr)->use_empty()) {
-      throw("Cannot destroy a value that still has uses!");
+    // release the uses of this result
+    detail::OpOperandImpl *first_use =
+        reinterpret_cast<detail::OpResultImpl *>(base_ptr)->first_use();
+    if (first_use != nullptr) {
+      first_use->release_source();
+      detail::OpOperandImpl *next_use = first_use->next_use();
+      while (next_use != nullptr) {
+        next_use->release_source();
+        next_use = next_use->next_use();
+      }
     }
+    // destory the result
     if (idx > max_inline_result_num) {
       reinterpret_cast<detail::OpOutlineResultImpl *>(base_ptr)
           ->~OpOutlineResultImpl();
@@ -117,11 +128,8 @@ void Operation::destroy() {
 
 Operation::Operation(uint32_t num_results,
                      uint32_t num_operands,
-                     ir::DictionaryAttribute attribute,
+                     const AttributeMap &attribute,
                      ir::OpInfo op_info) {
-  if (!attribute) {
-    throw("unexpected null attribute dictionary");
-  }
   num_results_ = num_results;
   num_operands_ = num_operands;
   attribute_ = attribute;
@@ -153,6 +161,15 @@ ir::OpResult Operation::GetResultByIndex(uint32_t index) {
   }
 }
 
+ir::OpOperand Operation::GetOperandByIndex(uint32_t index) {
+  if (index >= num_operands_) {
+    throw("index exceeds OP input range.");
+  }
+  char *ptr = reinterpret_cast<char *>(this) + sizeof(Operation) +
+              (index) * sizeof(detail::OpOperandImpl);
+  return ir::OpOperand(reinterpret_cast<detail::OpOperandImpl *>(ptr));
+}
+
 std::string Operation::print() {
   std::stringstream result;
   result << "{ " << num_results_ << " outputs, " << num_operands_
@@ -173,4 +190,9 @@ std::string Operation::print() {
   return result.str();
 }
 
+std::string Operation::op_name() const {
+  return op_info_.impl()->dialect()->name() + "." +
+         std::string(op_info_.impl()->name());
+}
+
 }  // namespace ir
diff --git a/paddle/ir/operation.h b/paddle/ir/operation.h
index a51043ad5687e..9730244851fc4 100644
--- a/paddle/ir/operation.h
+++ b/paddle/ir/operation.h
@@ -24,25 +24,35 @@ template <class ConcreteTrait>
 class OpTraitBase;
 template <typename ConcreteInterface>
 class OpInterfaceBase;
+class Program;
+
+using AttributeMap = std::unordered_map<std::string, Attribute>;
 
 class alignas(8) Operation final {
  public:
   ///
   /// \brief Malloc memory and construct objects in the following order:
   /// OpResultImpls|Operation|OpOperandImpls.
+  /// NOTE: Similar to new and delete, the destroy() and the create() need to be
+  /// used in conjunction.
   ///
   static Operation *create(const std::vector<ir::OpResult> &inputs,
                            const std::vector<ir::Type> &output_types,
-                           ir::DictionaryAttribute attribute,
+                           const AttributeMap &attribute,
                            ir::OpInfo op_info);
 
+  ///
+  /// \brief Destroy the operation objects and free memeory by create().
+  ///
   void destroy();
 
   ir::OpResult GetResultByIndex(uint32_t index);
 
+  ir::OpOperand GetOperandByIndex(uint32_t index);
+
   std::string print();
 
-  ir::DictionaryAttribute attribute() const { return attribute_; }
+  const AttributeMap &attribute() const { return attribute_; }
 
   ir::OpInfo op_info() const { return op_info_; }
 
@@ -50,6 +60,8 @@ class alignas(8) Operation final {
 
   uint32_t num_operands() const { return num_operands_; }
 
+  std::string op_name() const;
+
   template <typename T>
   T dyn_cast() const {
     return CastUtil<T>::call(this);
@@ -65,10 +77,16 @@ class alignas(8) Operation final {
     return op_info_.HasInterface<Interface>();
   }
 
+  Program *parent_program() const { return parent_program_; }
+
+  void set_parent_program(Program *parent_program) {
+    parent_program_ = parent_program;
+  }
+
  private:
   Operation(uint32_t num_results,
             uint32_t num_operands,
-            ir::DictionaryAttribute attribute,
+            const AttributeMap &attribute,
             ir::OpInfo op_info);
 
   template <typename T, typename Enabler = void>
@@ -92,13 +110,15 @@ class alignas(8) Operation final {
     }
   };
 
-  ir::DictionaryAttribute attribute_;
+  AttributeMap attribute_;
 
   ir::OpInfo op_info_;
 
   uint32_t num_results_ = 0;
 
   uint32_t num_operands_ = 0;
+
+  ir::Program *parent_program_{nullptr};
 };
 
 }  // namespace ir
diff --git a/paddle/ir/parameter.h b/paddle/ir/parameter.h
new file mode 100644
index 0000000000000..38d74ea3cbff2
--- /dev/null
+++ b/paddle/ir/parameter.h
@@ -0,0 +1,70 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/ir/type.h"
+
+namespace ir {
+///
+/// \brief Parameter represents the weight in the calculation graph.
+///
+class Parameter {
+ public:
+  Parameter(void* data, size_t size, ir::Type type) {
+    data_ = malloc(size);
+    memcpy(data_, data, size);
+    size_ = size;
+    type_ = type;
+  }
+
+  Parameter(const Parameter& param) {
+    data_ = malloc(param.size_);
+    memcpy(data_, param.data_, param.size_);
+    size_ = param.size_;
+    type_ = param.type_;
+  }
+
+  Parameter& operator=(const Parameter& param) {
+    data_ = malloc(param.size_);
+    memcpy(data_, param.data_, param.size_);
+    size_ = param.size_;
+    type_ = param.type_;
+    return *this;
+  }
+
+  ~Parameter() { free(data_); }
+
+  Type type() const { return type_; }
+
+  void* data() const { return data_; }
+
+  bool is_mutable() const { return is_mutable_; }
+
+  void set_mutable() { is_mutable_ = true; }
+
+ private:
+  void* data_;
+
+  ///
+  /// \brief Number of bytes held in data_.
+  ///
+  size_t size_;
+
+  Type type_;
+
+  bool is_mutable_ = false;
+};
+
+}  // namespace ir
diff --git a/paddle/ir/program.cc b/paddle/ir/program.cc
new file mode 100644
index 0000000000000..4caa7a8051341
--- /dev/null
+++ b/paddle/ir/program.cc
@@ -0,0 +1,42 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/ir/program.h"
+#include "paddle/ir/ir_context.h"
+
+namespace ir {
+Program::~Program() {
+  for (auto op : ops_) {
+    op->destroy();
+  }
+}
+
+void Program::InsertOp(Operation* op) {
+  ops_.push_back(op);
+  op->set_parent_program(this);
+}
+
+Parameter* Program::GetParameter(std::string name) const {
+  if (parameters_.count(name) != 0) {
+    return parameters_.at(name).get();
+  }
+  return nullptr;
+}
+
+void Program::SetParameter(std::string name,
+                           std::unique_ptr<Parameter>&& parameter) {
+  parameters_[name].reset(parameter.release());
+}
+
+}  // namespace ir
diff --git a/paddle/ir/program.h b/paddle/ir/program.h
new file mode 100644
index 0000000000000..8b0a54d77c321
--- /dev/null
+++ b/paddle/ir/program.h
@@ -0,0 +1,59 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <list>
+#include <unordered_map>
+
+#include "paddle/ir/builtin_attribute.h"
+#include "paddle/ir/operation.h"
+#include "paddle/ir/parameter.h"
+
+namespace ir {
+///
+/// \brief Program is an abstraction of model structure, divided into
+/// computational graphs and weights. At the current stage, a computational
+/// graph is represented in the form of a list<Operation *>. Todo: In the
+/// future, detailed design of control flow operators will be carried out, and
+/// concepts such as basic blocks, closures, and functions will be introduced to
+/// continuously improve Program's ability to represent computational graphs.
+///
+class Program {
+ public:
+  ~Program();
+
+  std::list<Operation*> ops() const { return ops_; }
+
+  size_t parameters_num() const { return parameters_.size(); }
+
+  ///
+  /// \brief Insert the Operation* constructed by Operation::create(...) into
+  /// this Program. NOTE: At this time, the memory management permission of
+  /// Operation* will be owned by this Program. The user does not need to call
+  /// Operation::destroy() manually
+  ///
+  void InsertOp(Operation* op);
+
+  Parameter* GetParameter(std::string name) const;
+
+  void SetParameter(std::string name, std::unique_ptr<Parameter>&& parameter);
+
+ private:
+  std::list<Operation*> ops_;  // owned
+
+  std::unordered_map<std::string, std::unique_ptr<Parameter>> parameters_;
+};
+
+}  // namespace ir
diff --git a/paddle/ir/type.h b/paddle/ir/type.h
index e44157365744b..37ceb39a687fe 100644
--- a/paddle/ir/type.h
+++ b/paddle/ir/type.h
@@ -57,7 +57,7 @@ class Type {
 
   const Storage *storage() const { return storage_; }
 
-  const Dialect &dialect() const { return storage_->abstract_type().dialect(); }
+  Dialect &dialect() const { return storage_->abstract_type().dialect(); }
 
   IrContext *ir_context() const;
 
diff --git a/paddle/ir/type_base.h b/paddle/ir/type_base.h
index 0757810e9b746..c78cd54c9b392 100644
--- a/paddle/ir/type_base.h
+++ b/paddle/ir/type_base.h
@@ -62,7 +62,7 @@ class AbstractType {
   ///
   /// \return The dialect this type was registered to.
   ///
-  const Dialect &dialect() const { return dialect_; }
+  Dialect &dialect() const { return const_cast<Dialect &>(dialect_); }
 
   ///
   /// \brief Find the AbstractType instance whose TypeId is type_id from
diff --git a/paddle/ir/value.cc b/paddle/ir/value.cc
index 2f1d5e5616f41..5c066902e3039 100644
--- a/paddle/ir/value.cc
+++ b/paddle/ir/value.cc
@@ -106,6 +106,10 @@ ir::Operation *OpOperandImpl::owner() const { return owner_; }
 
 ir::detail::OpOperandImpl *OpOperandImpl::next_use() { return next_use_; }
 
+ir::Value OpOperandImpl::source() const { return source_; }
+
+void OpOperandImpl::release_source() { source_ = nullptr; }
+
 OpOperandImpl::OpOperandImpl(ir::Value source, ir::Operation *owner)
     : source_(source), owner_(owner) {
   prev_use_addr_ = source.impl()->first_use_addr();
diff --git a/paddle/ir/value_impl.h b/paddle/ir/value_impl.h
index 2fa236dddd833..145c937b0cf00 100644
--- a/paddle/ir/value_impl.h
+++ b/paddle/ir/value_impl.h
@@ -31,6 +31,10 @@ class OpOperandImpl {
 
   ir::detail::OpOperandImpl *next_use();
 
+  ir::Value source() const;
+
+  void release_source();
+
   /// Remove this operand from the current use list.
   void remove_from_ud_chain();
 
diff --git a/test/cpp/ir/CMakeLists.txt b/test/cpp/ir/CMakeLists.txt
index 9d1d9b9d42b75..0c5385a32fbfa 100644
--- a/test/cpp/ir/CMakeLists.txt
+++ b/test/cpp/ir/CMakeLists.txt
@@ -3,4 +3,13 @@ if(WITH_NEWIR)
   cc_test_old(ir_attribute_test SRCS ir_attribute_test.cc DEPS new_ir gtest)
   cc_test_old(ir_value_test SRCS ir_value_test.cc DEPS new_ir gtest)
   cc_test_old(ir_op_test SRCS ir_op_test.cc DEPS new_ir gtest)
+  cc_test_old(
+    ir_program_test
+    SRCS
+    ir_program_test.cc
+    DEPS
+    new_ir
+    pd_dialect
+    phi
+    gtest)
 endif()
diff --git a/test/cpp/ir/ir_attribute_test.cc b/test/cpp/ir/ir_attribute_test.cc
index f4949cd30fb14..8aec0bec4b64d 100644
--- a/test/cpp/ir/ir_attribute_test.cc
+++ b/test/cpp/ir/ir_attribute_test.cc
@@ -58,34 +58,3 @@ TEST(attribute_test, built_in_attribute) {
   EXPECT_EQ(string_attr_cast_1.isa<ir::StrAttribute>(), true);
   EXPECT_EQ(string_attr_cast_1.size() == 8, 1);
 }
-
-TEST(attribute_test, dictionary_attribute) {
-  ir::IrContext *ctx = ir::IrContext::Instance();
-  std::string str_attr1_name = "attr1_name";
-  std::string str_attr1_value = "attr1_value";
-  ir::StrAttribute attr1_name = ir::StrAttribute::get(ctx, str_attr1_name);
-  ir::Attribute attr1_value = ir::StrAttribute::get(ctx, str_attr1_value);
-  std::string str_attr2_name = "attr2_name";
-  std::string str_attr2_value = "attr2_value";
-  ir::StrAttribute attr2_name = ir::StrAttribute::get(ctx, str_attr2_name);
-  ir::Attribute attr2_value = ir::StrAttribute::get(ctx, str_attr2_value);
-
-  std::map<ir::StrAttribute, ir::Attribute> named_attr1;
-  named_attr1.insert(
-      std::pair<ir::StrAttribute, ir::Attribute>(attr1_name, attr1_value));
-  named_attr1.insert(
-      std::pair<ir::StrAttribute, ir::Attribute>(attr2_name, attr2_value));
-  ir::DictionaryAttribute dic_attr1 =
-      ir::DictionaryAttribute::get(ctx, named_attr1);
-  std::map<ir::StrAttribute, ir::Attribute> named_attr2;
-  named_attr2.insert(
-      std::pair<ir::StrAttribute, ir::Attribute>(attr2_name, attr2_value));
-  named_attr2.insert(
-      std::pair<ir::StrAttribute, ir::Attribute>(attr1_name, attr1_value));
-  ir::DictionaryAttribute dic_attr2 =
-      ir::DictionaryAttribute::get(ctx, named_attr2);
-
-  EXPECT_EQ(dic_attr1, dic_attr2);
-  EXPECT_EQ(attr1_value, dic_attr1.GetValue(attr1_name));
-  EXPECT_EQ(attr2_value, dic_attr1.GetValue(attr2_name));
-}
diff --git a/test/cpp/ir/ir_op_test.cc b/test/cpp/ir/ir_op_test.cc
index 342eb5330e504..f6fbff5dd3b0e 100644
--- a/test/cpp/ir/ir_op_test.cc
+++ b/test/cpp/ir/ir_op_test.cc
@@ -99,15 +99,14 @@ class TestDialect : public ir::Dialect {
   void initialize() { RegisterOps<Operation1, Operation2>(); }
 };
 
-ir::DictionaryAttribute CreateAttribute(std::string attribute_name,
-                                        std::string attribute) {
+ir::AttributeMap CreateAttributeMap(std::string attribute_name,
+                                    std::string attribute) {
   ir::IrContext *ctx = ir::IrContext::Instance();
-  ir::StrAttribute attr_name = ir::StrAttribute::get(ctx, attribute_name);
   ir::Attribute attr_value = ir::StrAttribute::get(ctx, attribute);
-  std::map<ir::StrAttribute, ir::Attribute> named_attr;
-  named_attr.insert(
-      std::pair<ir::StrAttribute, ir::Attribute>(attr_name, attr_value));
-  return ir::DictionaryAttribute::get(ctx, named_attr);
+  ir::AttributeMap attr_map;
+  attr_map.insert(
+      std::pair<std::string, ir::Attribute>(attribute_name, attr_value));
+  return attr_map;
 }
 
 TEST(op_test, op_test) {
@@ -137,7 +136,7 @@ TEST(op_test, op_test) {
   ir::Operation *op =
       ir::Operation::create(op_inputs,
                             op_output_types,
-                            CreateAttribute("op1_name", "op1_attr"),
+                            CreateAttributeMap("op1_name", "op1_attr"),
                             op2_info);
 
   if (op->HasTrait<ReadOnlyTrait>()) {
diff --git a/test/cpp/ir/ir_program_test.cc b/test/cpp/ir/ir_program_test.cc
new file mode 100644
index 0000000000000..91e0fbf9bf4f8
--- /dev/null
+++ b/test/cpp/ir/ir_program_test.cc
@@ -0,0 +1,206 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include "paddle/fluid/dialect/pd_dialect.h"
+#include "paddle/fluid/dialect/pd_type.h"
+#include "paddle/fluid/dialect/utils.h"
+#include "paddle/ir/builtin_attribute.h"
+#include "paddle/ir/builtin_dialect.h"
+#include "paddle/ir/builtin_op.h"
+#include "paddle/ir/builtin_type.h"
+#include "paddle/ir/ir_context.h"
+#include "paddle/ir/program.h"
+#include "paddle/ir/utils.h"
+#include "paddle/phi/core/meta_tensor.h"
+#include "paddle/phi/infermeta/binary.h"
+#include "paddle/phi/kernels/elementwise_add_kernel.h"
+
+class AddOp : public ir::Op<AddOp> {
+ public:
+  using Op::Op;
+  static const char *name() { return "Add"; }
+  static const char **attributes_name_;
+  static uint32_t attributes_num() { return 0; }
+};
+const char **AddOp::attributes_name_ = nullptr;
+
+TEST(program_test, program) {
+  // (1) Init environment.
+  ir::IrContext *ctx = ir::IrContext::Instance();
+  ir::Dialect *builtin_dialect =
+      ctx->GetOrRegisterDialect<ir::BuiltinDialect>();
+  builtin_dialect->RegisterOp<AddOp>();
+  ir::Dialect *paddle_dialect =
+      ctx->GetOrRegisterDialect<paddle::dialect::PaddleDialect>();
+
+  // (2) Create an empty program object
+  ir::Program program;
+  //   ir::Program *program = new ir::Program();
+  EXPECT_EQ(program.ops().size() == 0, true);
+
+  // (3) Create a float32 DenseTensor Parameter and save into Program
+  ir::Type fp32_dtype = ir::Float32Type::get(ctx);
+  paddle::dialect::DenseTensorTypeStorage::Dim dims = {2, 2};
+  paddle::dialect::DenseTensorTypeStorage::DataLayout data_layout =
+      paddle::dialect::DenseTensorTypeStorage::DataLayout::NCHW;
+  paddle::dialect::DenseTensorTypeStorage::LoD lod = {{0, 1, 2}};
+  size_t offset = 0;
+  ir::Type dense_tensor_dtype = paddle::dialect::DenseTensorType::get(
+      ctx, fp32_dtype, dims, data_layout, lod, offset);
+
+  std::vector<float> data_a = {1, 2, 3, 4};
+  std::unique_ptr<ir::Parameter> parameter_a =
+      std::make_unique<ir::Parameter>(reinterpret_cast<void *>(data_a.data()),
+                                      4 * sizeof(float),
+                                      dense_tensor_dtype);
+  program.SetParameter("a", std::move(parameter_a));
+  EXPECT_EQ(program.parameters_num() == 1, true);
+
+  std::vector<float> data_b = {5, 6, 7, 8};
+  std::unique_ptr<ir::Parameter> parameter_b =
+      std::make_unique<ir::Parameter>(reinterpret_cast<void *>(data_b.data()),
+                                      4 * sizeof(float),
+                                      dense_tensor_dtype);
+  program.SetParameter("b", std::move(parameter_b));
+  EXPECT_EQ(program.parameters_num() == 2, true);
+
+  // (4) Def a = GetParameterOp("a"), and create DenseTensor for a.
+  std::string op1_name =
+      builtin_dialect->name() + "." + std::string(ir::GetParameterOp::name());
+  ir::OpInfoImpl *op1_info = ctx->GetRegisteredOpInfo(op1_name);
+  std::unordered_map<std::string, ir::Attribute> op1_attribute{
+      {"parameter_name", ir::StrAttribute::get(ctx, "a")}};
+  ir::Operation *op1 =
+      ir::Operation::create({}, {dense_tensor_dtype}, op1_attribute, op1_info);
+
+  program.InsertOp(op1);
+
+  EXPECT_EQ(op1->GetResultByIndex(0).type().dialect().id(),
+            paddle_dialect->id());
+  using Interface = paddle::dialect::ParameterConvertInterface;
+  Interface *a_interface = op1->GetResultByIndex(0)
+                               .type()
+                               .dialect()
+                               .GetRegisteredInterface<Interface>();
+  std::shared_ptr<paddle::framework::Variable> a_var =
+      a_interface->ParameterToVariable(program.GetParameter("a"));
+  const phi::DenseTensor &a_tensor = a_var->Get<phi::DenseTensor>();
+  EXPECT_EQ(a_tensor.numel(), 4);
+  EXPECT_EQ(a_tensor.dims(), phi::DDim(dims.data(), dims.size()));
+  EXPECT_EQ(a_tensor.dtype(), paddle::dialect::TransToPhiDataType(fp32_dtype));
+  EXPECT_EQ(a_tensor.layout(),
+            paddle::dialect::TransToPhiDataLayout(data_layout));
+  EXPECT_EQ(a_tensor.lod(), lod);
+  EXPECT_EQ(a_tensor.offset(), offset);
+  for (int64_t i = 0; i < a_tensor.numel(); i++) {
+    EXPECT_EQ(*(a_tensor.data<float>() + i), data_a[i]);
+  }
+
+  // (5) Def b = GetParameterOp("b"), and create DenseTensor for b.
+  std::string op2_name =
+      builtin_dialect->name() + "." + std::string(ir::GetParameterOp::name());
+  ir::OpInfoImpl *op2_info = ctx->GetRegisteredOpInfo(op2_name);
+  std::unordered_map<std::string, ir::Attribute> op2_attribute{
+      {"parameter_name", ir::StrAttribute::get(ctx, "b")}};
+  ir::Operation *op2 =
+      ir::Operation::create({}, {dense_tensor_dtype}, op2_attribute, op2_info);
+  program.InsertOp(op2);
+
+  EXPECT_EQ(op2->GetResultByIndex(0).type().dialect().id(),
+            paddle_dialect->id());
+  Interface *b_interface = op2->GetResultByIndex(0)
+                               .type()
+                               .dialect()
+                               .GetRegisteredInterface<Interface>();
+  std::shared_ptr<paddle::framework::Variable> b_var =
+      b_interface->ParameterToVariable(program.GetParameter("b"));
+  const phi::DenseTensor &b_tensor = b_var->Get<phi::DenseTensor>();
+  EXPECT_EQ(b_tensor.numel(), 4);
+  EXPECT_EQ(b_tensor.dims(), phi::DDim(dims.data(), dims.size()));
+  EXPECT_EQ(b_tensor.dtype(), paddle::dialect::TransToPhiDataType(fp32_dtype));
+  EXPECT_EQ(b_tensor.layout(),
+            paddle::dialect::TransToPhiDataLayout(data_layout));
+  EXPECT_EQ(b_tensor.lod(), lod);
+  EXPECT_EQ(b_tensor.offset(), offset);
+  for (int64_t i = 0; i < b_tensor.numel(); i++) {
+    EXPECT_EQ(*(b_tensor.data<float>() + i), data_b[i]);
+  }
+
+  // (6) Def c = AddOp(a, b), execute this op.
+  std::string op3_name =
+      builtin_dialect->name() + "." + std::string(AddOp::name());
+  ir::OpInfoImpl *op3_info = ctx->GetRegisteredOpInfo(op3_name);
+  std::unordered_map<std::string, ir::Attribute> op3_attribute;
+  ir::Operation *op3 = ir::Operation::create(
+      {op1->GetResultByIndex(0), op2->GetResultByIndex(0)},
+      {dense_tensor_dtype},
+      op3_attribute,
+      op3_info);
+  program.InsertOp(op3);
+
+  phi::CPUContext *dev_ctx = static_cast<phi::CPUContext *>(
+      paddle::platform::DeviceContextPool::Instance().Get(
+          paddle::platform::CPUPlace()));
+  phi::DenseTensor c_tensor =
+      phi::Add<float, phi::CPUContext>(*dev_ctx, a_tensor, b_tensor);
+  std::shared_ptr<paddle::framework::Variable> variable_c =
+      std::make_shared<paddle::framework::Variable>();
+  auto *dst_tensor = variable_c->GetMutable<phi::DenseTensor>();
+  *dst_tensor = c_tensor;
+  EXPECT_EQ(dst_tensor->numel(), b_tensor.numel());
+  EXPECT_EQ(dst_tensor->dims(), b_tensor.dims());
+  EXPECT_EQ(dst_tensor->dtype(), b_tensor.dtype());
+  EXPECT_EQ(dst_tensor->layout(), b_tensor.layout());
+  EXPECT_EQ(dst_tensor->lod(), b_tensor.lod());
+  EXPECT_EQ(dst_tensor->offset(), b_tensor.offset());
+  for (int64_t i = 0; i < dst_tensor->numel(); i++) {
+    EXPECT_EQ(*(dst_tensor->data<float>() + i), data_a[i] + data_b[i]);
+  }
+
+  // (7) Def SetParameterOp(c, "c")
+  std::string op4_name =
+      builtin_dialect->name() + "." + std::string(ir::SetParameterOp::name());
+  ir::OpInfoImpl *op4_info = ctx->GetRegisteredOpInfo(op4_name);
+  std::unordered_map<std::string, ir::Attribute> op4_attribute{
+      {"parameter_name", ir::StrAttribute::get(ctx, "c")}};
+  ir::Operation *op4 = ir::Operation::create(
+      {op3->GetResultByIndex(0)}, {}, op4_attribute, op4_info);
+  program.InsertOp(op4);
+
+  EXPECT_EQ(op4->GetOperandByIndex(0).impl()->source().type().dialect().id(),
+            paddle_dialect->id());
+  Interface *c_interface = op4->GetOperandByIndex(0)
+                               .impl()
+                               ->source()
+                               .type()
+                               .dialect()
+                               .GetRegisteredInterface<Interface>();
+  //   ir::Parameter *parameter_c =
+  //       c_interface->VariableToParameter(variable_c.get());
+  std::unique_ptr<ir::Parameter> parameter_c =
+      c_interface->VariableToParameter(variable_c.get());
+  EXPECT_EQ(parameter_c->type(), dense_tensor_dtype);
+  for (int64_t i = 0; i < dst_tensor->numel(); i++) {
+    EXPECT_EQ(*(dst_tensor->data<float>() + i),
+              *(static_cast<float *>(parameter_c->data()) + i));
+  }
+  program.SetParameter("c", std::move(parameter_c));
+
+  // (8) Traverse Program
+  std::list<ir::Operation *> ops = program.ops();
+  EXPECT_EQ(ops.size() == 4, true);
+  EXPECT_EQ(program.parameters_num() == 3, true);
+}
diff --git a/test/cpp/ir/ir_value_test.cc b/test/cpp/ir/ir_value_test.cc
index 102619905fe7b..e0f0d83312a89 100644
--- a/test/cpp/ir/ir_value_test.cc
+++ b/test/cpp/ir/ir_value_test.cc
@@ -23,16 +23,14 @@
 // This unittest is used to test the construction interfaces of value class and
 // operation. The constructed test scenario is: a = OP1(); b = OP2(); c = OP3(a,
 // b); d, e, f, g, h, i, j = OP4(a, c);
-
-ir::DictionaryAttribute CreateAttribute(std::string attribute_name,
-                                        std::string attribute) {
+ir::AttributeMap CreateAttributeMap(std::string attribute_name,
+                                    std::string attribute) {
   ir::IrContext *ctx = ir::IrContext::Instance();
-  ir::StrAttribute attr_name = ir::StrAttribute::get(ctx, attribute_name);
   ir::Attribute attr_value = ir::StrAttribute::get(ctx, attribute);
-  std::map<ir::StrAttribute, ir::Attribute> named_attr;
-  named_attr.insert(
-      std::pair<ir::StrAttribute, ir::Attribute>(attr_name, attr_value));
-  return ir::DictionaryAttribute::get(ctx, named_attr);
+  ir::AttributeMap attr_map;
+  attr_map.insert(
+      std::pair<std::string, ir::Attribute>(attribute_name, attr_value));
+  return attr_map;
 }
 
 TEST(value_test, value_test) {
@@ -43,7 +41,7 @@ TEST(value_test, value_test) {
   ir::Operation *op1 =
       ir::Operation::create(op1_inputs,
                             op1_output_types,
-                            CreateAttribute("op1_name", "op1_attr"),
+                            CreateAttributeMap("op1_name", "op1_attr"),
                             nullptr);
   std::cout << op1->print() << std::endl;
   // 2. Construct OP2: b = OP2();
@@ -52,7 +50,7 @@ TEST(value_test, value_test) {
   ir::Operation *op2 =
       ir::Operation::create(op2_inputs,
                             op2_output_types,
-                            CreateAttribute("op2_name", "op2_attr"),
+                            CreateAttributeMap("op2_name", "op2_attr"),
                             nullptr);
   std::cout << op2->print() << std::endl;
   // 3. Construct OP3: c = OP3(a, b);
@@ -62,7 +60,7 @@ TEST(value_test, value_test) {
   ir::Operation *op3 =
       ir::Operation::create(op3_inputs,
                             op3_output_types,
-                            CreateAttribute("op3_name", "op3_attr"),
+                            CreateAttributeMap("op3_name", "op3_attr"),
                             nullptr);
   std::cout << op3->print() << std::endl;
   // 4. Construct OP4: d, e, f, g, h, i, j = OP4(a, c);
@@ -75,7 +73,7 @@ TEST(value_test, value_test) {
   ir::Operation *op4 =
       ir::Operation::create(op4_inputs,
                             op4_output_types,
-                            CreateAttribute("op4_name", "op4_attr"),
+                            CreateAttributeMap("op4_name", "op4_attr"),
                             nullptr);
   std::cout << op4->print() << std::endl;
 
diff --git a/test/cpp/ir/type_test.cc b/test/cpp/ir/type_test.cc
index d21afdcb80a59..8613c9d6afa67 100644
--- a/test/cpp/ir/type_test.cc
+++ b/test/cpp/ir/type_test.cc
@@ -124,46 +124,10 @@ TEST(type_test, built_in_type) {
             &ir::AbstractType::lookup(int64_1.type_id(), ctx));
   EXPECT_EQ(ir::Int64Type::classof(int64_1), 1);
 
-  // Test 2: Test the parameteric built-in type of IrContext.
-  ir::DenseTensorTypeStorage::Dim dims = {1, 2, 3};
-  ir::DenseTensorTypeStorage::DataLayout data_layout =
-      ir::DenseTensorTypeStorage::DataLayout::NCHW;
-  ir::DenseTensorTypeStorage::LoD lod = {{1, 2, 3}, {4, 5, 6}};
-  size_t offset = 0;
-
-  ir::Type dense_tensor_1 =
-      ir::DenseTensorType::get(ctx, fp32_1, dims, data_layout, lod, offset);
-  ir::Type dense_tensor_2 =
-      ir::DenseTensorType::get(ctx, fp32_2, dims, data_layout, lod, offset);
-  ir::Type dense_tensor_3 =
-      ir::DenseTensorType::get(ctx, fp32_1, dims, data_layout, lod, 2);
-
-  EXPECT_EQ(dense_tensor_1, dense_tensor_2);
-  EXPECT_NE(dense_tensor_1, dense_tensor_3);
-  EXPECT_EQ(dense_tensor_1.type_id(), dense_tensor_2.type_id());
-  EXPECT_EQ(ir::DenseTensorType::classof(dense_tensor_1), 1);
-
-  ir::DenseTensorType dense_tensor_4 =
-      ir::DenseTensorType::get(ctx, fp32_1, dims, data_layout, lod, 2);
-  EXPECT_EQ(dense_tensor_4.offset() == 2, 1);
-  EXPECT_EQ(dense_tensor_4.dtype().isa<ir::Float32Type>(), true);
-  EXPECT_EQ(dense_tensor_4.data_layout(), data_layout);
-
-  // Test 3: Test isa and dyn_cast.
+  // Test 2: Test isa and dyn_cast.
   EXPECT_EQ(fp16_1.isa<ir::Float16Type>(), true);
   EXPECT_EQ(fp16_1.isa<ir::Float32Type>(), false);
-  EXPECT_EQ(fp16_1.isa<ir::DenseTensorType>(), false);
   EXPECT_EQ(fp16_1.isa<ir::Type>(), true);
-  EXPECT_EQ(dense_tensor_1.isa<ir::DenseTensorType>(), true);
-
-  ir::DenseTensorType dense_tensor_cast_1 =
-      dense_tensor_1.dyn_cast<ir::DenseTensorType>();
-  EXPECT_EQ(dense_tensor_cast_1.isa<ir::DenseTensorType>(), true);
-  EXPECT_EQ(dense_tensor_cast_1.offset() == 0, 1);
-  const ir::DenseTensorType dense_tensor_cast_2 =
-      ir::dyn_cast<ir::DenseTensorType>(dense_tensor_1);
-  EXPECT_EQ(dense_tensor_cast_2.isa<ir::DenseTensorType>(), true);
-  EXPECT_EQ(dense_tensor_cast_2.offset() == 0, 1);
 }
 
 // Customize a parameterized TypeStorage IntegerTypeStorage.

From 2cb280147afe74f28ed0bbe4904bd4473e2e93dd Mon Sep 17 00:00:00 2001
From: JYChen <zoooo0820@qq.com>
Date: Wed, 17 May 2023 14:52:35 +0800
Subject: [PATCH 376/405] remove fluid
 memory_usage_calc&model_stat&op_frequence (#53838)

---
 python/paddle/fluid/contrib/__init__.py       |   8 -
 .../paddle/fluid/contrib/memory_usage_calc.py | 119 ----------
 python/paddle/fluid/contrib/model_stat.py     | 211 ------------------
 python/paddle/fluid/contrib/op_frequence.py   | 109 ---------
 .../tests/unittests/test_memory_usage.py      |  74 ------
 5 files changed, 521 deletions(-)
 delete mode 100644 python/paddle/fluid/contrib/memory_usage_calc.py
 delete mode 100644 python/paddle/fluid/contrib/model_stat.py
 delete mode 100644 python/paddle/fluid/contrib/op_frequence.py
 delete mode 100644 python/paddle/fluid/tests/unittests/test_memory_usage.py

diff --git a/python/paddle/fluid/contrib/__init__.py b/python/paddle/fluid/contrib/__init__.py
index deeb9b2f15b82..7f8edf0c8cbc6 100644
--- a/python/paddle/fluid/contrib/__init__.py
+++ b/python/paddle/fluid/contrib/__init__.py
@@ -14,21 +14,13 @@
 # limitations under the License.
 
 
-from . import memory_usage_calc
-from .memory_usage_calc import *
-from . import op_frequence
-from .op_frequence import *
 from . import extend_optimizer
 from .extend_optimizer import *
-from . import model_stat
-from .model_stat import *
 
 from . import optimizer
 from .optimizer import *
 
 __all__ = []
 
-__all__ += memory_usage_calc.__all__
-__all__ += op_frequence.__all__
 __all__ += extend_optimizer.__all__
 __all__ += optimizer.__all__
diff --git a/python/paddle/fluid/contrib/memory_usage_calc.py b/python/paddle/fluid/contrib/memory_usage_calc.py
deleted file mode 100644
index 8a8dcb55a00b7..0000000000000
--- a/python/paddle/fluid/contrib/memory_usage_calc.py
+++ /dev/null
@@ -1,119 +0,0 @@
-#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-This module provides a memory usage calculate function for user.
-The purpose of this API is to allow users to estimate memory usage of
-a program under a special batch size, then user can set appropriate
-batch size to fully utilize a GPU.
-
-This API is still under active development and may change drastically.
-"""
-
-from .. import core
-from ..framework import Program, Variable
-
-__all__ = ['memory_usage']
-
-dtype_to_size = {
-    core.VarDesc.VarType.FP16: 2,
-    core.VarDesc.VarType.FP32: 4,
-    core.VarDesc.VarType.FP64: 8,
-    core.VarDesc.VarType.INT16: 2,
-    core.VarDesc.VarType.INT32: 4,
-    core.VarDesc.VarType.INT64: 8,
-    core.VarDesc.VarType.BOOL: 1,
-    core.VarDesc.VarType.UINT8: 1,
-}
-
-DEBUG = False
-
-
-def memory_usage(program, batch_size):
-    r"""
-    Get the estimate memory usage of program with input batch size.
-
-    Args:
-        program(Program): The current Program.
-        batch_size(int): The current input data batch_size.
-
-    Returns:
-        min_total_memory(float): the estimate memory usage lower bound.
-        max_total_memory(float): the estimate memory usage upper bound.
-        unit_str(string): the unit of estimate usage result.
-
-    Examples:
-
-        >>> import paddle.fluid as fluid
-        >>> lower_usage, upper_usage, unit = fluid.contrib.memory_usage(
-                fluid.default_main_program(), batch_size=10)
-        >>> print "memory usage is about %.3f - %.3f %s" % \
-                (lower_usage, upper_usage, unit)
-
-    """
-
-    # Parameters check
-    if not isinstance(program, Program):
-        raise TypeError(
-            "Calculating Memory Usage requires Program as its Parameter."
-            "But you passed in %s" % (type(program))
-        )
-    if batch_size <= 0:
-        raise ValueError("The batch size need to be positive.")
-
-    # Get the var_name list of first block and calculate
-    total_memory = 0.0
-    processed_var_names = set(["@EMPTY@"])
-    for op in program.global_block().ops:
-        for var_name in op.output_arg_names:
-            if var_name in processed_var_names:
-                continue
-            processed_var_names.add(var_name)
-            var = program.global_block().vars[var_name]
-            if var.desc.type() != core.VarDesc.VarType.LOD_TENSOR:
-                continue
-
-            data_count = 1
-            neg_dim_count = 0
-            for x in var.shape:
-                if x < 0:
-                    if neg_dim_count >= 1:
-                        raise ValueError(
-                            "Var %s has more than one negative dim."
-                            % (var_name)
-                        )
-                    neg_dim_count += 1
-                    data_count *= batch_size * (-x)
-                else:
-                    data_count *= x
-            var_memory = data_count * dtype_to_size[var.dtype]
-            if DEBUG:
-                print("%s memory usage: %d" % (var.name, var_memory))
-            total_memory += var_memory
-    if DEBUG:
-        print("total memory usage: %.2f" % (total_memory))
-
-    # Convert appropriate unit
-    unit_str = "B"
-    if total_memory > 1024:
-        total_memory /= 1024
-        unit_str = "KB"
-        if total_memory > 1024:
-            total_memory /= 1024
-            unit_str = "MB"
-
-    # Append extra memory consumption (5% - 10%)
-    min_total_memory = total_memory * 1.05
-    max_total_memory = total_memory * 1.1
-
-    return min_total_memory, max_total_memory, unit_str
diff --git a/python/paddle/fluid/contrib/model_stat.py b/python/paddle/fluid/contrib/model_stat.py
deleted file mode 100644
index 0c18deec5da22..0000000000000
--- a/python/paddle/fluid/contrib/model_stat.py
+++ /dev/null
@@ -1,211 +0,0 @@
-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-'''
-Example:
-    >>from paddle.fluid.contrib.model_stat import summary
-    >>main_program = ...
-    >>summary(main_program)
-    +-----+------------+----------------+----------------+---------+------------+
-    | No. |       TYPE |          INPUT |         OUTPUT |  PARAMs |      FLOPs |
-    +-----+------------+----------------+----------------+---------+------------+
-    |   0 |     conv2d |  (3, 200, 200) | (64, 100, 100) |    9408 |  188160000 |
-    |   1 | batch_norm | (64, 100, 100) | (64, 100, 100) |     256 |     640000 |
-    |   2 |       relu | (64, 100, 100) | (64, 100, 100) |       0 |     640000 |
-    |   3 |     pool2d | (64, 100, 100) |   (64, 50, 50) |       0 |    1440000 |
-    ...
-    | 176 |     conv2d |    (512, 7, 7) |    (512, 7, 7) | 2359296 |  231211008 |
-    | 177 |       relu |    (512, 7, 7) |    (512, 7, 7) |       0 |      25088 |
-    | 178 |     conv2d |    (512, 7, 7) |   (2048, 7, 7) | 1048576 |  102760448 |
-    | 179 |       relu |   (2048, 7, 7) |   (2048, 7, 7) |       0 |     100352 |
-    | 180 |     pool2d |   (2048, 7, 7) |   (2048, 1, 1) |       0 |     100352 |
-    +-----+------------+----------------+----------------+---------+------------+
-    Total PARAMs: 48017344(0.0480G)
-    Total FLOPs: 11692747751(11.69G)
-'''
-from collections import OrderedDict
-
-
-def summary(main_prog):
-    '''
-    It can summary model's PARAMS, FLOPs until now.
-    It support common operator like conv, fc, pool, relu, sigmoid, bn etc.
-    Args:
-        main_prog: main program
-    Returns:
-        print summary on terminal
-    '''
-    collected_ops_list = []
-    for one_b in main_prog.blocks:
-        block_vars = one_b.vars
-        for one_op in one_b.ops:
-            op_info = OrderedDict()
-            spf_res = _summary_model(block_vars, one_op)
-            if spf_res is None:
-                continue
-            # TODO: get the operator name
-            op_info['type'] = one_op.type
-            op_info['input_shape'] = spf_res[0][1:]
-            op_info['out_shape'] = spf_res[1][1:]
-            op_info['PARAMs'] = spf_res[2]
-            op_info['FLOPs'] = spf_res[3]
-            collected_ops_list.append(op_info)
-
-    summary_table, total = _format_summary(collected_ops_list)
-    _print_summary(summary_table, total)
-
-
-def _summary_model(block_vars, one_op):
-    '''
-    Compute operator's params and flops.
-    Args:
-        block_vars: all vars of one block
-        one_op: one operator to count
-    Returns:
-        in_data_shape: one operator's input data shape
-        out_data_shape: one operator's output data shape
-        params: one operator's PARAMs
-        flops: : one operator's FLOPs
-    '''
-    if one_op.type in ['conv2d', 'depthwise_conv2d']:
-        k_arg_shape = block_vars[one_op.input("Filter")[0]].shape
-        in_data_shape = block_vars[one_op.input("Input")[0]].shape
-        out_data_shape = block_vars[one_op.output("Output")[0]].shape
-        c_out, c_in, k_h, k_w = k_arg_shape
-        _, c_out_, h_out, w_out = out_data_shape
-        assert c_out == c_out_, 'shape error!'
-        k_groups = one_op.attr("groups")
-        kernel_ops = k_h * k_w * (c_in / k_groups)
-        bias_ops = 0 if one_op.input("Bias") == [] else 1
-        params = c_out * (kernel_ops + bias_ops)
-        flops = h_out * w_out * c_out * (kernel_ops + bias_ops)
-        # base nvidia paper, include mul and add
-        flops = 2 * flops
-
-    elif one_op.type == 'pool2d':
-        in_data_shape = block_vars[one_op.input("X")[0]].shape
-        out_data_shape = block_vars[one_op.output("Out")[0]].shape
-        _, c_out, h_out, w_out = out_data_shape
-        k_size = one_op.attr("ksize")
-        params = 0
-        flops = h_out * w_out * c_out * (k_size[0] * k_size[1])
-
-    elif one_op.type == 'mul':
-        k_arg_shape = block_vars[one_op.input("Y")[0]].shape
-        in_data_shape = block_vars[one_op.input("X")[0]].shape
-        out_data_shape = block_vars[one_op.output("Out")[0]].shape
-        # TODO: fc has mul ops
-        # add attr to mul op, tell us whether it belongs to 'fc'
-        # this's not the best way
-        if 'fc' not in one_op.output("Out")[0]:
-            return None
-        k_in, k_out = k_arg_shape
-        # bias in sum op
-        params = k_in * k_out + 1
-        flops = k_in * k_out
-
-    elif one_op.type in ['sigmoid', 'tanh', 'relu', 'leaky_relu', 'prelu']:
-        in_data_shape = block_vars[one_op.input("X")[0]].shape
-        out_data_shape = block_vars[one_op.output("Out")[0]].shape
-        params = 0
-        if one_op.type == 'prelu':
-            params = 1
-        flops = 1
-        for one_dim in in_data_shape:
-            flops *= one_dim
-
-    elif one_op.type == 'batch_norm':
-        in_data_shape = block_vars[one_op.input("X")[0]].shape
-        out_data_shape = block_vars[one_op.output("Y")[0]].shape
-        _, c_in, h_out, w_out = in_data_shape
-        # gamma, beta
-        params = c_in * 2
-        # compute mean and std
-        flops = h_out * w_out * c_in * 2
-
-    else:
-        return None
-
-    return in_data_shape, out_data_shape, params, flops
-
-
-def _format_summary(collected_ops_list):
-    '''
-    Format summary report.
-    Args:
-        collected_ops_list: the collected operator with summary
-    Returns:
-        summary_table: summary report format
-        total: sum param and flops
-    '''
-    _verify_dependent_package()
-
-    from prettytable import PrettyTable
-
-    summary_table = PrettyTable(
-        ["No.", "TYPE", "INPUT", "OUTPUT", "PARAMs", "FLOPs"]
-    )
-    summary_table.align = 'r'
-
-    total = {}
-    total_params = []
-    total_flops = []
-    for i, one_op in enumerate(collected_ops_list):
-        # notice the order
-        table_row = [
-            i,
-            one_op['type'],
-            one_op['input_shape'],
-            one_op['out_shape'],
-            int(one_op['PARAMs']),
-            int(one_op['FLOPs']),
-        ]
-        summary_table.add_row(table_row)
-        total_params.append(int(one_op['PARAMs']))
-        total_flops.append(int(one_op['FLOPs']))
-
-    total['params'] = total_params
-    total['flops'] = total_flops
-
-    return summary_table, total
-
-
-def _verify_dependent_package():
-    """
-    Verify whether `prettytable` is installed.
-    """
-    try:
-        from prettytable import PrettyTable
-    except ImportError:
-        raise ImportError(
-            "paddle.summary() requires package `prettytable`, place install it firstly using `pip install prettytable`. "
-        )
-
-
-def _print_summary(summary_table, total):
-    '''
-    Print all the summary on terminal.
-    Args:
-        summary_table: summary report format
-        total: sum param and flops
-    '''
-    parmas = total['params']
-    flops = total['flops']
-    print(summary_table)
-    print(
-        'Total PARAMs: {}({:.4f}M)'.format(sum(parmas), sum(parmas) / (10**6))
-    )
-    print('Total FLOPs: {}({:.2f}G)'.format(sum(flops), sum(flops) / 10**9))
-    print(
-        "Notice: \n now supported ops include [Conv, DepthwiseConv, FC(mul), BatchNorm, Pool, Activation(sigmoid, tanh, relu, leaky_relu, prelu)]"
-    )
diff --git a/python/paddle/fluid/contrib/op_frequence.py b/python/paddle/fluid/contrib/op_frequence.py
deleted file mode 100644
index ee435ac657af3..0000000000000
--- a/python/paddle/fluid/contrib/op_frequence.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from collections import OrderedDict
-
-from ..framework import Program
-
-__all__ = ['op_freq_statistic']
-
-
-def op_freq_statistic(program):
-    """
-    Statistics of Op frequency.
-
-    Args:
-        program(Program): The current Program.
-
-    Returns:
-        uni_op_freq(dict): the single op frequency.
-        adj_2_op_freq(dict): the two adjacent ops frequency.
-
-    Examples:
-
-        >>> import paddle.fluid as fluid
-        >>> uni_op_freq, adj_2_op_freq = fluid.contrib.op_freq_statistic(
-        >>>        fluid.default_main_program())
-        >>> for op_type, op_num in uni_op_freq:
-        >>>     print("%s  \t  %d" % (op_type, op_num))
-        >>> for op_type, op_num in adj_2_op_freq:
-        >>>     print("%s  \t  %d" % (op_type, op_num))
-
-    """
-
-    if not isinstance(program, Program):
-        raise TypeError(
-            "The input type should be Porgram."
-            "But you passed in %s" % (type(program))
-        )
-
-    uni_op_freq = OrderedDict()
-    adj_2_op_freq = OrderedDict()
-    op_in_ops = OrderedDict()
-
-    parameters = [p.name for p in program.blocks[0].all_parameters()]
-
-    # get uni_op_freq
-    for op in program.global_block().ops:
-        had_recorded = False
-        for var_name in op.output_arg_names:
-            if var_name in parameters:
-                continue
-            if not had_recorded and uni_op_freq.has_key(op.type):
-                uni_op_freq[op.type] += 1
-                had_recorded = True
-            elif not had_recorded:
-                uni_op_freq[op.type] = 1
-                had_recorded = True
-
-    # get adj_2_op_freq
-    var_gen_op = {}
-    for op in program.global_block().ops:
-        for var_name in op.input_arg_names:
-            if var_name in parameters:
-                continue
-            if var_gen_op.has_key(var_name):
-                assert len(var_gen_op[var_name]) > 0
-                if op_in_ops.has_key(op.type):
-                    op_in_ops[op.type].append(var_gen_op[var_name][-1])
-                else:
-                    op_in_ops[op.type] = [var_gen_op[var_name][-1]]
-            else:
-                print(
-                    "Var's generate op is not found,%s, %s"
-                    % (var_name, op.type)
-                )
-
-        for var_name in op.output_arg_names:
-            if var_gen_op.has_key(var_name):
-                var_gen_op[var_name].append(op.type)
-            else:
-                var_gen_op[var_name] = [op.type]
-
-    for op, in_ops in op_in_ops.iteritems():
-        for in_op in in_ops:
-            op_op = in_op + "->" + op
-            if adj_2_op_freq.has_key(op_op):
-                adj_2_op_freq[op_op] += 1
-            else:
-                adj_2_op_freq[op_op] = 1
-
-    uni_op_freq = sorted(
-        uni_op_freq.items(), key=lambda item: item[1], reverse=True
-    )
-    adj_2_op_freq = sorted(
-        adj_2_op_freq.items(), key=lambda item: item[1], reverse=True
-    )
-
-    return uni_op_freq, adj_2_op_freq
diff --git a/python/paddle/fluid/tests/unittests/test_memory_usage.py b/python/paddle/fluid/tests/unittests/test_memory_usage.py
deleted file mode 100644
index 55b3eaa998959..0000000000000
--- a/python/paddle/fluid/tests/unittests/test_memory_usage.py
+++ /dev/null
@@ -1,74 +0,0 @@
-#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import contextlib
-import unittest
-
-import paddle
-from paddle import fluid
-
-
-def train_simulator(test_batch_size=10):
-    if test_batch_size <= 0:
-        raise ValueError(
-            "batch_size should be a positive integeral value, "
-            "but got batch_size={}".format(test_batch_size)
-        )
-
-    x = paddle.static.data(name='x', shape=[-1, 13], dtype='float32')
-    y_predict = paddle.static.nn.fc(x, size=1, activation=None)
-    y = paddle.static.data(name='y', shape=[-1, 1], dtype='float32')
-
-    cost = paddle.nn.functional.square_error_cost(input=y_predict, label=y)
-    avg_cost = paddle.mean(cost)
-
-    sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001)
-    sgd_optimizer.minimize(avg_cost)
-
-    # Calculate memory usage in current network config
-    lower_usage, upper_usage, unit = fluid.contrib.memory_usage(
-        fluid.default_main_program(), batch_size=test_batch_size
-    )
-
-    print(
-        "memory usage is about %.3f - %.3f %s"
-        % (lower_usage, upper_usage, unit)
-    )
-
-
-class TestMemoryUsage(unittest.TestCase):
-    def test_with_unit_B(self):
-        with self.program_scope_guard():
-            train_simulator()
-
-    def test_with_unit_KB(self):
-        with self.program_scope_guard():
-            train_simulator(test_batch_size=1000)
-
-    def test_with_unit_MB(self):
-        with self.program_scope_guard():
-            train_simulator(test_batch_size=100000)
-
-    @contextlib.contextmanager
-    def program_scope_guard(self):
-        prog = fluid.Program()
-        startup_prog = fluid.Program()
-        scope = fluid.core.Scope()
-        with fluid.scope_guard(scope):
-            with fluid.program_guard(prog, startup_prog):
-                yield
-
-
-if __name__ == '__main__':
-    unittest.main()

From f6be95400d47b131ffd643735cf55c8742de37c9 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Wed, 17 May 2023 18:06:05 +0800
Subject: [PATCH 377/405] Polish codes.

---
 .../operators/fused/fused_gate_attention.h    | 91 +++++++++----------
 1 file changed, 42 insertions(+), 49 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 884c08ec09c8f..341e7fb1f8ff0 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -866,7 +866,6 @@ class FlashAttnWithGating {
                       phi::DenseTensor* qkv_transpose_out,
                       phi::DenseTensor* softmax_lse,
                       phi::DenseTensor* fmha_out,
-                      phi::DenseTensor* gate_out,
                       GateAttentionConfig<T>* config) {
 #ifdef PADDLE_WITH_FLASHATTN
     bool is_bf16 =
@@ -889,22 +888,16 @@ class FlashAttnWithGating {
     T* k_ptr = q_ptr + q_size;
     T* v_ptr = k_ptr + q_size;
 
-    // 2. Dealing with cu_seq_q and cu_seq_k for flash_attn.
+    // 2. flash_attn parameter setting.
     phi::DenseTensor cu_seq_q;
     phi::DenseTensor cu_seq_k;
-    int seq_batch_size = static_cast<int>(config->batch_size) *
-                         static_cast<int>(config->seq_len_m);
-    int64_t step = static_cast<int64_t>(config->seq_len_r);
-    AllocAndInitSeqQK(seq_batch_size, step, &cu_seq_q, &cu_seq_k);
+    InitArgumentsAndSeqTensors(config, &cu_seq_q, &cu_seq_k);
 
     // 3. Dealing with mask and bias for flash_attn.
     std::vector<int64_t> temp_mask_dim = GetCompressedDim(src_mask);
     std::vector<int64_t> temp_bias_dim = GetCompressedDim(nonbatched_bias);
 
-    // 4. flash_attn parameter setting.
-    InitFlashAttnArguments(config);
-
-    // 5. construct softmax_lse
+    // 4. construct softmax_lse
     int softmax_lse_last_dim = ((fa_max_seqlen_q_ + 16 - 1) / 16) * 16;
     softmax_lse->Resize({fa_batch_size_, fa_num_heads_, softmax_lse_last_dim});
     AllocWithDebugInfo<float>(dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
@@ -917,10 +910,9 @@ class FlashAttnWithGating {
       VLOG(6) << TensorDebugString(qkv_transpose_out, "qkv_transpose_out");
       VLOG(6) << TensorDebugString(softmax_lse, "softmax_lse");
       VLOG(6) << TensorDebugString(fmha_out, "fmha_out");
-      VLOG(6) << TensorDebugString(gate_out, "gate_out");
     }
 
-    // 6. flas_attn part one, get temp worksapce size.
+    // 5. flash_attn part one, get temp worksapce size.
     cudaStream_t stream = dev_ctx_.stream();
     uint64_t workspace_size;
     bool succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
@@ -952,11 +944,10 @@ class FlashAttnWithGating {
         nonbatched_bias ? nonbatched_bias->data() : nullptr,
         src_mask ? temp_mask_dim.data() : nullptr,
         nonbatched_bias ? temp_bias_dim.data() : nullptr);
-    if (!succ) {
-      PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
-    }
+    PADDLE_ENFORCE_EQ(
+        succ, true, phi::errors::External(phi::dynload::flash_attn_error()));
 
-    // 8. Run flash-attention kernel.
+    // 6. Run flash-attention kernel.
     phi::DenseTensor workspace = CreateWorkspace(workspace_size);
     succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
         static_cast<const void*>(q_ptr),
@@ -987,15 +978,9 @@ class FlashAttnWithGating {
         nonbatched_bias ? nonbatched_bias->data() : nullptr,
         src_mask ? temp_mask_dim.data() : nullptr,
         nonbatched_bias ? temp_bias_dim.data() : nullptr);
-    if (!succ) {
-      PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
-    }
+    PADDLE_ENFORCE_EQ(
+        succ, true, phi::errors::External(phi::dynload::flash_attn_error()));
     WaitWithDebugInfo(dev_ctx_);
-    VLOG(5) << "[ComputeForward]: Run SUCCESS!!!";
-
-    if (config->has_gating) {
-      gate_out->Resize(config->gate_out_dims);
-    }
 #else
     PADDLE_THROW(phi::errors::Unimplemented(
         "FlashAttention is unsupported, please set use_flash_attn to false."));
@@ -1036,12 +1021,10 @@ class FlashAttnWithGating {
     T* v_grad_ptr = k_grad_ptr + q_size;
     WaitWithDebugInfo(dev_ctx_);
 
-    // 2. Init with cu_seq_q and cu_seq_k for flash_attn.
-    phi::DenseTensor cu_seq_q, cu_seq_k;
-    int seq_batch_size = static_cast<int>(config->batch_size) *
-                         static_cast<int>(config->seq_len_m);
-    int64_t step = static_cast<int64_t>(config->seq_len_r);
-    AllocAndInitSeqQK(seq_batch_size, step, &cu_seq_q, &cu_seq_k);
+    // 2. flash_attn parameter setting.
+    phi::DenseTensor cu_seq_q;
+    phi::DenseTensor cu_seq_k;
+    InitArgumentsAndSeqTensors(config, &cu_seq_q, &cu_seq_k);
     const int32_t* cu_seq_q_ptr = cu_seq_q.data<int32_t>();
     const int32_t* cu_seq_k_ptr = cu_seq_k.data<int32_t>();
 
@@ -1049,10 +1032,7 @@ class FlashAttnWithGating {
     std::vector<int64_t> temp_mask_dim = GetCompressedDim(src_mask);
     std::vector<int64_t> temp_bias_dim = GetCompressedDim(nonbatched_bias);
 
-    // 4. flash_attn parameter setting.
-    InitFlashAttnArguments(config);
-
-    // 5. construct softmax_lse
+    // 4. construct softmax_lse
     phi::DenseTensor softmax_d;
     softmax_d.Resize(softmax_lse->dims());
     AllocWithDebugInfo<float>(dev_ctx_, "d_softmax_lse", &softmax_d);
@@ -1073,7 +1053,7 @@ class FlashAttnWithGating {
       VLOG(6) << TensorDebugString(&bias_d, "bias_d");
     }
 
-    // 6. flas_attn part one, get temp worksapce size.
+    // 5. flas_attn part one, get temp worksapce size.
     uint64_t workspace_size;
     cudaStream_t stream = dev_ctx_.stream();
     bool succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
@@ -1111,9 +1091,8 @@ class FlashAttnWithGating {
         nonbatched_bias ? nonbatched_bias->data() : nullptr,
         src_mask ? temp_mask_dim.data() : nullptr,
         nonbatched_bias ? temp_bias_dim.data() : nullptr);
-    if (!succ) {
-      PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
-    }
+    PADDLE_ENFORCE_EQ(
+        succ, true, phi::errors::External(phi::dynload::flash_attn_error()));
 
     phi::DenseTensor workspace = CreateWorkspace(workspace_size);
     succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
@@ -1151,11 +1130,9 @@ class FlashAttnWithGating {
         nonbatched_bias ? nonbatched_bias->data() : nullptr,
         src_mask ? temp_mask_dim.data() : nullptr,
         nonbatched_bias ? temp_bias_dim.data() : nullptr);
-    if (!succ) {
-      PADDLE_THROW(phi::errors::External(phi::dynload::flash_attn_error()));
-    }
+    PADDLE_ENFORCE_EQ(
+        succ, true, phi::errors::External(phi::dynload::flash_attn_error()));
     WaitWithDebugInfo(dev_ctx_);
-    VLOG(5) << "[ComputeBackward]: Run SUCCESS!!!";
 
     if (nonbatched_bias) {
       // compare block reduce
@@ -1239,17 +1216,20 @@ class FlashAttnWithGating {
     return gen->IncrementOffset(inc);
   }
 
-  void InitFlashAttnArguments(GateAttentionConfig<T>* config) {
+  void InitArgumentsAndSeqTensors(GateAttentionConfig<T>* config,
+                                  phi::DenseTensor* cu_seq_q,
+                                  phi::DenseTensor* cu_seq_k) {
     fa_batch_size_ = static_cast<int>(config->batch_size) *
                      static_cast<int>(config->seq_len_m);
-    fa_total_q_ =
-        fa_batch_size_ * static_cast<int>(config->seq_len_r);  // qkv_dims[1];
-    fa_total_k_ = fa_total_q_;                                 // qkv_dims[1];
-    fa_num_heads_ = static_cast<int>(config->num_heads);       // qkv_dims[2];
-    fa_head_size_ = static_cast<int>(config->head_dim);        // qkv_dims[3];
+    fa_num_heads_ = static_cast<int>(config->num_heads);  // qkv_dims[2];
+    fa_head_size_ = static_cast<int>(config->head_dim);   // qkv_dims[3];
     fa_max_seqlen_q_ = config->seq_len_r;
     fa_max_seqlen_k_ = config->m_size;
-    fa_num_splits_ = 0;  // 0 for an internal heuristic, which is optimal
+    fa_total_q_ = fa_batch_size_ * fa_max_seqlen_q_;
+    fa_total_k_ = fa_batch_size_ * fa_max_seqlen_k_;
+
+    // 0 for an internal heuristic, which is optimal
+    fa_num_splits_ = 0;
 
     fa_softmax_scale_ = static_cast<float>(1.0f / std::sqrt(fa_head_size_));
     fa_dropout_prob_ = 0.0f;
@@ -1261,6 +1241,19 @@ class FlashAttnWithGating {
 
     fa_zero_tensors_ = false;
 
+    phi::ArangeNullaryKernel<int32_t, phi::GPUContext>(
+        dev_ctx_,
+        0,
+        (fa_batch_size_ + 1) * fa_max_seqlen_q_,
+        fa_max_seqlen_q_,
+        cu_seq_q);
+    phi::ArangeNullaryKernel<int32_t, phi::GPUContext>(
+        dev_ctx_,
+        0,
+        (fa_batch_size_ + 1) * fa_max_seqlen_k_,
+        fa_max_seqlen_k_,
+        cu_seq_k);
+
     if (VLOG_IS_ON(6)) {
       VLOG(6) << "fa_batch_size    : " << fa_batch_size_;
       VLOG(6) << "fa_total_q       : " << fa_total_q_;

From 734dc4488834a58d8c22ecbf5371ffb4ae10498b Mon Sep 17 00:00:00 2001
From: risemeup1 <62429225+risemeup1@users.noreply.github.com>
Date: Wed, 17 May 2023 19:46:23 +0800
Subject: [PATCH 378/405] Supports offline compilation of Paddle third-party
 libraries (#53744)

* optimize logsumexp in small data scale

* fix

* fix

* add #pragma once

* compile protobuf offline

* add submodlu gflags

* check_submodules

* check_submodules

* add_submodule protobuf

* add_submodule_protobuf

* add_submodule

* add .gitmodules

* add_submodules

* fix_compiler error

* support offline compile

* support offline compile

* support offline_compile

* remove cub

* remove brpc

* support offline compile

* support offline compile

* canning patching on cryptopp

* modify .gitigonre of cryptopp

* test

* offline compile

* add_submodule zlib

* modify .gitmodules

* modify .gitmodules

* fix setup.py bug

* delete submodule cryptopp

* fix windows compile bug

* fix xxhash compile problem

---------

Co-authored-by: Asthestarsfalll <1186454801@qq.com>
Co-authored-by: Asthestarsfalll <72954905+Asthestarsfalll@users.noreply.github.com>
---
 .gitignore                      |  4 ++--
 .gitmodules                     | 39 ++++++++++++++++++++++++++++++
 cmake/external/dlpack.cmake     | 12 ++++------
 cmake/external/eigen.cmake      | 21 +++++++----------
 cmake/external/gflags.cmake     |  8 +++----
 cmake/external/glog.cmake       |  6 ++---
 cmake/external/gloo.cmake       | 22 ++++++++---------
 cmake/external/protobuf.cmake   |  8 +++----
 cmake/external/threadpool.cmake | 10 ++++----
 cmake/external/utf8proc.cmake   |  8 +++----
 cmake/external/warpctc.cmake    |  8 +++----
 cmake/external/warprnnt.cmake   |  8 +++----
 cmake/external/xxhash.cmake     | 16 ++++++-------
 cmake/external/zlib.cmake       |  8 +++----
 cmake/third_party.cmake         |  4 +++-
 setup.py                        | 42 ++++++++++++++++++++++++++++++++-
 third_party/dlpack              |  1 +
 third_party/eigen3              |  1 +
 third_party/gflags              |  1 +
 third_party/glog                |  1 +
 third_party/gloo                |  1 +
 third_party/leveldb             |  1 +
 third_party/protobuf            |  1 +
 third_party/threadpool          |  1 +
 third_party/utf8proc            |  1 +
 third_party/warpctc             |  1 +
 third_party/warprnnt            |  1 +
 third_party/xxhash              |  1 +
 third_party/zlib                |  1 +
 29 files changed, 153 insertions(+), 84 deletions(-)
 create mode 100644 .gitmodules
 create mode 160000 third_party/dlpack
 create mode 160000 third_party/eigen3
 create mode 160000 third_party/gflags
 create mode 160000 third_party/glog
 create mode 160000 third_party/gloo
 create mode 160000 third_party/leveldb
 create mode 160000 third_party/protobuf
 create mode 160000 third_party/threadpool
 create mode 160000 third_party/utf8proc
 create mode 160000 third_party/warpctc
 create mode 160000 third_party/warprnnt
 create mode 160000 third_party/xxhash
 create mode 160000 third_party/zlib

diff --git a/.gitignore b/.gitignore
index 6fef519e976c0..615017e30a6e7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -52,11 +52,11 @@ CMakeSettings.json
 Makefile
 .test_env/
 .cache/
-third_party/
+build/third_party/
 
 *~
 bazel-*
-third_party/
+build/third_party/
 
 build_*
 # clion workspace.
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000000000..29f526216b3f7
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,39 @@
+[submodule "third_party/protobuf"]
+	path = third_party/protobuf
+	url = https://github.com/protocolbuffers/protobuf.git
+[submodule "third_party/gflags"]
+	path = third_party/gflags
+	url = https://github.com/gflags/gflags.git
+[submodule "third_party/gloo"]
+	path = third_party/gloo
+	url = https://github.com/ziyoujiyi/gloo.git
+[submodule "third_party/dlpack"]
+	path = third_party/dlpack
+	url = https://github.com/dmlc/dlpack.git
+[submodule "third_party/utf8proc"]
+	path = third_party/utf8proc
+	url = https://github.com/JuliaStrings/utf8proc.git
+[submodule "third_party/warpctc"]
+	path = third_party/warpctc
+	url = https://github.com/baidu-research/warp-ctc.git
+[submodule "third_party/warprnnt"]
+	path = third_party/warprnnt
+	url = https://github.com/PaddlePaddle/warp-transducer.git
+[submodule "third_party/xxhash"]
+	path = third_party/xxhash
+	url = https://github.com/Cyan4973/xxHash.git
+[submodule "third_party/eigen3"]
+	path = third_party/eigen3
+	url = https://gitlab.com/libeigen/eigen.git
+[submodule "third_party/leveldb"]
+	path = third_party/leveldb
+	url = https://github.com/google/leveldb
+[submodule "third_party/threadpool"]
+	path = third_party/threadpool
+	url = https://github.com/progschj/ThreadPool.git
+[submodule "third_party/zlib"]
+	path = third_party/zlib
+	url = https://github.com/madler/zlib.git
+[submodule "third_party/glog"]
+	path = third_party/glog
+	url = https://github.com/google/glog.git
diff --git a/cmake/external/dlpack.cmake b/cmake/external/dlpack.cmake
index fc631c1d184aa..60752d99e76fd 100644
--- a/cmake/external/dlpack.cmake
+++ b/cmake/external/dlpack.cmake
@@ -15,19 +15,15 @@
 include(ExternalProject)
 
 set(DLPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/dlpack)
-
-set(DLPACK_REPOSITORY ${GIT_URL}/dmlc/dlpack.git)
-
 set(DLPACK_TAG v0.4)
-
 set(DLPACK_INCLUDE_DIR ${THIRD_PARTY_PATH}/dlpack/src/extern_dlpack/include)
-include_directories(${DLPACK_INCLUDE_DIR})
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/dlpack)
+include_directories(${SOURCE_DIR}/include)
 
 ExternalProject_Add(
   extern_dlpack
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${DLPACK_REPOSITORY}
-  GIT_TAG ${DLPACK_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${DLPACK_PREFIX_DIR}
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""
diff --git a/cmake/external/eigen.cmake b/cmake/external/eigen.cmake
index 8ae880773a4f4..b47c4996e7c8a 100644
--- a/cmake/external/eigen.cmake
+++ b/cmake/external/eigen.cmake
@@ -17,8 +17,8 @@ include(ExternalProject)
 # update eigen to the commit id f612df27 on 03/16/2021
 set(EIGEN_PREFIX_DIR ${THIRD_PARTY_PATH}/eigen3)
 set(EIGEN_SOURCE_DIR ${THIRD_PARTY_PATH}/eigen3/src/extern_eigen3)
-set(EIGEN_REPOSITORY https://gitlab.com/libeigen/eigen.git)
 set(EIGEN_TAG f612df273689a19d25b45ca4f8269463207c4fee)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/eigen3)
 
 if(WIN32)
   add_definitions(-DEIGEN_STRONG_INLINE=inline)
@@ -28,14 +28,12 @@ elseif(LINUX)
     # which will cause compiler error of using __host__ funciont
     # in __host__ __device__
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Meta.h native_src)
-    file(TO_NATIVE_PATH ${EIGEN_SOURCE_DIR}/Eigen/src/Core/util/Meta.h
-         native_dst)
+    file(TO_NATIVE_PATH ${SOURCE_DIR}/Eigen/src/Core/util/Meta.h native_dst)
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/TensorReductionGpu.h
          native_src1)
-    file(
-      TO_NATIVE_PATH
-      ${EIGEN_SOURCE_DIR}/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h
-      native_dst1)
+    file(TO_NATIVE_PATH
+         ${SOURCE_DIR}/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h
+         native_dst1)
     set(EIGEN_PATCH_COMMAND cp ${native_src} ${native_dst} && cp ${native_src1}
                             ${native_dst1})
   endif()
@@ -54,18 +52,17 @@ if(CMAKE_COMPILER_IS_GNUCC)
     # See: [Why calling some `git` commands before `patch`?]
     set(EIGEN_PATCH_COMMAND
         git checkout -- . && git checkout ${EIGEN_TAG} && patch -Nd
-        ${EIGEN_SOURCE_DIR}/Eigen/src/Core/arch/SSE/ < ${complex_header})
+        ${SOURCE_DIR}/Eigen/src/Core/arch/SSE/ < ${complex_header})
   endif()
 endif()
 
-set(EIGEN_INCLUDE_DIR ${EIGEN_SOURCE_DIR})
+set(EIGEN_INCLUDE_DIR ${SOURCE_DIR})
 include_directories(${EIGEN_INCLUDE_DIR})
 
 ExternalProject_Add(
   extern_eigen3
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${EIGEN_REPOSITORY}
-  GIT_TAG ${EIGEN_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${EIGEN_PREFIX_DIR}
   UPDATE_COMMAND ""
   PATCH_COMMAND ${EIGEN_PATCH_COMMAND}
diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake
index d73469003eed3..06827732e5966 100755
--- a/cmake/external/gflags.cmake
+++ b/cmake/external/gflags.cmake
@@ -19,9 +19,8 @@ set(GFLAGS_PREFIX_DIR ${THIRD_PARTY_PATH}/gflags)
 set(GFLAGS_INCLUDE_DIR
     "${GFLAGS_INSTALL_DIR}/include"
     CACHE PATH "gflags include directory." FORCE)
-set(GFLAGS_REPOSITORY ${GIT_URL}/gflags/gflags.git)
 set(GFLAGS_TAG "v2.2.2")
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gflags)
 if(WIN32)
   set(GFLAGS_LIBRARIES
       "${GFLAGS_INSTALL_DIR}/lib/gflags_static.lib"
@@ -64,10 +63,9 @@ if(WITH_ARM_BRPC)
 else()
   ExternalProject_Add(
     extern_gflags
-    ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${GFLAGS_REPOSITORY}
-    GIT_TAG ${GFLAGS_TAG}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
     PREFIX ${GFLAGS_PREFIX_DIR}
+    SOURCE_DIR ${SOURCE_DIR}
     UPDATE_COMMAND ""
     BUILD_COMMAND ${BUILD_COMMAND}
     INSTALL_COMMAND ${INSTALL_COMMAND}
diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake
index 456c651a197f9..8e3e79e78cc9e 100755
--- a/cmake/external/glog.cmake
+++ b/cmake/external/glog.cmake
@@ -19,9 +19,8 @@ set(GLOG_INSTALL_DIR ${THIRD_PARTY_PATH}/install/glog)
 set(GLOG_INCLUDE_DIR
     "${GLOG_INSTALL_DIR}/include"
     CACHE PATH "glog include directory." FORCE)
-set(GLOG_REPOSITORY ${GIT_URL}/google/glog.git)
 set(GLOG_TAG v0.4.0)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/glog)
 if(WIN32)
   set(GLOG_LIBRARIES
       "${GLOG_INSTALL_DIR}/lib/glog.lib"
@@ -67,8 +66,7 @@ else()
   ExternalProject_Add(
     extern_glog
     ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${GLOG_REPOSITORY}
-    GIT_TAG ${GLOG_TAG}
+    SOURCE_DIR ${SOURCE_DIR}
     DEPENDS gflags
     PREFIX ${GLOG_PREFIX_DIR}
     UPDATE_COMMAND ""
diff --git a/cmake/external/gloo.cmake b/cmake/external/gloo.cmake
index ebdcabd770cb0..d5a757242fad2 100755
--- a/cmake/external/gloo.cmake
+++ b/cmake/external/gloo.cmake
@@ -25,12 +25,11 @@ set(GLOO_LIBRARY_DIR
     "${GLOO_INSTALL_DIR}/lib"
     CACHE PATH "gloo library directory." FORCE)
 # As we add extra features for gloo, we use the non-official repo
-set(GLOO_REPOSITORY ${GIT_URL}/ziyoujiyi/gloo.git)
 set(GLOO_TAG v0.0.3)
 set(GLOO_LIBRARIES
     "${GLOO_INSTALL_DIR}/lib/libgloo.a"
     CACHE FILEPATH "gloo library." FORCE)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gloo)
 set(GLOO_PATCH_COMMAND "")
 if(WITH_GPU)
   if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0 AND ${CMAKE_CXX_COMPILER_VERSION}
@@ -39,7 +38,7 @@ if(WITH_GPU)
          native_dst)
     set(GLOO_PATCH_COMMAND
         git checkout -- . && git checkout ${GLOO_TAG} &&patch -Nd
-        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst})
+        ${SOURCE_DIR}/gloo/transport/tcp < ${native_dst})
   endif()
 endif()
 
@@ -58,28 +57,27 @@ if(CMAKE_COMPILER_IS_GNUCC)
     # See: [Why calling some `git` commands before `patch`?]
     set(GLOO_PATCH_COMMAND
         git checkout -- . && git checkout ${GLOO_TAG} && patch -Nd
-        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch -Nd
-        ${GLOO_SOURCE_DIR}/gloo/ < ${types_header})
+        ${SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch -Nd
+        ${SOURCE_DIR}/gloo/ < ${types_header})
   endif()
 endif()
 include_directories(${GLOO_INCLUDE_DIR})
 
 ExternalProject_Add(
   ${GLOO_PROJECT}
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${GLOO_REPOSITORY}
-  GIT_TAG ${GLOO_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX "${GLOO_PREFIX_DIR}"
   UPDATE_COMMAND ""
   PATCH_COMMAND ${GLOO_PATCH_COMMAND}
   CONFIGURE_COMMAND ""
   BUILD_COMMAND
-    mkdir -p ${GLOO_SOURCE_DIR}/build && cd ${GLOO_SOURCE_DIR}/build && cmake ..
-    -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && ${CMAKE_COMMAND} --build . && mkdir
-    -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/glo
+    mkdir -p ${GLOO_SOURCE_DIR}/build && cd ${GLOO_SOURCE_DIR}/build && cmake
+    ${SOURCE_DIR} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && ${CMAKE_COMMAND}
+    --build . && mkdir -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/glo
   INSTALL_COMMAND ${CMAKE_COMMAND} -E copy
                   ${GLOO_SOURCE_DIR}/build/gloo/libgloo.a ${GLOO_LIBRARY_DIR}
-  COMMAND ${CMAKE_COMMAND} -E copy_directory "${GLOO_SOURCE_DIR}/gloo/"
+  COMMAND ${CMAKE_COMMAND} -E copy_directory "${SOURCE_DIR}/gloo/"
           "${GLOO_INCLUDE_DIR}/gloo"
   BUILD_BYPRODUCTS ${GLOO_LIBRARIES})
 
diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake
index 9ad5054b1bae9..8e18bc49442a2 100755
--- a/cmake/external/protobuf.cmake
+++ b/cmake/external/protobuf.cmake
@@ -188,6 +188,7 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
   set(PROTOBUF_SOURCE_DIR
       ${THIRD_PARTY_PATH}/${TARGET_DIR_NAME}/src/${TARGET_NAME})
   set(PROTOBUF_INSTALL_DIR ${THIRD_PARTY_PATH}/install/${TARGET_DIR_NAME})
+  set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/${TARGET_DIR_NAME})
 
   set(${TARGET_NAME}_INCLUDE_DIR
       "${PROTOBUF_INSTALL_DIR}/include"
@@ -289,14 +290,13 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
   else()
     ExternalProject_Add(
       ${TARGET_NAME}
-      ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-      GIT_REPOSITORY ${PROTOBUF_REPOSITORY}
-      GIT_TAG ${PROTOBUF_TAG}
+      ${EXTERNAL_PROJECT_LOG_ARGS}
       PREFIX ${PROTOBUF_PREFIX_DIR}
+      SOURCE_DIR ${SOURCE_DIR}
       UPDATE_COMMAND ""
       DEPENDS zlib
       CONFIGURE_COMMAND
-        ${CMAKE_COMMAND} ${PROTOBUF_SOURCE_DIR}/cmake ${OPTIONAL_ARGS}
+        ${CMAKE_COMMAND} ${SOURCE_DIR}/cmake ${OPTIONAL_ARGS}
         -G${CMAKE_GENERATOR} -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_SKIP_RPATH=ON
         -DCMAKE_POSITION_INDEPENDENT_CODE=ON
         -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
diff --git a/cmake/external/threadpool.cmake b/cmake/external/threadpool.cmake
index afeacdc833906..3d2af4d71e98c 100644
--- a/cmake/external/threadpool.cmake
+++ b/cmake/external/threadpool.cmake
@@ -15,17 +15,15 @@
 include(ExternalProject)
 
 set(THREADPOOL_PREFIX_DIR ${THIRD_PARTY_PATH}/threadpool)
-set(THREADPOOL_REPOSITORY ${GIT_URL}/progschj/ThreadPool.git)
 set(THREADPOOL_TAG 9a42ec1329f259a5f4881a291db1dcb8f2ad9040)
-
-set(THREADPOOL_INCLUDE_DIR ${THIRD_PARTY_PATH}/threadpool/src/extern_threadpool)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/threadpool)
+set(THREADPOOL_INCLUDE_DIR ${SOURCE_DIR})
 include_directories(${THREADPOOL_INCLUDE_DIR})
 
 ExternalProject_Add(
   extern_threadpool
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${THREADPOOL_REPOSITORY}
-  GIT_TAG ${THREADPOOL_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${THREADPOOL_PREFIX_DIR}
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""
diff --git a/cmake/external/utf8proc.cmake b/cmake/external/utf8proc.cmake
index 753c22fe2655e..138b325a5f127 100644
--- a/cmake/external/utf8proc.cmake
+++ b/cmake/external/utf8proc.cmake
@@ -17,9 +17,8 @@ include(ExternalProject)
 set(UTF8PROC_PREFIX_DIR ${THIRD_PARTY_PATH}/utf8proc)
 set(UTF8PROC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/utf8proc)
 # As we add extra features for utf8proc, we use the non-official repo
-set(UTF8PROC_REPOSITORY ${GIT_URL}/JuliaStrings/utf8proc.git)
 set(UTF8PROC_TAG v2.6.1)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/utf8proc)
 if(WIN32)
   set(UTF8PROC_LIBRARIES "${UTF8PROC_INSTALL_DIR}/lib/utf8proc_static.lib")
   add_definitions(-DUTF8PROC_STATIC)
@@ -31,9 +30,8 @@ include_directories(${UTF8PROC_INSTALL_DIR}/include)
 
 ExternalProject_Add(
   extern_utf8proc
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${UTF8PROC_REPOSITORY}
-  GIT_TAG ${UTF8PROC_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${UTF8PROC_PREFIX_DIR}
   UPDATE_COMMAND ""
   CMAKE_ARGS -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake
index 46befee8bd255..749de1b46ef91 100644
--- a/cmake/external/warpctc.cmake
+++ b/cmake/external/warpctc.cmake
@@ -22,9 +22,8 @@ set(WARPCTC_PREFIX_DIR ${THIRD_PARTY_PATH}/warpctc)
 set(WARPCTC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/warpctc)
 # in case of low internet speed
 #set(WARPCTC_REPOSITORY  https://gitee.com/tianjianhe/warp-ctc.git)
-set(WARPCTC_REPOSITORY ${GIT_URL}/baidu-research/warp-ctc.git)
 set(WARPCTC_TAG bdc2b4550453e0ef2d3b5190f9c6103a84eff184)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/warpctc)
 set(WARPCTC_SOURCE_DIR ${THIRD_PARTY_PATH}/warpctc/src/extern_warpctc)
 set(WARPCTC_PATCH_COMMAND "")
 set(WARPCTC_CCBIN_OPTION "")
@@ -85,9 +84,8 @@ endif()
 
 ExternalProject_Add(
   extern_warpctc
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${WARPCTC_REPOSITORY}
-  GIT_TAG ${WARPCTC_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${WARPCTC_PREFIX_DIR}
   UPDATE_COMMAND ""
   PATCH_COMMAND ${WARPCTC_PATCH_COMMAND}
diff --git a/cmake/external/warprnnt.cmake b/cmake/external/warprnnt.cmake
index 64197d5ec3043..2c4321e16f566 100644
--- a/cmake/external/warprnnt.cmake
+++ b/cmake/external/warprnnt.cmake
@@ -20,9 +20,8 @@ endif()
 
 set(WARPRNNT_PREFIX_DIR ${THIRD_PARTY_PATH}/warprnnt)
 set(WARPRNNT_INSTALL_DIR ${THIRD_PARTY_PATH}/install/warprnnt)
-set(WARPRNNT_REPOSITORY ${GIT_URL}/PaddlePaddle/warp-transducer.git)
 set(WARPRNNT_TAG 7ea6bfe748779c245a0fcaa5dd9383826273eff2)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/warprnnt)
 set(WARPRNNT_INCLUDE_DIR
     "${WARPRNNT_INSTALL_DIR}/include"
     CACHE PATH "Warp-rnnt Directory" FORCE)
@@ -70,9 +69,8 @@ else()
 endif()
 ExternalProject_Add(
   extern_warprnnt
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${WARPRNNT_REPOSITORY}
-  GIT_TAG ${WARPRNNT_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${WARPRNNT_PREFIX_DIR}
   UPDATE_COMMAND ""
   PATCH_COMMAND ""
diff --git a/cmake/external/xxhash.cmake b/cmake/external/xxhash.cmake
index 648a060f9b7e8..1e2989e359729 100644
--- a/cmake/external/xxhash.cmake
+++ b/cmake/external/xxhash.cmake
@@ -18,8 +18,8 @@ set(XXHASH_PREFIX_DIR ${THIRD_PARTY_PATH}/xxhash)
 set(XXHASH_SOURCE_DIR ${THIRD_PARTY_PATH}/xxhash/src/extern_xxhash)
 set(XXHASH_INSTALL_DIR ${THIRD_PARTY_PATH}/install/xxhash)
 set(XXHASH_INCLUDE_DIR "${XXHASH_INSTALL_DIR}/include")
-set(XXHASH_REPOSITORY ${GIT_URL}/Cyan4973/xxHash.git)
 set(XXHASH_TAG v0.6.5)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/xxhash)
 
 include_directories(${XXHASH_INCLUDE_DIR})
 
@@ -29,7 +29,7 @@ if(APPLE)
       -i
       \"\"
       "s/-Wstrict-prototypes -Wundef/-Wstrict-prototypes -Wundef -fPIC/g"
-      ${XXHASH_SOURCE_DIR}/Makefile
+      ${SOURCE_DIR}/Makefile
       &&
       make
       lib)
@@ -38,7 +38,7 @@ elseif(UNIX)
       sed
       -i
       "s/-Wstrict-prototypes -Wundef/-Wstrict-prototypes -Wundef -fPIC/g"
-      ${XXHASH_SOURCE_DIR}/Makefile
+      ${SOURCE_DIR}/Makefile
       &&
       make
       lib)
@@ -57,14 +57,13 @@ endif()
 if(WIN32)
   ExternalProject_Add(
     extern_xxhash
-    ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${XXHASH_REPOSITORY}
-    GIT_TAG ${XXHASH_TAG}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    SOURCE_DIR ${SOURCE_DIR}
     PREFIX ${XXHASH_PREFIX_DIR}
     UPDATE_COMMAND ""
     PATCH_COMMAND ""
     CONFIGURE_COMMAND
-      ${CMAKE_COMMAND} ${XXHASH_SOURCE_DIR}/cmake_unofficial
+      ${CMAKE_COMMAND} ${SOURCE_DIR}/cmake_unofficial
       -DCMAKE_INSTALL_PREFIX:PATH=${XXHASH_INSTALL_DIR}
       -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
       -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
@@ -83,8 +82,7 @@ else()
   ExternalProject_Add(
     extern_xxhash
     ${EXTERNAL_PROJECT_LOG_ARGS}
-    GIT_REPOSITORY ${XXHASH_REPOSITORY}
-    GIT_TAG ${XXHASH_TAG}
+    SOURCE_DIR ${SOURCE_DIR}
     PREFIX ${XXHASH_PREFIX_DIR}
     UPDATE_COMMAND ""
     CONFIGURE_COMMAND ""
diff --git a/cmake/external/zlib.cmake b/cmake/external/zlib.cmake
index c6ad1506bbe48..a9506ec8645c8 100644
--- a/cmake/external/zlib.cmake
+++ b/cmake/external/zlib.cmake
@@ -22,9 +22,8 @@ set(ZLIB_ROOT
 set(ZLIB_INCLUDE_DIR
     "${ZLIB_INSTALL_DIR}/include"
     CACHE PATH "zlib include directory." FORCE)
-set(ZLIB_REPOSITORY ${GIT_URL}/madler/zlib.git)
 set(ZLIB_TAG v1.2.8)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/zlib)
 include_directories(${ZLIB_INCLUDE_DIR}
 )# For zlib code to include its own headers.
 include_directories(${THIRD_PARTY_PATH}/install
@@ -42,9 +41,8 @@ endif()
 
 ExternalProject_Add(
   extern_zlib
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${ZLIB_REPOSITORY}
-  GIT_TAG ${ZLIB_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${ZLIB_PREFIX_DIR}
   UPDATE_COMMAND ""
   CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake
index f8cb9c85c5202..28032f50726ce 100755
--- a/cmake/third_party.cmake
+++ b/cmake/third_party.cmake
@@ -29,7 +29,9 @@ set(third_party_deps)
 
 include(ProcessorCount)
 ProcessorCount(NPROC)
-
+if(NOT WITH_SETUP_INSTALL)
+  execute_process(COMMAND git submodule update --init --recursive)
+endif()
 # cache funciton to avoid repeat download code of third_party.
 # This function has 4 parameters, URL / REPOSITOR / TAG / DIR:
 # 1. URL:           specify download url of 3rd party
diff --git a/setup.py b/setup.py
index 104120d9e8c2b..e10308e1dffb8 100644
--- a/setup.py
+++ b/setup.py
@@ -22,6 +22,7 @@
 import shutil
 import subprocess
 import sys
+import time
 from contextlib import contextmanager
 from distutils.spawn import find_executable
 from subprocess import CalledProcessError
@@ -1597,13 +1598,52 @@ def install_cpp_dist_and_build_test(install_dir, lib_test_dir, headers, libs):
     subprocess.check_call([CMAKE, "--build", lib_test_dir])
 
 
+def check_submodules():
+    def get_submodule_folder():
+        git_submodules_path = os.path.join(TOP_DIR, ".gitmodules")
+        with open(git_submodules_path) as f:
+            return [
+                os.path.join(TOP_DIR, line.split("=", 1)[1].strip())
+                for line in f.readlines()
+                if line.strip().startswith("path")
+            ]
+
+    def submodules_not_exists_or_empty(folder):
+        return not os.path.exists(folder) or (
+            os.path.isdir(folder) and len(os.listdir(folder)) == 0
+        )
+
+    submodule_folders = get_submodule_folder()
+    # f none of the submodule folders exists, try to initialize them
+    if any(
+        submodules_not_exists_or_empty(folder) for folder in submodule_folders
+    ):
+        try:
+            print(' --- Trying to initialize submodules')
+            start = time.time()
+            subprocess.check_call(
+                ["git", "submodule", "update", "--init", "--recursive"],
+                cwd=TOP_DIR,
+            )
+            end = time.time()
+            print(
+                ' --- Submodule initialization took {:.2f} sec'.format(
+                    end - start
+                )
+            )
+        except Exception:
+            print(' --- Submodule initalization failed')
+            print('Please run:\n\tgit submodule update --init --recursive')
+            sys.exit(1)
+
+
 def main():
     # Parse the command line and check arguments before we proceed with building steps and setup
     parse_input_command(filter_args_list)
 
     # check build dependency
     check_build_dependency()
-
+    check_submodules()
     # Execute the build process,cmake and make
     if cmake_and_build:
         build_steps()
diff --git a/third_party/dlpack b/third_party/dlpack
new file mode 160000
index 0000000000000..a07f962d446b5
--- /dev/null
+++ b/third_party/dlpack
@@ -0,0 +1 @@
+Subproject commit a07f962d446b577adf4baef2b347a0f3a2a20617
diff --git a/third_party/eigen3 b/third_party/eigen3
new file mode 160000
index 0000000000000..f612df273689a
--- /dev/null
+++ b/third_party/eigen3
@@ -0,0 +1 @@
+Subproject commit f612df273689a19d25b45ca4f8269463207c4fee
diff --git a/third_party/gflags b/third_party/gflags
new file mode 160000
index 0000000000000..e171aa2d15ed9
--- /dev/null
+++ b/third_party/gflags
@@ -0,0 +1 @@
+Subproject commit e171aa2d15ed9eb17054558e0b3a6a413bb01067
diff --git a/third_party/glog b/third_party/glog
new file mode 160000
index 0000000000000..96a2f23dca4cc
--- /dev/null
+++ b/third_party/glog
@@ -0,0 +1 @@
+Subproject commit 96a2f23dca4cc7180821ca5f32e526314395d26a
diff --git a/third_party/gloo b/third_party/gloo
new file mode 160000
index 0000000000000..8b6b61dfa0dca
--- /dev/null
+++ b/third_party/gloo
@@ -0,0 +1 @@
+Subproject commit 8b6b61dfa0dca02b226a01262bfcf0484382048f
diff --git a/third_party/leveldb b/third_party/leveldb
new file mode 160000
index 0000000000000..803d69203a62f
--- /dev/null
+++ b/third_party/leveldb
@@ -0,0 +1 @@
+Subproject commit 803d69203a62faf50f1b77897310a3a1fcae712b
diff --git a/third_party/protobuf b/third_party/protobuf
new file mode 160000
index 0000000000000..f0dc78d7e6e33
--- /dev/null
+++ b/third_party/protobuf
@@ -0,0 +1 @@
+Subproject commit f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c
diff --git a/third_party/threadpool b/third_party/threadpool
new file mode 160000
index 0000000000000..9a42ec1329f25
--- /dev/null
+++ b/third_party/threadpool
@@ -0,0 +1 @@
+Subproject commit 9a42ec1329f259a5f4881a291db1dcb8f2ad9040
diff --git a/third_party/utf8proc b/third_party/utf8proc
new file mode 160000
index 0000000000000..3203baa7374d6
--- /dev/null
+++ b/third_party/utf8proc
@@ -0,0 +1 @@
+Subproject commit 3203baa7374d67132384e2830b2183c92351bffc
diff --git a/third_party/warpctc b/third_party/warpctc
new file mode 160000
index 0000000000000..bdc2b4550453e
--- /dev/null
+++ b/third_party/warpctc
@@ -0,0 +1 @@
+Subproject commit bdc2b4550453e0ef2d3b5190f9c6103a84eff184
diff --git a/third_party/warprnnt b/third_party/warprnnt
new file mode 160000
index 0000000000000..7ea6bfe748779
--- /dev/null
+++ b/third_party/warprnnt
@@ -0,0 +1 @@
+Subproject commit 7ea6bfe748779c245a0fcaa5dd9383826273eff2
diff --git a/third_party/xxhash b/third_party/xxhash
new file mode 160000
index 0000000000000..7cc9639699f64
--- /dev/null
+++ b/third_party/xxhash
@@ -0,0 +1 @@
+Subproject commit 7cc9639699f64b750c0b82333dced9ea77e8436e
diff --git a/third_party/zlib b/third_party/zlib
new file mode 160000
index 0000000000000..5089329162165
--- /dev/null
+++ b/third_party/zlib
@@ -0,0 +1 @@
+Subproject commit 50893291621658f355bc5b4d450a8d06a563053d

From 9e045eebfc462ff5f8f68cc1a4950916e901a63c Mon Sep 17 00:00:00 2001
From: duanyanhui <45005871+YanhuiDua@users.noreply.github.com>
Date: Wed, 17 May 2023 21:03:58 +0800
Subject: [PATCH 379/405] [CustomDevice] suport device_guard for custom device
 (#53808)

* suport device_guard for npu

* fix comment

* fix typo
---
 .../interpreter/interpreter_util.cc           | 37 ++++++++++++++++
 paddle/fluid/framework/operator.cc            | 43 ++++++++++++++++++-
 paddle/fluid/framework/operator.h             |  3 ++
 python/paddle/fluid/framework.py              |  7 ++-
 4 files changed, 87 insertions(+), 3 deletions(-)

diff --git a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
index 2db10435fdae4..5b3f0f3738b1b 100644
--- a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
+++ b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc
@@ -36,6 +36,9 @@
 #include "paddle/fluid/platform/mkldnn_helper.h"
 #endif
 
+#ifdef PADDLE_WITH_CUSTOM_DEVICE
+#include "paddle/phi/backends/device_manager.h"
+#endif
 PADDLE_DEFINE_EXPORTED_bool(
     new_executor_log_memory_stats,
     false,
@@ -400,6 +403,40 @@ void ApplyDeviceGuard(const OperatorBase* op_base,
       }
       VLOG(3) << "Switch into " << expected_kernel_key->place_
               << " by device_guard.";
+    } else if (platform::is_custom_place(place)) {
+#ifdef PADDLE_WITH_CUSTOM_DEVICE
+      auto device_types = phi::DeviceManager::GetAllCustomDeviceTypes();
+      bool is_custom_device_op = false;
+      for (auto dev_type : device_types) {
+        if (op_device.find(dev_type) != std::string::npos) {
+          is_custom_device_op = true;
+          break;
+        }
+      }
+      PADDLE_ENFORCE_EQ(
+          is_custom_device_op,
+          true,
+          phi::errors::Unimplemented(
+              "Unsupported current device %s with Paddle CustomDevice ",
+              op_device));
+#else
+      VLOG(1) << string::Sprintf(
+          "Cannot use get_all_custom_device_type because you have installed"
+          "CPU/GPU version PaddlePaddle.\n"
+          "If you want to use get_all_custom_device_type, please try to "
+          "install CustomDevice version "
+          "PaddlePaddle by: pip install paddlepaddle\n");
+#endif
+      if (op_base->SupportCustomDevice()) {
+        expected_kernel_key->place_ = place;
+      } else {
+        expected_kernel_key->place_ = platform::CPUPlace();
+        LOG_FIRST_N(WARNING, 1) << "Op(" << op_base->Type()
+                                << ") has no Custom Place implementation. It "
+                                   "will be assigned to CPUPlace.";
+      }
+      VLOG(3) << "Switch into " << expected_kernel_key->place_
+              << " by device_guard.";
     } else {
       PADDLE_THROW(
           platform::errors::Fatal("Unsupported current place %s", op_device));
diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc
index 65d31b87ed355..992f475fd6919 100644
--- a/paddle/fluid/framework/operator.cc
+++ b/paddle/fluid/framework/operator.cc
@@ -1365,6 +1365,42 @@ bool OperatorWithKernel::SupportXPU() const {
 #endif
 }
 
+bool OperatorWithKernel::SupportCustomDevice() const {
+#ifdef PADDLE_WITH_CUSTOM_DEVICE
+  auto phi_kernels = phi::KernelFactory::Instance().SelectKernelMap(
+      phi::TransToPhiKernelName(type_));
+  auto has_phi_kernel =
+      std::any_of(phi_kernels.begin(),
+                  phi_kernels.end(),
+                  [](phi::KernelKeyMap::const_reference kern_pair) {
+                    return platform::is_custom_place(
+                        phi::TransToPhiPlace(kern_pair.first.backend()));
+                  });
+  if (has_phi_kernel) {
+    return true;
+  } else {
+    auto kernel_iter = OperatorWithKernel::AllOpKernels().find(type_);
+    if (kernel_iter == OperatorWithKernel::AllOpKernels().end()) {
+      return false;
+    } else {
+      auto& op_kernels = kernel_iter->second;
+      return std::any_of(
+          op_kernels.begin(),
+          op_kernels.end(),
+          [this](OpKernelMap::const_reference kern_pair) {
+            return platform::is_custom_place(kern_pair.first.place_);
+          });
+    }
+  }
+#else
+  PADDLE_THROW(platform::errors::PreconditionNotMet(
+      "should not call OperatorWithKernel::SupportCustomDevice() when not "
+      "compiled with "
+      "CustomDevice support."));
+  return false;
+#endif
+}
+
 bool OperatorWithKernel::SupportsMKLDNN(const phi::DataType data_type) const {
   auto phi_kernels = phi::KernelFactory::Instance().SelectKernelMap(
       phi::TransToPhiKernelName(type_));
@@ -2088,7 +2124,12 @@ OpKernelType OperatorWithKernel::InnerGetExpectedKernelType(
       // CPUKernel will be executed and a warning will be given at the same
       // time.
       expected_kernel_key.place_ = platform::CPUPlace();
-
+#ifdef PADDLE_WITH_CUSTOM_DEVICE
+      if (SupportCustomDevice()) {
+        auto& dev_ctx = ctx.device_context();
+        expected_kernel_key.place_ = dev_ctx.GetPlace();
+      }
+#endif
       if (platform::is_cpu_place(expected_kernel_key.place_)) {
         LOG_FIRST_N(WARNING, 1)
             << "Op(" << type_
diff --git a/paddle/fluid/framework/operator.h b/paddle/fluid/framework/operator.h
index 23aac65dd1fdb..3de6560653cb6 100644
--- a/paddle/fluid/framework/operator.h
+++ b/paddle/fluid/framework/operator.h
@@ -288,6 +288,7 @@ class OperatorBase {
 
   virtual bool SupportGPU() const { return false; }
   virtual bool SupportXPU() const { return false; }
+  virtual bool SupportCustomDevice() const { return false; }
 
   const std::string& Type() const { return type_; }
 
@@ -748,6 +749,8 @@ class OperatorWithKernel : public OperatorBase {
 
   bool SupportXPU() const override;
 
+  bool SupportCustomDevice() const override;
+
   bool SupportsMKLDNN(phi::DataType data_type) const;
 
   bool SupportsCUDNN(phi::DataType data_type) const;
diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py
index 2ba5b7c0d5ecc..ad7140c14b657 100644
--- a/python/paddle/fluid/framework.py
+++ b/python/paddle/fluid/framework.py
@@ -7431,9 +7431,12 @@ def device_guard(device=None):
         device, index = device.split(':')
         if device == 'cpu':
             raise ValueError("Should not set device id for cpu.")
-    if device not in ['cpu', 'gpu', 'xpu', '', None]:
+    if (
+        device not in ['cpu', 'gpu', 'xpu', '', None]
+        and device not in core.get_all_custom_device_type()
+    ):
         raise ValueError(
-            "The Attr(device) should be 'cpu' or 'gpu', and it can also be empty string or None "
+            "The Attr(device) should be 'cpu', 'xpu', 'gpu' or custom device, and it can also be empty string or None "
             "when there is no need to specify device. But received %s" % device
         )
     if index:

From 4f1bf199b6dd01a9209c72c1d0d77f03ef1ff431 Mon Sep 17 00:00:00 2001
From: jiangcheng <thisjiang@qq.com>
Date: Wed, 17 May 2023 21:17:16 +0800
Subject: [PATCH 380/405] [CINN] extend cinn single test timeout from 150 to
 200, test=document_fix (#53908)

---
 python/paddle/fluid/tests/unittests/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt
index 0cf206084a21c..a9e7504d671d0 100755
--- a/python/paddle/fluid/tests/unittests/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt
@@ -1120,7 +1120,7 @@ set(TEST_CINN_OPS
 foreach(TEST_CINN_OPS ${TEST_CINN_OPS})
   if(WITH_CINN)
     set_tests_properties(${TEST_CINN_OPS} PROPERTIES LABELS "RUN_TYPE=CINN")
-    set_tests_properties(${TEST_CINN_OPS} PROPERTIES TIMEOUT 150)
+    set_tests_properties(${TEST_CINN_OPS} PROPERTIES TIMEOUT 200)
   endif()
 endforeach()
 

From 972581d89ead62399b3ebfa82856097c34b20e0b Mon Sep 17 00:00:00 2001
From: Shaojie WANG <wsjmessi@163.com>
Date: Wed, 17 May 2023 19:19:34 -0700
Subject: [PATCH 381/405] [AMP]Master grad in static graph (#53362)

* add master gradients on static graph

* add unit test for bf16 master grad static graph

* use float16 as v100 test dtype

* only skip GPU which do not support bf16

* use linear layer to test master grad

* 1.push master grad creation before all optimizer ops; 2.remove useless unittest; 3.use a function to create master grad states
---
 python/paddle/optimizer/adamw.py        |   2 +
 python/paddle/optimizer/optimizer.py    |  28 +++-
 python/paddle/static/amp/decorator.py   |  59 +++++++
 python/paddle/static/amp/fp16_utils.py  |   4 +-
 test/amp/amp_base_models.py             | 127 ++++++++++++++-
 test/amp/test_amp_api.py                |   1 +
 test/amp/test_amp_master_grad_static.py | 203 ++++++++++++++++++++++++
 test/amp/test_model_cast_to_bf16.py     |  47 ++----
 8 files changed, 431 insertions(+), 40 deletions(-)
 create mode 100644 test/amp/test_amp_master_grad_static.py

diff --git a/python/paddle/optimizer/adamw.py b/python/paddle/optimizer/adamw.py
index c7e550f7aa117..3af9457142cf6 100644
--- a/python/paddle/optimizer/adamw.py
+++ b/python/paddle/optimizer/adamw.py
@@ -283,6 +283,8 @@ def __init__(
         self._auxiliary_vars = {}
         self._already_create_accumulater = set()
 
+        self._create_master_grad_states()
+
     def _set_auxiliary_var(self, key, val):
         self._auxiliary_vars[key] = val
 
diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py
index cc014e7cf1a8f..df20134e923e2 100644
--- a/python/paddle/optimizer/optimizer.py
+++ b/python/paddle/optimizer/optimizer.py
@@ -275,6 +275,14 @@ def __init__(
         self._auxiliary_vars = {}
         self._already_create_accumulater = set()
 
+        # create master gradients' states
+        self._create_master_grad_states()
+
+    def _create_master_grad_states(self):
+        # master gradients states
+        self._master_grads = {}
+        self._master_grad = False
+
     def _set_auxiliary_var(self, key, val):
         self._auxiliary_vars[key] = val
 
@@ -669,6 +677,25 @@ def _create_master_weight(self, param):
             self._master_weights[param.name] = var
         return var
 
+    def _create_master_grad(self, grad):
+        assert self._is_dtype_fp16_or_bf16(grad.dtype)
+        if grad.name in self._master_grads:
+            var = self._master_grads[grad.name]
+        else:
+            var_name = grad.name + "_fp32_master"
+            var_name = unique_name.generate(var_name)
+            var = grad.block.create_var(
+                name=var_name,
+                shape=grad.shape,
+                value=0,
+                dtype='float32',
+                lod_level=grad.lod_level,
+                persistable=grad.persistable,
+                is_data=grad.is_data,
+            )
+            self._master_grads[grad.name] = var
+        return var
+
     def _create_accumulators(self, block, parameters):
         """Create all accumulators needed by the parameters
 
@@ -1168,7 +1195,6 @@ def apply_gradients(self, params_grads):
         if self._grad_clip is not None:
             params_grads = self._grad_clip(params_grads)
         else:
-
             params_grads = paddle.nn.clip.append_gradient_clip_ops(params_grads)
 
         # Add regularization if any
diff --git a/python/paddle/static/amp/decorator.py b/python/paddle/static/amp/decorator.py
index dca2a4e024c35..06169c668b8e8 100644
--- a/python/paddle/static/amp/decorator.py
+++ b/python/paddle/static/amp/decorator.py
@@ -80,6 +80,7 @@ class OptimizerWithMixedPrecision:
                            the loss scaling.
         use_amp_guard(bool): Whether to use `fp16_guard` when constructing the program.
                            Default None, which means that its value is equal to `use_pure_fp16`.
+        use_master_grad(bool): Whether to use fp32 master gradients during optimizer. Default is False.
         use_promote(bool): Whether to promotes to fp32 when op has any float32 inputs. Default is False.
     """
 
@@ -96,6 +97,7 @@ def __init__(
         incr_ratio,
         decr_ratio,
         use_amp_guard=None,
+        use_master_grad=False,
         use_promote=False,
     ):
         self._optimizer = optimizer
@@ -104,6 +106,7 @@ def __init__(
         self._train_program = None
 
         self._is_distributed = False
+        self._use_master_grad = False
         self._scaled_loss = None
         self._loss_scaling = None
         self._init_loss_scaling = init_loss_scaling
@@ -122,6 +125,9 @@ def __init__(
         self._learning_rate = optimizer._learning_rate
         self._learning_rate_map = optimizer._learning_rate_map
         self._use_pure_fp16 = level == "O2"
+        if self._use_pure_fp16 and (dtype == "bfloat16" or dtype == "float16"):
+            self._use_master_grad = use_master_grad
+            self._optimizer._master_grad = use_master_grad
         self._amp_level = level
         self._use_fp16_guard = use_amp_guard
         self._to_fp16_var_names = None
@@ -384,6 +390,51 @@ def run_example_code():
                     use_promote=self.use_promote,
                 )
 
+    def _append_cast_to_master_grad_op(self, param_grads):
+        """
+        Create master gradient vars and add cast gradient to master gradient op in main program
+
+        Args:
+          param_grads(list(tuple(Tensor, Tensor))): A list of (parameter, gradient) pair to update.
+
+        Returns:
+          list: A list of (parameter, master_gradient) pair. In the following grad clip step and optimizer step, params can be updated by master gradient. main_prog will also append cast ops before grad clip ops.
+
+        """
+
+        if not self._use_master_grad:
+            return param_grads
+
+        global_block = self._train_program.global_block()
+        target_block = global_block
+        current_block = self._train_program.current_block()
+        if current_block.idx != global_block.idx:
+            target_block = self._train_program.blocks[
+                current_block.backward_block_idx
+            ]
+        params_master_grads = []
+
+        assert isinstance(target_block, paddle.fluid.framework.Block)
+        # create
+        for p, g in param_grads:
+            if g.name not in self._optimizer._master_grads.keys():
+                if self._optimizer._is_dtype_fp16_or_bf16(g.dtype):
+                    master_g = self._optimizer._create_master_grad(g)
+                    params_master_grads.append((p, master_g))
+                    target_block.append_op(
+                        type="cast",
+                        inputs={"X": [g]},
+                        outputs={"Out": [master_g]},
+                        attrs={
+                            "in_dtype": g.dtype,
+                            "out_dtype": master_g.dtype,
+                        },
+                    )
+                else:
+                    params_master_grads.append((p, g))
+
+        return params_master_grads
+
     def apply_gradients(self, params_grads):
         """
         Check scaled gradients to determine whether to update loss scaling and update
@@ -400,6 +451,9 @@ def apply_gradients(self, params_grads):
         # transferred across GPUs can be FP16.
         update_role_var_grad(self._train_program, params_grads)
 
+        # Create master grad and add cast op into program
+        params_grads = self._append_cast_to_master_grad_op(params_grads)
+
         # When not using dynamic loss scaling and the init loss scaling value is equal to 1.0,
         # the model can be optimized.
         if (
@@ -756,6 +810,7 @@ def decorate(
     level='O1',
     dtype='float16',
     master_weight=None,
+    master_grad=False,
     init_loss_scaling=2**15,
     incr_every_n_steps=1000,
     decr_every_n_nan_or_inf=2,
@@ -782,6 +837,9 @@ def decorate(
         master_weight(bool, optinal): For level='O2', whether to use multi-precision
             during weight updating. If master_weight is None, in O2 level optimizer
             will use multi-precision. Default is None.
+        master_grad(bool, optinal): For level='O2', whether to use master_grad
+            during weight updating. If master_grad is False, in O2 level optimizer
+            will not use master grad. Default is False.
         init_loss_scaling(float, optional): The initial loss scaling factor.
             Default is 32768.
         incr_every_n_steps(int, optional): Increases loss scaling every n
@@ -883,6 +941,7 @@ def forward(self, x):
         decr_ratio=decr_ratio,
         use_amp_guard=use_amp_guard,
         use_promote=use_promote,
+        use_master_grad=master_grad,
     )
 
     return mp_optimizer
diff --git a/python/paddle/static/amp/fp16_utils.py b/python/paddle/static/amp/fp16_utils.py
index 342a2618258df..1e85770251dc3 100644
--- a/python/paddle/static/amp/fp16_utils.py
+++ b/python/paddle/static/amp/fp16_utils.py
@@ -478,9 +478,9 @@ def op_need_keep_fp32(op, amp_lists, use_fp16_guard, params_list):
         need_keep_fp32 = True
         for in_name in op.input_names:
             for params in params_list:
-                if op.input(in_name)[0] == params.name:
+                if params.name in op.input(in_name):
                     fp16_varname_list_in_fp32_op = (
-                        fp16_varname_list_in_fp32_op.union(op.input(in_name))
+                        fp16_varname_list_in_fp32_op.union([params.name])
                     )
 
     return need_keep_fp32, fp16_varname_list_in_fp32_op
diff --git a/test/amp/amp_base_models.py b/test/amp/amp_base_models.py
index aabf9e82f7b84..29e8234e9137a 100644
--- a/test/amp/amp_base_models.py
+++ b/test/amp/amp_base_models.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import copy
+import struct
 import unittest
 
 import numpy as np
@@ -22,6 +23,34 @@
 from paddle.fluid import core
 from paddle.fluid.framework import _non_static_mode
 
+
+def copy_bits_from_float_to_uint16(f):
+    return struct.unpack('<I', struct.pack('<f', f))[0] >> 16
+
+
+def convert_float_to_uint16(in_list):
+    if in_list.dtype == np.float32:
+        new_output = []
+        for x in np.nditer(in_list):
+            new_output.append(np.uint16(copy_bits_from_float_to_uint16(x)))
+        new_output = np.reshape(new_output, in_list.shape).view(np.uint16)
+        return new_output
+    else:
+        return in_list
+
+
+def convert_uint16_to_float(in_list):
+    if in_list.dtype == np.uint16:
+        in_list = np.asarray(in_list)
+        out = np.vectorize(
+            lambda x: struct.unpack('<f', struct.pack('<I', x << 16))[0],
+            otypes=[np.float32],
+        )(in_list.flat)
+        return np.reshape(out, in_list.shape)
+    else:
+        return in_list
+
+
 _fixed_add_param = np.random.random(size=[16, 16]).astype("float32")
 
 
@@ -32,6 +61,7 @@ def _build_optimizer(
     amp_lists=None,
     use_grad_clip=False,
     use_promote=False,
+    use_master_grad=False,
     model=None,
 ):
     if use_grad_clip:
@@ -58,6 +88,7 @@ def _build_optimizer(
             amp_lists,
             level=amp_level,
             dtype=amp_dtype,
+            master_grad=use_master_grad,
             use_promote=use_promote,
         )
     return optimizer
@@ -80,6 +111,15 @@ def forward(self, x):
         return x + self.weight
 
 
+def cast_add_param(amp_dtype):
+    global _fixed_add_param
+    if amp_dtype == "bfloat16":
+        _fixed_add_param_bf16 = convert_float_to_uint16(_fixed_add_param)
+        _fixed_add_param = convert_uint16_to_float(_fixed_add_param_bf16)
+    else:
+        pass
+
+
 def build_add_model(
     use_amp, amp_dtype="float16", amp_level="O1", use_promote=False
 ):
@@ -93,6 +133,7 @@ def build_add_model(
                     x_dtype = "uint16"
                 elif amp_dtype == "float16":
                     x_dtype = "float16"
+            cast_add_param(amp_dtype)
             model = SimpleAddNet(x_dtype)
             x = paddle.static.data(name='input', shape=[16, 16], dtype=x_dtype)
             out = model(x)
@@ -177,8 +218,6 @@ def __init__(self):
         super().__init__()
         self.vocab_size = 128
         self.hidden_size = 16
-        self.vocab_size = 128
-        self.hidden_size = 16
         self.embedding = nn.Embedding(self.vocab_size, self.hidden_size)
         self.linear = nn.Linear(in_features=16, out_features=10)
 
@@ -192,7 +231,11 @@ def forward(self, x):
 
 
 def build_embedding_model(
-    use_amp, amp_dtype="float16", amp_level="O1", use_promote=False
+    use_amp,
+    amp_dtype="float16",
+    amp_level="O1",
+    use_promote=False,
+    use_master_grad=False,
 ):
     main_program = paddle.static.Program()
     startup_program = paddle.static.Program()
@@ -202,16 +245,90 @@ def build_embedding_model(
             x = paddle.static.data(name='x', shape=[None, 32], dtype='int64')
             out = model(x)
             loss = paddle.mean(out)
+            if use_amp:
+                amp_lists = paddle.static.amp.AutoMixedPrecisionLists(
+                    custom_white_list=["elementwise_mul"],
+                    custom_black_list=["reduce_mean"],
+                    dtype=amp_dtype,
+                )
+            else:
+                amp_lists = None
             optimizer = _build_optimizer(
                 use_amp,
                 amp_dtype,
                 amp_level,
-                None,
+                amp_lists,
                 True,
                 use_promote=use_promote,
+                use_master_grad=use_master_grad,
             )
             optimizer.minimize(loss)
-    return main_program, startup_program
+
+    feed_vars = [x]
+    fetch_vars = [loss]
+    return main_program, startup_program, optimizer, feed_vars, fetch_vars
+
+
+class SimpleMLPNet(nn.Layer):
+    def __init__(self):
+        super().__init__()
+        self.linear0 = paddle.nn.Linear(16, 10)
+        self.linear1 = paddle.nn.Linear(10, 32)
+
+    def forward(self, x):
+        out = self.linear0(x)
+        out = nn.functional.relu(out)
+        out = self.linear1(out)
+        out = nn.functional.relu(out)
+        out = nn.functional.dropout(out, p=0.2)
+        return out
+
+
+def build_MLP_model(
+    use_amp,
+    use_grad_clip=False,
+    amp_dtype="float16",
+    amp_level="O1",
+    use_promote=False,
+    use_master_grad=False,
+):
+    main_program = paddle.static.Program()
+    startup_program = paddle.static.Program()
+    with paddle.utils.unique_name.guard():
+        with paddle.static.program_guard(main_program, startup_program):
+            model = SimpleMLPNet()
+            x_dtype = "float32"
+            if use_amp and amp_level == "O2":
+                if amp_dtype == "bfloat16":
+                    x_dtype = "uint16"
+                elif amp_dtype == "float16":
+                    x_dtype = "float16"
+            x = paddle.static.data(name='x', shape=[None, 16], dtype=x_dtype)
+            out = model(x)
+            loss = paddle.mean(out)
+
+            if use_amp:
+                amp_lists = paddle.static.amp.AutoMixedPrecisionLists(
+                    custom_black_list=["reduce_mean"],
+                    dtype=amp_dtype,
+                )
+            else:
+                amp_lists = None
+
+            optimizer = _build_optimizer(
+                use_amp,
+                amp_dtype,
+                amp_level,
+                amp_lists,
+                use_grad_clip=use_grad_clip,
+                use_promote=use_promote,
+                use_master_grad=use_master_grad,
+            )
+            optimizer.minimize(loss)
+
+    feed_vars = [x]
+    fetch_vars = [loss]
+    return main_program, startup_program, optimizer, feed_vars, fetch_vars
 
 
 class SimpleWhileNet(nn.Layer):
diff --git a/test/amp/test_amp_api.py b/test/amp/test_amp_api.py
index ecef047ad6ce7..1bd41517f8f48 100644
--- a/test/amp/test_amp_api.py
+++ b/test/amp/test_amp_api.py
@@ -81,6 +81,7 @@ def check_results(
             exe,
             x_fp32,
             max_iters,
+            dtype,
             level,
         )
 
diff --git a/test/amp/test_amp_master_grad_static.py b/test/amp/test_amp_master_grad_static.py
new file mode 100644
index 0000000000000..096377833fb6e
--- /dev/null
+++ b/test/amp/test_amp_master_grad_static.py
@@ -0,0 +1,203 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import random
+import unittest
+
+import numpy as np
+from amp_base_models import (
+    AmpTestBase,
+    build_embedding_model,
+    build_MLP_model,
+    convert_float_to_uint16,
+    convert_uint16_to_float,
+)
+
+import paddle
+from paddle.static import amp
+
+paddle.enable_static()
+
+
+class TestStaticMasterGradProgramFP16(AmpTestBase):
+    def _check_optimizer(self, program, expected_num_mp):
+        optimizers = []
+        for block in program.blocks:
+            for op in block.ops:
+                if "Param" in op.input_names and "Grad" in op.input_names:
+                    optimizers.append(op)
+
+        actual_num_mp = 0
+        for op in optimizers:
+            if op.has_attr("multi_precision") and op.attr("multi_precision"):
+                actual_num_mp += 1
+        self.assertEqual(
+            actual_num_mp,
+            expected_num_mp,
+            f"The number of optimizers with multi_precison = True is expected to be {expected_num_mp}, but recieved {actual_num_mp}.",
+        )
+
+    def amp_fp16_o2(self, use_master_grad):
+        main_program, _, _, _, _ = build_embedding_model(
+            True, "float16", "O2", use_master_grad=use_master_grad
+        )
+        self.assertEqual(main_program.num_blocks, 1)
+
+        amp.debugging.collect_operator_stats(main_program)
+        op_stats_list = amp.debugging._get_op_stats_list(main_program)
+        expected_fp32_calls = {"lookup_table_v2": 1}
+        if use_master_grad:
+            expected_fp16_calls = {
+                "matmul_v2": 1,
+                "elementwise_add": 1,
+                "dropout": 1,
+                "lookup_table_v2": 0,
+                "squared_l2_norm": 0,
+                "adamw": 3,
+            }
+        else:
+            expected_fp16_calls = {
+                "matmul_v2": 1,
+                "elementwise_add": 1,
+                "dropout": 1,
+                "lookup_table_v2": 0,
+                "squared_l2_norm": 3,
+                "adamw": 3,
+            }
+        self._check_optimizer(
+            main_program,
+            expected_fp16_calls["matmul_v2"]
+            + expected_fp16_calls["elementwise_add"]
+            + expected_fp32_calls["lookup_table_v2"],
+        )
+        self._check_op_calls(
+            op_stats_list[0], expected_fp16_calls=expected_fp16_calls
+        )
+
+    def test_amp_fp16_o2(self):
+        use_master_grad_list = [False, True]
+        for master_grad in use_master_grad_list:
+            self.amp_fp16_o2(master_grad)
+
+
+class TestMasterGradAccuracy(AmpTestBase):
+    def _generate_feed_x(self, dtype="float16"):
+        seed = 0
+        paddle.seed(seed)
+        np.random.seed(seed)
+        random.seed(seed)
+
+        x = np.random.random(size=[64, 16]).astype("float32")
+        if dtype == "bfloat16":
+            x_f16 = convert_float_to_uint16(x)
+            x_f32 = convert_uint16_to_float(x_f16)
+        elif dtype == "float16":
+            x_f16 = x.astype(np.float16)
+            x_f32 = x_f16.astype(np.float32)
+        else:
+            raise AssertionError(f"unkown dtype:{dtype}")
+        return x_f32, x_f16
+
+    def test_compare_o1_and_o2_master_grad(self):
+        def _run(
+            place,
+            exe,
+            x_np,
+            max_iters,
+            level,
+            use_grad_clip,
+            dtype="float16",
+            use_master_grad=False,
+        ):
+            (
+                main_program,
+                startup_program,
+                optimizer,
+                feed_vars,
+                fetch_vars,
+            ) = build_MLP_model(
+                True,
+                use_grad_clip=use_grad_clip,
+                amp_dtype=dtype,
+                amp_level=level,
+                use_master_grad=use_master_grad,
+            )
+
+            seed = 0
+            paddle.seed(seed)
+            np.random.seed(seed)
+            random.seed(seed)
+
+            losses = self.run_program(
+                main_program,
+                startup_program,
+                optimizer,
+                feed_vars,
+                fetch_vars,
+                place,
+                exe,
+                x_np,
+                max_iters,
+                dtype,
+                level,
+            )
+            return losses
+
+        dtype = "float16"
+        max_iters = 25
+        x_f32, x_f16 = self._generate_feed_x(dtype)
+        place = paddle.CUDAPlace(0)
+        exe = paddle.static.Executor(place)
+        use_grad_clip_list = [False, True]
+        for use_grad_clip in use_grad_clip_list:
+            losses_o1 = _run(
+                place, exe, x_f32, max_iters, 'O1', use_grad_clip, dtype=dtype
+            )
+            losses_o2_no_master_grad = _run(
+                place,
+                exe,
+                x_f16,
+                max_iters,
+                'O2',
+                use_grad_clip,
+                dtype=dtype,
+                use_master_grad=False,
+            )
+            losses_o2_master_grad = _run(
+                place,
+                exe,
+                x_f16,
+                max_iters,
+                'O2',
+                use_grad_clip,
+                dtype=dtype,
+                use_master_grad=True,
+            )
+
+            self.assertNotEqual(
+                losses_o1,
+                losses_o2_no_master_grad,
+                f"dtype: {dtype}, loss of o1 and o2-wo-master_grad should not be equal, but recieved loss o1: {losses_o1}, loss o2: {losses_o2_no_master_grad}",
+            )
+
+            self.assertEqual(
+                losses_o1,
+                losses_o2_master_grad,
+                f"dtype: {dtype}, loss of o1 and o2-w-master_grad should be equal, but recieved loss o1: {losses_o1}, loss o2: {losses_o2_master_grad}",
+            )
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/amp/test_model_cast_to_bf16.py b/test/amp/test_model_cast_to_bf16.py
index 373a5ed06ace3..69d955b0f5872 100644
--- a/test/amp/test_model_cast_to_bf16.py
+++ b/test/amp/test_model_cast_to_bf16.py
@@ -13,11 +13,16 @@
 # limitations under the License.
 
 import contextlib
-import struct
 import unittest
 
 import numpy as np
-from amp_base_models import AmpTestBase, build_add_model, build_embedding_model
+from amp_base_models import (
+    AmpTestBase,
+    build_add_model,
+    build_embedding_model,
+    convert_float_to_uint16,
+    convert_uint16_to_float,
+)
 
 import paddle
 from paddle import fluid
@@ -26,34 +31,6 @@
 
 paddle.enable_static()
 
-
-def copy_bits_from_float_to_uint16(f):
-    return struct.unpack('<I', struct.pack('<f', f))[0] >> 16
-
-
-def convert_float_to_uint16(in_list):
-    if in_list.dtype == np.float32:
-        new_output = []
-        for x in np.nditer(in_list):
-            new_output.append(np.uint16(copy_bits_from_float_to_uint16(x)))
-        new_output = np.reshape(new_output, in_list.shape).view(np.uint16)
-        return new_output
-    else:
-        return in_list
-
-
-def convert_uint16_to_float(in_list):
-    if in_list.dtype == np.uint16:
-        in_list = np.asarray(in_list)
-        out = np.vectorize(
-            lambda x: struct.unpack('<f', struct.pack('<I', x << 16))[0],
-            otypes=[np.float32],
-        )(in_list.flat)
-        return np.reshape(out, in_list.shape)
-    else:
-        return in_list
-
-
 cutf = convert_uint16_to_float
 
 
@@ -239,7 +216,7 @@ def _check_optimizer(self, program, expected_num_mp):
         )
 
     def test_amp_bf16_o1(self):
-        main_program, startup_program = build_embedding_model(
+        main_program, startup_program, _, _, _ = build_embedding_model(
             True, "bfloat16", "O1"
         )
         self.assertEqual(main_program.num_blocks, 1)
@@ -258,7 +235,7 @@ def test_amp_bf16_o1(self):
         self._check_op_calls(op_stats_list[0], expected_bf16_calls)
 
     def test_amp_bf16_o2(self):
-        main_program, startup_program = build_embedding_model(
+        main_program, startup_program, _, _, _ = build_embedding_model(
             True, "bfloat16", "O2"
         )
         self.assertEqual(main_program.num_blocks, 1)
@@ -322,6 +299,12 @@ def _run(place, exe, x_np, max_iters, level):
         losses_o1 = _run(place, exe, x_fp32, max_iters, 'O1')
         losses_o2 = _run(place, exe, x_bf16, max_iters, 'O2')
 
+        self.assertEqual(
+            losses_o1,
+            losses_o2,
+            f"loss of o1 and o2 should be equal, but recieved loss o1: {losses_o1}, loss o2: {losses_o2}",
+        )
+
 
 if __name__ == '__main__':
     unittest.main()

From 65ce6886fb56673b1316c87e1ddca9bcca0820f6 Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Thu, 18 May 2023 10:29:51 +0800
Subject: [PATCH 382/405] Fix typos in send_v2_op.cu.cc (#53904)

---
 .../operators/collective/recv_v2_op.cu.cc     | 20 +++++++++----------
 .../operators/collective/send_v2_op.cu.cc     | 20 +++++++++----------
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/paddle/fluid/operators/collective/recv_v2_op.cu.cc b/paddle/fluid/operators/collective/recv_v2_op.cu.cc
index a10257b45b300..bfa12f911946d 100644
--- a/paddle/fluid/operators/collective/recv_v2_op.cu.cc
+++ b/paddle/fluid/operators/collective/recv_v2_op.cu.cc
@@ -40,24 +40,24 @@ framework::DDim recv_shape_info(const platform::Place &place,
                           "to send the shape info."));
   }
 
-  phi::DataType shape_dytpe = phi::DataType::INT32;
+  phi::DataType shape_dtype = phi::DataType::INT32;
   ncclDataType_t nccl_dtype =
-      platform::ToNCCLDataType(framework::TransToProtoVarType(shape_dytpe));
+      platform::ToNCCLDataType(framework::TransToProtoVarType(shape_dtype));
 
   // step1: recv the shape size
-  phi::DenseTensor gpu_shape_size_tensor(shape_dytpe);
+  phi::DenseTensor gpu_shape_size_tensor(shape_dtype);
   if (!group) {
     gpu_shape_size_tensor.Resize({1});
-    gpu_shape_size_tensor.mutable_data(place, shape_dytpe);
+    gpu_shape_size_tensor.mutable_data(place, shape_dtype);
     auto *gpu_data = gpu_shape_size_tensor.data<int>();
     PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::ncclRecv(
         gpu_data, 1, nccl_dtype, peer, comm->comm(), stream));
   }
 
   // copy the shape size tensor to cpu
-  phi::DenseTensor *cpu_shape_size_tensor = new phi::DenseTensor(shape_dytpe);
+  phi::DenseTensor *cpu_shape_size_tensor = new phi::DenseTensor(shape_dtype);
   cpu_shape_size_tensor->Resize({1});
-  cpu_shape_size_tensor->mutable_data(platform::CPUPlace(), shape_dytpe);
+  cpu_shape_size_tensor->mutable_data(platform::CPUPlace(), shape_dtype);
   if (group) {
     std::vector<phi::DenseTensor> shape_size_tensor;
     shape_size_tensor.emplace_back(*cpu_shape_size_tensor);
@@ -71,19 +71,19 @@ framework::DDim recv_shape_info(const platform::Place &place,
   VLOG(3) << "recv the shape size: " << shape_size << " from peer";
 
   // step2: recv the shape
-  phi::DenseTensor gpu_shape_tensor(shape_dytpe);
+  phi::DenseTensor gpu_shape_tensor(shape_dtype);
   if (!group) {
     gpu_shape_tensor.Resize({shape_size});
-    gpu_shape_tensor.mutable_data(place, shape_dytpe);
+    gpu_shape_tensor.mutable_data(place, shape_dtype);
     auto *gpu_shape_data = gpu_shape_tensor.data<int>();
     PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::ncclRecv(
         gpu_shape_data, shape_size, nccl_dtype, peer, comm->comm(), stream));
   }
 
   // copy the shape tensor to cpu
-  phi::DenseTensor *cpu_shape_tensor = new phi::DenseTensor(shape_dytpe);
+  phi::DenseTensor *cpu_shape_tensor = new phi::DenseTensor(shape_dtype);
   cpu_shape_tensor->Resize({shape_size});
-  cpu_shape_tensor->mutable_data(platform::CPUPlace(), shape_dytpe);
+  cpu_shape_tensor->mutable_data(platform::CPUPlace(), shape_dtype);
   if (group) {
     std::vector<phi::DenseTensor> shape_tensor;
     shape_tensor.emplace_back(*cpu_shape_tensor);
diff --git a/paddle/fluid/operators/collective/send_v2_op.cu.cc b/paddle/fluid/operators/collective/send_v2_op.cu.cc
index 7c1ab8ace3404..adea7db0b8088 100644
--- a/paddle/fluid/operators/collective/send_v2_op.cu.cc
+++ b/paddle/fluid/operators/collective/send_v2_op.cu.cc
@@ -39,16 +39,16 @@ void send_shape_info(const phi::DenseTensor& x,
                           "NCCLComm and Stream should be provided if use NCCL "
                           "to send the shape info."));
   }
-  phi::DataType shape_dytpe = phi::DataType::INT32;
+  phi::DataType shape_dtype = phi::DataType::INT32;
   ncclDataType_t nccl_dtype =
-      platform::ToNCCLDataType(framework::TransToProtoVarType(shape_dytpe));
+      platform::ToNCCLDataType(framework::TransToProtoVarType(shape_dtype));
   auto dims = x.dims();
   int shape_size = dims.size();
 
   // step1: send the shape size
-  phi::DenseTensor cpu_shape_size_tensor(shape_dytpe);
+  phi::DenseTensor cpu_shape_size_tensor(shape_dtype);
   cpu_shape_size_tensor.Resize({1});
-  cpu_shape_size_tensor.mutable_data(platform::CPUPlace(), shape_dytpe);
+  cpu_shape_size_tensor.mutable_data(platform::CPUPlace(), shape_dtype);
   auto* cpu_data = cpu_shape_size_tensor.data<int>();
   cpu_data[0] = shape_size;
 
@@ -58,9 +58,9 @@ void send_shape_info(const phi::DenseTensor& x,
     auto shape_size_task = group->Send(shape_size_tensor, peer);
   } else {
     // copy the shape size tensor to gpu and send
-    phi::DenseTensor* gpu_shape_size_tensor = new phi::DenseTensor(shape_dytpe);
+    phi::DenseTensor* gpu_shape_size_tensor = new phi::DenseTensor(shape_dtype);
     gpu_shape_size_tensor->Resize({1});
-    gpu_shape_size_tensor->mutable_data(place, shape_dytpe);
+    gpu_shape_size_tensor->mutable_data(place, shape_dtype);
     framework::TensorCopySync(
         cpu_shape_size_tensor, place, gpu_shape_size_tensor);
     PADDLE_ENFORCE_GPU_SUCCESS(
@@ -74,9 +74,9 @@ void send_shape_info(const phi::DenseTensor& x,
   VLOG(3) << "send the shape size: " << shape_size << " to peer";
 
   // step2: send the shape
-  phi::DenseTensor cpu_shape_tensor(shape_dytpe);
+  phi::DenseTensor cpu_shape_tensor(shape_dtype);
   cpu_shape_tensor.Resize({shape_size});
-  cpu_shape_tensor.mutable_data(platform::CPUPlace(), shape_dytpe);
+  cpu_shape_tensor.mutable_data(platform::CPUPlace(), shape_dtype);
   auto* cpu_shape_data = cpu_shape_tensor.data<int>();
   for (int i = 0; i < shape_size; ++i) {
     cpu_shape_data[i] = dims[i];
@@ -88,9 +88,9 @@ void send_shape_info(const phi::DenseTensor& x,
     auto shape_task = group->Send(shape_tensor, peer);
   } else {
     // copy the shape tensor to gpu and send
-    phi::DenseTensor* gpu_shape_tensor = new phi::DenseTensor(shape_dytpe);
+    phi::DenseTensor* gpu_shape_tensor = new phi::DenseTensor(shape_dtype);
     gpu_shape_tensor->Resize({shape_size});
-    gpu_shape_tensor->mutable_data(place, shape_dytpe);
+    gpu_shape_tensor->mutable_data(place, shape_dtype);
     framework::TensorCopySync(cpu_shape_tensor, place, gpu_shape_tensor);
     PADDLE_ENFORCE_GPU_SUCCESS(
         platform::dynload::ncclSend(gpu_shape_tensor->data<int>(),

From 92121d17dbd14233469157f5014d41a94ccaf462 Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Thu, 18 May 2023 10:37:36 +0800
Subject: [PATCH 383/405] Fix typos, test=document_fix (#53916)

---
 python/paddle/amp/debugging.py                               | 4 ++--
 python/paddle/device/__init__.py                             | 4 ++--
 python/paddle/distributed/fleet/base/distributed_strategy.py | 2 +-
 python/paddle/distribution/beta.py                           | 2 +-
 python/paddle/distribution/kl.py                             | 2 +-
 python/paddle/distribution/transform.py                      | 2 +-
 python/paddle/fluid/dygraph/learning_rate_scheduler.py       | 2 +-
 python/paddle/fluid/multiprocess_utils.py                    | 2 +-
 python/paddle/fluid/tests/unittests/cc_imp_py_test.cc        | 2 +-
 python/paddle/io/multiprocess_utils.py                       | 2 +-
 python/paddle/optimizer/lr.py                                | 2 +-
 11 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/python/paddle/amp/debugging.py b/python/paddle/amp/debugging.py
index 3672ecd6b353d..3c99e003490fa 100644
--- a/python/paddle/amp/debugging.py
+++ b/python/paddle/amp/debugging.py
@@ -315,7 +315,7 @@ def enable_operator_stats_collection():
     """
     Enable to collect the number of operators for different data types.
     The statistical data are categorized according to four data types, namely
-    float32, float16, bfloat16 and others. This funciton is used in pair with
+    float32, float16, bfloat16 and others. This function is used in pair with
     the corresponding disable function.
 
     Examples:
@@ -351,7 +351,7 @@ def enable_operator_stats_collection():
 def disable_operator_stats_collection():
     """
     Disable the collection the number of operators for different data types.
-    This funciton is used in pair with the corresponding enable function.
+    This function is used in pair with the corresponding enable function.
     The statistical data are categorized according to four data types, namely
     float32, float16, bfloat16 and others, and will be printed after the
     function call.
diff --git a/python/paddle/device/__init__.py b/python/paddle/device/__init__.py
index 5fce6b1442f95..c19e6e08b4d2d 100644
--- a/python/paddle/device/__init__.py
+++ b/python/paddle/device/__init__.py
@@ -135,7 +135,7 @@ def XPUPlace(dev_id):
 
 def get_cudnn_version():
     """
-    This funciton return the version of cudnn. the retuen value is int which represents the
+    This function return the version of cudnn. the retuen value is int which represents the
     cudnn version. For example, if it return 7600, it represents the version of cudnn is 7.6.
 
     Returns:
@@ -270,7 +270,7 @@ def set_device(device):
 
 def get_device():
     """
-    This funciton can get the current global device of the program is running.
+    This function can get the current global device of the program is running.
     It's a string which is like 'cpu', 'gpu:x', 'xpu:x' and 'npu:x'. if the global device is not
     set, it will return a string which is 'gpu:x' when cuda is avaliable or it
     will return a string which is 'cpu' when cuda is not avaliable.
diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py
index b751989102483..87d9f77018b70 100755
--- a/python/paddle/distributed/fleet/base/distributed_strategy.py
+++ b/python/paddle/distributed/fleet/base/distributed_strategy.py
@@ -2388,7 +2388,7 @@ def conv_workspace_size_limit(self):
         """
 
         The workspace limit size in MB unit for choosing cuDNN convolution algorithms.
-        The inner funciton of cuDNN obtain the fastest suited algorithm that fits within this memory limit.
+        The inner function of cuDNN obtain the fastest suited algorithm that fits within this memory limit.
         Usually, large workspace size may lead to choose faster algorithms,
         but significant increasing memory workspace. Users need to trade-off between memory and speed.
         Default Value: 4000
diff --git a/python/paddle/distribution/beta.py b/python/paddle/distribution/beta.py
index ebf373bf6b115..64da77b978f1e 100644
--- a/python/paddle/distribution/beta.py
+++ b/python/paddle/distribution/beta.py
@@ -120,7 +120,7 @@ def prob(self, value):
         return paddle.exp(self.log_prob(value))
 
     def log_prob(self, value):
-        """Log probability density funciton evaluated at value
+        """Log probability density function evaluated at value
 
         Args:
             value (Tensor): Value to be evaluated
diff --git a/python/paddle/distribution/kl.py b/python/paddle/distribution/kl.py
index a47eefc7893e1..40be41e06c5f6 100644
--- a/python/paddle/distribution/kl.py
+++ b/python/paddle/distribution/kl.py
@@ -73,7 +73,7 @@ def register_kl(cls_p, cls_q):
     functions registered by ``register_kl``, according to multi-dispatch pattern.
     If an implemention function is found, it will return the result, otherwise,
     it will raise ``NotImplementError`` exception. Users can register
-    implemention funciton by the decorator.
+    implemention function by the decorator.
 
     Args:
         cls_p (Distribution): The Distribution type of Instance p. Subclass derived from ``Distribution``.
diff --git a/python/paddle/distribution/transform.py b/python/paddle/distribution/transform.py
index f1ee702c15b66..53316911559e8 100644
--- a/python/paddle/distribution/transform.py
+++ b/python/paddle/distribution/transform.py
@@ -66,7 +66,7 @@ class Transform:
 
     Suppose :math:`X` is a K-dimensional random variable with probability
     density function :math:`p_X(x)`. A new random variable :math:`Y = f(X)` may
-    be defined by transforming :math:`X` with a suitably well-behaved funciton
+    be defined by transforming :math:`X` with a suitably well-behaved function
     :math:`f`. It suffices for what follows to note that if `f` is one-to-one and
     its inverse :math:`f^{-1}` have a well-defined Jacobian, then the density of
     :math:`Y` is
diff --git a/python/paddle/fluid/dygraph/learning_rate_scheduler.py b/python/paddle/fluid/dygraph/learning_rate_scheduler.py
index 9951f5a7c40f4..2da10b2e52dce 100644
--- a/python/paddle/fluid/dygraph/learning_rate_scheduler.py
+++ b/python/paddle/fluid/dygraph/learning_rate_scheduler.py
@@ -1234,7 +1234,7 @@ class LambdaDecay(_LearningRateEpochDecay):
     :api_attr: imperative
 
     Sets the learning rate of ``optimizer`` to the initial lr times a multiplicative factor, and this multiplicative
-    factor is computed by function ``lr_lambda`` . ``lr_lambda`` is funciton which receives ``epoch`` .
+    factor is computed by function ``lr_lambda`` . ``lr_lambda`` is function which receives ``epoch`` .
 
     The algorithm can be described as the code below.
 
diff --git a/python/paddle/fluid/multiprocess_utils.py b/python/paddle/fluid/multiprocess_utils.py
index bc7d58d1284f1..b763446930fdb 100644
--- a/python/paddle/fluid/multiprocess_utils.py
+++ b/python/paddle/fluid/multiprocess_utils.py
@@ -44,7 +44,7 @@ def _clear_multiprocess_queue_set():
 def _cleanup():
     # NOTE: inter-process Queue shared memory objects clear function
     _clear_multiprocess_queue_set()
-    # NOTE: main process memory map files clear funciton
+    # NOTE: main process memory map files clear function
     core._cleanup_mmap_fds()
 
 
diff --git a/python/paddle/fluid/tests/unittests/cc_imp_py_test.cc b/python/paddle/fluid/tests/unittests/cc_imp_py_test.cc
index a0b9ec5f9f6d4..e36c8527369d1 100644
--- a/python/paddle/fluid/tests/unittests/cc_imp_py_test.cc
+++ b/python/paddle/fluid/tests/unittests/cc_imp_py_test.cc
@@ -25,7 +25,7 @@ TEST(CC, IMPORT_PY) {
   ASSERT_FALSE(PyRun_SimpleString("import paddle"));
   ASSERT_FALSE(PyRun_SimpleString("print(paddle.to_tensor(1))"));
 
-  // 2. C/C++ Run Python funciton
+  // 2. C/C++ Run Python function
   PyRun_SimpleString("import sys");
   PyRun_SimpleString("import os");
   PyRun_SimpleString("sys.path.append(os.getcwd())");
diff --git a/python/paddle/io/multiprocess_utils.py b/python/paddle/io/multiprocess_utils.py
index 5792983ceb475..51b0c2b818214 100644
--- a/python/paddle/io/multiprocess_utils.py
+++ b/python/paddle/io/multiprocess_utils.py
@@ -45,7 +45,7 @@ def _clear_multiprocess_queue_set():
 def _cleanup():
     # NOTE: inter-process Queue shared memory objects clear function
     _clear_multiprocess_queue_set()
-    # NOTE: main process memory map files clear funciton
+    # NOTE: main process memory map files clear function
     core._cleanup_mmap_fds()
 
 
diff --git a/python/paddle/optimizer/lr.py b/python/paddle/optimizer/lr.py
index ad639a06c7a0f..d28abf5d631c9 100644
--- a/python/paddle/optimizer/lr.py
+++ b/python/paddle/optimizer/lr.py
@@ -1138,7 +1138,7 @@ def get_lr(self):
 
 class LambdaDecay(LRScheduler):
     """
-    Sets the learning rate of ``optimizer`` by function ``lr_lambda`` . ``lr_lambda`` is funciton which receives ``epoch`` .
+    Sets the learning rate of ``optimizer`` by function ``lr_lambda`` . ``lr_lambda`` is function which receives ``epoch`` .
 
     The algorithm can be described as the code below.
 

From 6d7076cc7f8e7bcdfb8ba20cfd0f9c3bb3bb5ed4 Mon Sep 17 00:00:00 2001
From: engineer1109 <1292846099@qq.com>
Date: Thu, 18 May 2023 10:47:07 +0800
Subject: [PATCH 384/405] fix -Werror=format-security (#53886)

---
 .../tensorrt/convert/generic_and_custom_plugin_creater.cc   | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc b/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc
index da64a8fbb6811..474a88dd0c86b 100644
--- a/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc
+++ b/paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc
@@ -77,8 +77,10 @@ class CustomPluginCreater : public OpConverter {
 
       // NOTE: to avoid string rewrite by iterator, deep copy here
       std::vector<char> plugin_attr_name(attr_name.length() + 1, 0);
-      snprintf(
-          plugin_attr_name.data(), attr_name.length() + 1, attr_name.c_str());
+      snprintf(plugin_attr_name.data(),
+               attr_name.length() + 1,
+               "%s",
+               attr_name.c_str());
       plugindata.name = plugin_attr_name.data();
 
       if (op_desc.GetAttrType(attr_name) == framework::proto::AttrType::INT) {

From 79ce3facd7f22ec5a01e82e5f029118a16cec184 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?=
 <83450930+Liyulingyue@users.noreply.github.com>
Date: Thu, 18 May 2023 10:53:56 +0800
Subject: [PATCH 385/405] rm cmake npu (#53869)

* rm cmake npu

* Update generic.cmake

* Update generic.cmake
---
 cmake/generic.cmake                                | 12 +-----------
 python/paddle/distributed/auto_parallel/cluster.py |  1 -
 2 files changed, 1 insertion(+), 12 deletions(-)

diff --git a/cmake/generic.cmake b/cmake/generic.cmake
index cf4708ccff942..09a5130674978 100644
--- a/cmake/generic.cmake
+++ b/cmake/generic.cmake
@@ -575,17 +575,7 @@ function(cc_test_old TARGET_NAME)
     cc_test_build(${TARGET_NAME} SRCS ${cc_test_SRCS} DEPS ${cc_test_DEPS})
     # we dont test hcom op, because it need complex configuration
     # with more than one machine
-    if(NOT
-       ("${TARGET_NAME}" STREQUAL "c_broadcast_op_npu_test"
-        OR "${TARGET_NAME}" STREQUAL "c_allreduce_sum_op_npu_test"
-        OR "${TARGET_NAME}" STREQUAL "c_allreduce_max_op_npu_test"
-        OR "${TARGET_NAME}" STREQUAL "c_reducescatter_op_npu_test"
-        OR "${TARGET_NAME}" STREQUAL "c_allgather_op_npu_test"
-        OR "${TARGET_NAME}" STREQUAL "send_v2_op_npu_test"
-        OR "${TARGET_NAME}" STREQUAL "c_reduce_sum_op_npu_test"
-        OR "${TARGET_NAME}" STREQUAL "recv_v2_op_npu_test"))
-      cc_test_run(${TARGET_NAME} COMMAND ${TARGET_NAME} ARGS ${cc_test_ARGS})
-    endif()
+    cc_test_run(${TARGET_NAME} COMMAND ${TARGET_NAME} ARGS ${cc_test_ARGS})
   elseif(WITH_TESTING AND NOT TEST ${TARGET_NAME})
     add_test(NAME ${TARGET_NAME} COMMAND ${CMAKE_COMMAND} -E echo CI skip
                                          ${TARGET_NAME}.)
diff --git a/python/paddle/distributed/auto_parallel/cluster.py b/python/paddle/distributed/auto_parallel/cluster.py
index 725037d20592c..f818fcc53b97c 100644
--- a/python/paddle/distributed/auto_parallel/cluster.py
+++ b/python/paddle/distributed/auto_parallel/cluster.py
@@ -29,7 +29,6 @@ class DeviceType(IntEnum):
     CPU = 1
     GPU = 2
     XPU = 3
-    NPU = 4
     DCU = 5
     NIC = 6
 

From d294eef906f684d4c5208fe54deed7d161d6c244 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?=
 <83450930+Liyulingyue@users.noreply.github.com>
Date: Thu, 18 May 2023 10:54:12 +0800
Subject: [PATCH 386/405] rm tools npu (#53870)

* rm tools npu

* Update get_pr_ut.py

* Update get_pr_ut.py
---
 tools/get_pr_ut.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/tools/get_pr_ut.py b/tools/get_pr_ut.py
index 4da57036b68c8..068d2fcb9ba3b 100644
--- a/tools/get_pr_ut.py
+++ b/tools/get_pr_ut.py
@@ -402,11 +402,7 @@ def get_pr_ut(self):
                     if f_judge.endswith('.md'):
                         ut_list.append('md_placeholder')
                         onlyCommentsFilesOrXpu.append(f_judge)
-                    elif (
-                        'test/xpu' in f_judge
-                        or 'tests/unittests/npu' in f_judge
-                        or 'op_npu.cc' in f_judge
-                    ):
+                    elif 'test/xpu' in f_judge:
                         ut_list.append('xpu_npu_placeholder')
                         onlyCommentsFilesOrXpu.append(f_judge)
                     elif f_judge.endswith(('.h', '.cu', '.cc', '.py')):

From 5d638feb208af9e5146c0e0b4b80f78833d12738 Mon Sep 17 00:00:00 2001
From: houj04 <35131887+houj04@users.noreply.github.com>
Date: Thu, 18 May 2023 10:58:59 +0800
Subject: [PATCH 387/405] [XPU] do not call check_nccl_version_for_p2p under
 xpu (#53862)

* [XPU] do not call check_nccl_version_for_p2p under xpu

* refine code.
---
 python/paddle/distributed/fleet/base/topology.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/python/paddle/distributed/fleet/base/topology.py b/python/paddle/distributed/fleet/base/topology.py
index f5f2dffac4d0e..f4f55d41399b6 100644
--- a/python/paddle/distributed/fleet/base/topology.py
+++ b/python/paddle/distributed/fleet/base/topology.py
@@ -189,7 +189,8 @@ def __init__(self, topology):
 
         # create p2p_groups
         if self._pp_degree > 1:
-            check_nccl_version_for_p2p()
+            if paddle.framework.core.is_compiled_with_nccl():
+                check_nccl_version_for_p2p()
             self._set_p2p_group()
 
         debug_str = (

From 236e742d30ef524b8effee5fee5da3bb2d4301bf Mon Sep 17 00:00:00 2001
From: HongyuJia <jiahongyu@baidu.com>
Date: Thu, 18 May 2023 11:22:45 +0800
Subject: [PATCH 388/405] [Fix Typo] Fix gpu_info.h, Wheter->Whether (#53564)

---
 paddle/fluid/platform/device/gpu/gpu_info.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/paddle/fluid/platform/device/gpu/gpu_info.h b/paddle/fluid/platform/device/gpu/gpu_info.h
index 925a150b60efb..de68329bba66d 100644
--- a/paddle/fluid/platform/device/gpu/gpu_info.h
+++ b/paddle/fluid/platform/device/gpu/gpu_info.h
@@ -39,7 +39,7 @@ int GetGPURuntimeVersion(int id);
 //! Get the driver version of the ith GPU
 int GetGPUDriverVersion(int id);
 
-//! Wheter the current device support TensorCore
+//! Whether the current device support TensorCore
 bool TensorCoreAvailable();
 
 //! Get the MultiProcessors of the ith GPU.

From 4f07b653fb29db78d032d4a1c525a0b3c13ed6b2 Mon Sep 17 00:00:00 2001
From: RedContritio <RedContritio@qq.com>
Date: Thu, 18 May 2023 11:24:00 +0800
Subject: [PATCH 389/405] support auto generate for op layer_norm (#53178)

* simplify layer_norm_op.cc

* support auto generate for op layer_norm

* update unittest for composite_layer_norm

* remove layer_norm_op.cc from scripts

* replace layer_norm_op with generated_op

* add get_expected_kernel for layer_norm

* update cmake kernel register function for layer_norm_mkldnn_op
---
 cmake/operators.cmake                         |  42 +++
 paddle/fluid/operators/CMakeLists.txt         |   4 +
 .../generator/get_expected_kernel_func.cc     |  16 +
 .../generator/get_expected_kernel_func.h      |   4 +
 paddle/fluid/operators/layer_norm_op.cc       | 336 ------------------
 paddle/fluid/operators/unity_build_rule.cmake |   2 +-
 paddle/phi/api/yaml/backward.yaml             |  14 +
 paddle/phi/api/yaml/legacy_backward.yaml      |  14 -
 paddle/phi/api/yaml/legacy_ops.yaml           |  11 -
 paddle/phi/api/yaml/op_compat.yaml            |   2 +
 paddle/phi/api/yaml/ops.yaml                  |  12 +
 paddle/phi/infermeta/ternary.cc               |  13 +
 paddle/phi/ops/compat/layer_norm_sig.cc       |  38 --
 python/paddle/nn/functional/norm.py           |   2 +-
 test/cpp/fluid/fused/CMakeLists.txt           |   6 +-
 .../test_composite_layer_norm.py              |  38 +-
 tools/enforce/count_enforce_by_file.sh        |   1 -
 17 files changed, 131 insertions(+), 424 deletions(-)
 delete mode 100644 paddle/fluid/operators/layer_norm_op.cc
 delete mode 100644 paddle/phi/ops/compat/layer_norm_sig.cc

diff --git a/cmake/operators.cmake b/cmake/operators.cmake
index 826d0e773a8a5..303830e928cf0 100644
--- a/cmake/operators.cmake
+++ b/cmake/operators.cmake
@@ -103,6 +103,48 @@ function(register_cu_kernel TARGET)
   endforeach()
 endfunction()
 
+# Just for those mkldnn kernels locating at "fluid/operators/mkldnn/", such as 'layer_norm_mkldnn_op.cc'.
+# Add other file modes if need in the future.
+function(register_mkldnn_kernel TARGET)
+  set(options "")
+  set(oneValueArgs "")
+  set(multiValueArgs SRCS DEPS)
+  cmake_parse_arguments(register_mkldnn_kernel "${options}" "${oneValueArgs}"
+                        "${multiValueArgs}" ${ARGN})
+
+  set(mkldnn_cc_srcs)
+  set(op_common_deps operator op_registry math_function layer
+                     common_infer_shape_functions)
+  foreach(mkldnn_src ${register_mkldnn_kernel_SRCS})
+    if(${mkldnn_src} MATCHES ".*_mkldnn_op.cc$")
+      list(APPEND mkldnn_cc_srcs mkldnn/${mkldnn_src})
+    endif()
+  endforeach()
+  list(LENGTH mkldnn_cc_srcs mkldnn_cc_srcs_len)
+  if(${mkldnn_cc_srcs_len} EQUAL 0)
+    message(
+      FATAL_ERROR
+        "The MKLDNN kernel file of ${TARGET} should contains at least one *.*_mkldnn_op.cc file"
+    )
+  endif()
+  if(WITH_MKLDNN)
+    cc_library(
+      ${TARGET}
+      SRCS ${mkldnn_cc_srcs}
+      DEPS ${op_library_DEPS} ${op_common_deps})
+  endif()
+  set(OP_LIBRARY
+      ${TARGET} ${OP_LIBRARY}
+      CACHE INTERNAL "op libs")
+  foreach(mkldnn_src ${mkldnn_cc_srcs})
+    set(op_name "")
+    find_register(${mkldnn_src} "REGISTER_OP_KERNEL" op_name)
+    if(NOT ${op_name} EQUAL "")
+      file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${op_name}, MKLDNN);\n")
+    endif()
+  endforeach()
+endfunction()
+
 function(op_library TARGET)
   # op_library is a function to create op library. The interface is same as
   # cc_library. But it handle split GPU/CPU code and link some common library
diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt
index 0cefb7e69ace3..b12ec19b9b9df 100644
--- a/paddle/fluid/operators/CMakeLists.txt
+++ b/paddle/fluid/operators/CMakeLists.txt
@@ -106,6 +106,10 @@ if (WITH_GPU OR WITH_ROCM)
     register_cu_kernel(class_center_sample_op SRCS class_center_sample_op.cu DEPS ${OP_HEADER_DEPS})
 endif()
 
+if (WITH_MKLDNN)
+    register_mkldnn_kernel(layer_norm_op SRCS layer_norm_mkldnn_op.cc DEPS ${OP_HEADER_DEPS})
+endif()
+
 if (WITH_GPU OR WITH_ROCM)
     op_library(activation_op SRCS activation_op.cc activation_op.kps soft_relu_op.cu DEPS ${OP_HEADER_DEPS})
 elseif (WITH_XPU_KP)
diff --git a/paddle/fluid/operators/generator/get_expected_kernel_func.cc b/paddle/fluid/operators/generator/get_expected_kernel_func.cc
index 026d57bba4f7d..28518859f4855 100644
--- a/paddle/fluid/operators/generator/get_expected_kernel_func.cc
+++ b/paddle/fluid/operators/generator/get_expected_kernel_func.cc
@@ -250,5 +250,21 @@ phi::KernelKey GetInstanceNormExpectedKernelType(
   return phi::KernelKey(input_data_type, ctx.GetPlace());
 }
 
+phi::KernelKey GetLayerNormExpectedKernelType(
+    const framework::ExecutionContext& ctx,
+    const framework::OperatorWithKernel* op_ptr) {
+  auto input_data_type =
+      op_ptr->OperatorWithKernel::IndicateVarDataType(ctx, "X");
+
+  // NOTE(jiahongyu): Below codes originally enclosed by PADDLE_WITH_MKLDNN
+  int begin_norm_axis = ctx.Attr<int>("begin_norm_axis");
+  if (begin_norm_axis != ctx.Input<phi::DenseTensor>("X")->dims().size() - 1) {
+    op_ptr->SetDnnFallback(true);
+  }
+  // NOTE(jiahongyu): Above codes originally enclosed by PADDLE_WITH_MKLDNN
+
+  return phi::KernelKey(input_data_type, ctx.GetPlace());
+}
+
 }  // namespace operators
 }  // namespace paddle
diff --git a/paddle/fluid/operators/generator/get_expected_kernel_func.h b/paddle/fluid/operators/generator/get_expected_kernel_func.h
index 7923c8d79fb2d..bb228dbe2e97a 100644
--- a/paddle/fluid/operators/generator/get_expected_kernel_func.h
+++ b/paddle/fluid/operators/generator/get_expected_kernel_func.h
@@ -64,5 +64,9 @@ phi::KernelKey GetYoloLossExpectedKernelType(
     const framework::ExecutionContext& ctx,
     const framework::OperatorWithKernel* op_ptr);
 
+phi::KernelKey GetLayerNormExpectedKernelType(
+    const framework::ExecutionContext& ctx,
+    const framework::OperatorWithKernel* op_ptr);
+
 }  // namespace operators
 }  // namespace paddle
diff --git a/paddle/fluid/operators/layer_norm_op.cc b/paddle/fluid/operators/layer_norm_op.cc
deleted file mode 100644
index facef32fa3b5c..0000000000000
--- a/paddle/fluid/operators/layer_norm_op.cc
+++ /dev/null
@@ -1,336 +0,0 @@
-/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include <memory>
-#include <string>
-
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/prim/api/composite_backward/composite_backward_api.h"
-#include "paddle/fluid/prim/utils/static/composite_grad_desc_maker.h"
-#include "paddle/fluid/prim/utils/static/desc_tensor.h"
-#include "paddle/phi/core/infermeta_utils.h"
-#include "paddle/phi/infermeta/ternary.h"
-
-namespace paddle {
-namespace operators {
-
-using DataLayout = phi::DataLayout;
-
-class LayerNormOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
-  void InferShape(framework::InferShapeContext *ctx) const override {
-    OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "LayerNorm");
-    OP_INOUT_CHECK(ctx->HasOutput("Y"), "Output", "Y", "LayerNorm");
-    OP_INOUT_CHECK(ctx->HasOutput("Mean"), "Output", "Mean", "LayerNorm");
-    OP_INOUT_CHECK(
-        ctx->HasOutput("Variance"), "Output", "Variance", "LayerNorm");
-
-    auto x_dim = ctx->GetInputDim("X");
-    auto begin_norm_axis = ctx->Attrs().Get<int>("begin_norm_axis");
-    PADDLE_ENFORCE_LT(
-        begin_norm_axis,
-        x_dim.size(),
-        platform::errors::InvalidArgument(
-            "'begin_norm_axis' must be less than the dimensions of X,"
-            "But received 'begin_norm_axis' is [%d],"
-            "received the dimensions of X is [%d].",
-            begin_norm_axis,
-            x_dim.size()));
-
-    auto matrix_dim = phi::flatten_to_2d(x_dim, begin_norm_axis);
-    int left = static_cast<int>(matrix_dim[0]);
-    int right = static_cast<int>(matrix_dim[1]);
-    if (ctx->HasInput("Scale")) {
-      PADDLE_ENFORCE_EQ(ctx->GetInputDim("Scale").size(),
-                        1,
-                        platform::errors::InvalidArgument(
-                            "The dimensions of Input(Scale) must be 1, but "
-                            "received dimensions of"
-                            "Input(Scale) is [%d]",
-                            ctx->GetInputDim("Scale").size()));
-
-      if (ctx->IsRuntime()) {
-        PADDLE_ENFORCE_EQ(
-            ctx->GetInputDim("Scale")[0],
-            right,
-            platform::errors::InvalidArgument(
-                "The first dimension value of Input(Scale) must equal to be the"
-                "second dimension value of the flattened 2D matrix of Input(X),"
-                "But received the first dimension value of Input(Scale) is"
-                "[%d], the second dimension value of the flattened 2D matrix of"
-                " Input(Scale) is [%d].",
-                ctx->GetInputDim("Scale")[0],
-                right));
-      }
-    }
-    if (ctx->HasInput("Bias")) {
-      PADDLE_ENFORCE_EQ(ctx->GetInputDim("Bias").size(),
-                        1,
-                        platform::errors::InvalidArgument(
-                            "The dimensions of Input(Bias) must be 1, but "
-                            "received dimensions of"
-                            "Input(Bias) is [%d]",
-                            ctx->GetInputDim("Bias").size()));
-      if (ctx->IsRuntime()) {
-        PADDLE_ENFORCE_EQ(
-            ctx->GetInputDim("Bias")[0],
-            right,
-            platform::errors::InvalidArgument(
-                "The first dimension value of Input(Bias) must equal to be the"
-                "second dimension value of the flattened 2D matrix of Input(X),"
-                "But received the first dimension value of Input(Bias) is"
-                "[%d], the second dimension value of the flattened 2D matrix of"
-                " Input(Bias) is [%d].",
-                ctx->GetInputDim("Scale")[0],
-                right));
-      }
-    }
-
-    ctx->SetOutputDim("Y", ctx->GetInputDim("X"));
-    ctx->SetOutputDim("Mean", {left});
-    ctx->SetOutputDim("Variance", {left});
-    ctx->ShareLoD("X", "Y");
-  }
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext &ctx) const override {
-    auto input_data_type = OperatorWithKernel::IndicateVarDataType(ctx, "X");
-
-    // NOTE(jiahongyu): Below codes originally enclosed by PADDLE_WITH_MKLDNN
-    int begin_norm_axis = ctx.Attr<int>("begin_norm_axis");
-    if (begin_norm_axis !=
-        ctx.Input<phi::DenseTensor>("X")->dims().size() - 1) {
-      this->SetDnnFallback(true);
-    }
-    // NOTE(jiahongyu): Above codes originally enclosed by PADDLE_WITH_MKLDNN
-
-    return phi::KernelKey(input_data_type, ctx.GetPlace());
-  }
-};
-
-class LayerNormOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput("X", "The input tensor.");
-    AddInput("Scale",
-             "(optional) Scale is a 1-dimensional tensor of size "
-             "H(`begin_norm_axis` splits the tensor(`X`) to a matrix [N,H])."
-             "It is applied to the output.")
-        .AsDispensable();
-    AddInput("Bias",
-             "(optional) Bias is a 1-dimensional tensor of size "
-             "H(`begin_norm_axis` splits the tensor(`X`) to a matrix [N,H])."
-             "It is applied to the output.")
-        .AsDispensable();
-    AddOutput("Y", "Result after normalization.");
-    AddOutput("Mean", "Mean of the current mini batch.").AsIntermediate();
-    AddOutput("Variance", "Variance of the current mini batch.")
-        .AsIntermediate();
-
-    AddAttr<float>("epsilon",
-                   "Constant for numerical stability [default 1e-5].")
-        .SetDefault(1e-5)
-        .AddCustomChecker([](const float &epsilon) {
-          PADDLE_ENFORCE_EQ(epsilon >= 0.0f && epsilon <= 0.001f,
-                            true,
-                            platform::errors::InvalidArgument(
-                                "'epsilon' in Op(LayerNorm) should be between"
-                                "0.0 and 0.001, But received [%s].",
-                                epsilon));
-        });
-    AddAttr<int>("begin_norm_axis",
-                 "the axis of `begin_norm_axis ... Rank(X) - 1` will be "
-                 "normalized. `begin_norm_axis` splits the tensor(`X`) to a "
-                 "matrix [N,H]. [default 1].")
-        .SetDefault(1)
-        .AddCustomChecker([](const int &begin_norm_axis) {
-          PADDLE_ENFORCE_GT(begin_norm_axis,
-                            0,
-                            platform::errors::InvalidArgument(
-                                "'begin_norm_axis' in Op(LayerNorm) should be"
-                                "greater than zero. But received [%d].",
-                                begin_norm_axis));
-        });
-    AddComment(R"DOC(
-Assume feature vectors exist on dimensions
-:attr:`begin_norm_axis ... rank(input)` and calculate the moment statistics
-along these dimensions for each feature vector :math:`a` with size
-:math:`H`, then normalize each feature vector using the corresponding
-statistics. After that, apply learnable gain and bias on the normalized
-tensor to scale and shift if :attr:`scale` and :attr:`shift` are set.
-
-Refer to `Layer Normalization <https://arxiv.org/pdf/1607.06450v1.pdf>`_
-)DOC");
-  }
-};
-
-class LayerNormGradOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
-  void InferShape(framework::InferShapeContext *ctx) const override {
-    // check input
-    OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "LayerNormGrad");
-    OP_INOUT_CHECK(ctx->HasInput("Mean"), "Input", "Mean", "LayerNormGrad");
-    OP_INOUT_CHECK(
-        ctx->HasInput("Variance"), "Input", "Variance", "LayerNormGrad");
-    OP_INOUT_CHECK(ctx->HasInput(framework::GradVarName("Y")),
-                   "Input",
-                   framework::GradVarName("Y"),
-                   "LayerNormGrad");
-
-    // check output
-    if (ctx->HasOutput(framework::GradVarName("X"))) {
-      ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X"));
-    }
-    if (ctx->HasOutput(framework::GradVarName("Scale"))) {
-      ctx->SetOutputDim(framework::GradVarName("Scale"),
-                        ctx->GetInputDim("Scale"));
-    }
-    if (ctx->HasOutput(framework::GradVarName("Bias"))) {
-      ctx->SetOutputDim(framework::GradVarName("Bias"),
-                        ctx->GetInputDim("Bias"));
-    }
-  }
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext &ctx) const override {
-    const auto *var = ctx.InputVar(framework::GradVarName("Y"));
-    PADDLE_ENFORCE_NOT_NULL(
-        var,
-        platform::errors::NotFound("Y@GRAD of LayerNorm Op is not found."));
-    const phi::DenseTensor *t = nullptr;
-    if (var->IsType<phi::DenseTensor>()) {
-      t = &var->Get<phi::DenseTensor>();
-    } else if (var->IsType<phi::DenseTensor>()) {
-      t = &var->Get<phi::DenseTensor>();
-    }
-    PADDLE_ENFORCE_NOT_NULL(
-        t, platform::errors::NotFound("Y@GRAD of LayerNorm Op is not found."));
-
-    return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(ctx, "X"),
-                          ctx.GetPlace());
-  }
-};
-
-template <typename T>
-class LayerNormGradOpMaker : public framework::SingleGradOpMaker<T> {
- public:
-  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
-
- protected:
-  void Apply(GradOpPtr<T> op) const override {
-    op->SetType("layer_norm_grad");
-    op->SetInput("X", this->Input("X"));
-    op->SetInput("Mean", this->Output("Mean"));
-    op->SetInput("Variance", this->Output("Variance"));
-    if (this->HasInput("Scale")) {
-      op->SetInput("Scale", this->Input("Scale"));
-      op->SetOutput(framework::GradVarName("Scale"), this->InputGrad("Scale"));
-    }
-
-    if (this->HasInput("Bias")) {
-      op->SetInput("Bias", this->Input("Bias"));
-      op->SetOutput(framework::GradVarName("Bias"), this->InputGrad("Bias"));
-    }
-
-    op->SetInput(framework::GradVarName("Y"), this->OutputGrad("Y"));
-    op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
-    op->SetAttrMap(this->Attrs());
-  }
-};
-
-DECLARE_NO_NEED_BUFFER_VARS_INFERER(LayerNormGradNoNeedBufferVarInferer,
-                                    "Bias");
-
-class LayerNormCompositeGradOpMaker : public prim::CompositeGradOpMakerBase {
-  using prim::CompositeGradOpMakerBase::CompositeGradOpMakerBase;
-
- public:
-  void Apply() override {
-    // get inputs
-    paddle::Tensor x = this->GetSingleForwardInput("X");
-    paddle::Tensor mean = this->GetSingleForwardOutput("Mean");
-    paddle::Tensor var = this->GetSingleForwardOutput("Variance");
-    paddle::Tensor y_grad = this->GetSingleOutputGrad("Y");
-    paddle::optional<paddle::Tensor> scale =
-        this->GetOptionalSingleForwardInput("Scale");
-    paddle::optional<paddle::Tensor> bias =
-        this->GetOptionalSingleForwardInput("Bias");
-
-    // get Attrs
-    auto epsilon = this->Attr<float>("epsilon");
-    auto begin_norm_axis = this->Attr<int>("begin_norm_axis");
-
-    // get outputs
-    paddle::Tensor x_grad = this->GetSingleInputGrad("X");
-    paddle::Tensor scale_grad = this->GetSingleInputGrad("Scale");
-    paddle::Tensor bias_grad = this->GetSingleInputGrad("Bias");
-
-    auto dx_ptr = this->GetOutputPtr(&x_grad);
-    std::string dx_name = this->GetOutputName(x_grad);
-    auto dscale_ptr = this->GetOutputPtr(&scale_grad);
-    std::string dscale_name = this->GetOutputName(scale_grad);
-    auto dbias_ptr = this->GetOutputPtr(&bias_grad);
-    std::string dbias_name = this->GetOutputName(bias_grad);
-
-    VLOG(6) << "Runing layer_norm_grad composite func";
-    prim::layer_norm_grad<prim::DescTensor>(x,
-                                            scale,
-                                            bias,
-                                            mean,
-                                            var,
-                                            y_grad,
-                                            epsilon,
-                                            begin_norm_axis,
-                                            dx_ptr,
-                                            dscale_ptr,
-                                            dbias_ptr);
-
-    this->RecoverOutputName(x_grad, dx_name);
-    this->RecoverOutputName(scale_grad, dscale_name);
-    this->RecoverOutputName(bias_grad, dbias_name);
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-
-DECLARE_INFER_SHAPE_FUNCTOR(layer_norm,
-                            LayerNormInferShapeFunctor,
-                            PD_INFER_META(phi::LayerNormInferMeta));
-
-REGISTER_OPERATOR(layer_norm,
-                  ops::LayerNormOp,
-                  ops::LayerNormOpMaker,
-                  ops::LayerNormGradOpMaker<paddle::framework::OpDesc>,
-                  ops::LayerNormGradOpMaker<paddle::imperative::OpBase>,
-                  ops::LayerNormCompositeGradOpMaker,
-                  LayerNormInferShapeFunctor);
-
-DECLARE_INFER_SHAPE_FUNCTOR(layer_norm_grad,
-                            LayerNormGradInferShapeFunctor,
-                            PD_INFER_META(phi::LayerNormGradInferMeta));
-
-REGISTER_OPERATOR(layer_norm_grad,
-                  ops::LayerNormGradOp,
-                  ops::LayerNormGradNoNeedBufferVarInferer,
-                  LayerNormGradInferShapeFunctor);
diff --git a/paddle/fluid/operators/unity_build_rule.cmake b/paddle/fluid/operators/unity_build_rule.cmake
index 740402b155fc2..6c5f4ccbaa4ad 100644
--- a/paddle/fluid/operators/unity_build_rule.cmake
+++ b/paddle/fluid/operators/unity_build_rule.cmake
@@ -135,7 +135,7 @@ register_unity_group(
   kron_op.cc
   l1_norm_op.cc
   label_smooth_op.cc
-  layer_norm_op.cc
+  generated_op
   mkldnn/layer_norm_mkldnn_op.cc
   mkldnn/layer_norm_mkldnn_op.cc
   linspace_op.cc
diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml
index 895f0ccb112af..661f59ef6a7d6 100644
--- a/paddle/phi/api/yaml/backward.yaml
+++ b/paddle/phi/api/yaml/backward.yaml
@@ -973,6 +973,20 @@
   kernel :
     func : label_smooth_grad
 
+- backward_op : layer_norm_grad
+  forward : layer_norm (Tensor x, Tensor scale, Tensor bias, float epsilon = 1e-5, int begin_norm_axis = 1) -> Tensor(out), Tensor(mean), Tensor(variance)
+  args : (Tensor x,  Tensor scale, Tensor bias, Tensor mean, Tensor variance, Tensor out_grad, float epsilon = 1e-5, int begin_norm_axis = 1)
+  output : Tensor(x_grad), Tensor(scale_grad), Tensor(bias_grad)
+  infer_meta :
+    func : LayerNormGradInferMeta
+    param : [x, scale, bias]
+  kernel :
+    func : layer_norm_grad
+    data_type : x
+  composite : layer_norm_grad(x, scale, bias, mean, variance, out_grad, epsilon, begin_norm_axis, x_grad, scale_grad, bias_grad)
+  no_need_buffer : bias
+  optional : scale, bias
+
 - backward_op : leaky_relu_double_grad
   forward : leaky_relu_grad (Tensor x, Tensor grad_out, float negative_slope) -> Tensor(grad_x)
   args : (Tensor x, Tensor grad_x_grad, float negative_slope)
diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml
index 3b39db07136ed..60e5354dd5354 100755
--- a/paddle/phi/api/yaml/legacy_backward.yaml
+++ b/paddle/phi/api/yaml/legacy_backward.yaml
@@ -454,20 +454,6 @@
   kernel :
     func : hsigmoid_loss_grad
 
-- backward_op : layer_norm_grad
-  forward : layer_norm (Tensor x, Tensor scale, Tensor bias, float epsilon, int begin_norm_axis) -> Tensor(out), Tensor(mean), Tensor(variance)
-  args : (Tensor x,  Tensor scale, Tensor bias, Tensor mean, Tensor variance, Tensor out_grad, float epsilon, int begin_norm_axis)
-  output : Tensor(x_grad), Tensor(scale_grad), Tensor(bias_grad)
-  infer_meta :
-    func : LayerNormGradInferMeta
-    param : [x, scale, bias]
-  kernel :
-    func : layer_norm_grad
-    data_type : out_grad
-  composite : layer_norm_grad(x, scale, bias, mean,varience, out_grad, epsilon, begin_norm_axis, x_grad, scale_grad, bias_grad)
-  no_need_buffer : bias
-  optional : scale, bias
-
 - backward_op : logsumexp_grad
   forward : logsumexp(Tensor x, int64_t[] axis,  bool keepdim,  bool reduce_all) -> Tensor(out)
   args : (Tensor x, Tensor out, Tensor out_grad, int64_t[] axis,  bool keepdim,  bool reduce_all)
diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml
index 0dff2ba6c24e3..921c28caf4798 100755
--- a/paddle/phi/api/yaml/legacy_ops.yaml
+++ b/paddle/phi/api/yaml/legacy_ops.yaml
@@ -588,17 +588,6 @@
     func : increment
   inplace : (x -> out)
 
-- op : layer_norm
-  args : (Tensor x, Tensor scale, Tensor bias, float epsilon, int begin_norm_axis)
-  output : Tensor(out), Tensor(mean), Tensor(variance)
-  infer_meta :
-    func : LayerNormInferMeta
-  kernel :
-    func : layer_norm
-    data_type : x
-  backward : layer_norm_grad
-  optional : scale, bias
-
 - op : less_equal
   args : (Tensor x, Tensor y)
   output : Tensor(out)
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index 7087e913e9450..edf08f5bb8fe5 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -1264,6 +1264,8 @@
     variance : Variance
   extra :
     attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32", bool is_test = false]
+  get_expected_kernel_type :
+    layer_norm : GetLayerNormExpectedKernelType
 
 - op : leaky_relu
   backward : leaky_relu_grad, leaky_relu_double_grad (leaky_relu_grad_grad)
diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml
index 8928a825e4636..c98c9b910d566 100644
--- a/paddle/phi/api/yaml/ops.yaml
+++ b/paddle/phi/api/yaml/ops.yaml
@@ -1135,6 +1135,18 @@
   optional : master_param, skip_update, beta1_pow_out, beta2_pow_out, master_param_outs
   inplace : (param -> param_out), (moment1 -> moment1_out), (moment2 -> moment2_out), (beta1_pow -> beta1_pow_out), (beta2_pow -> beta2_pow_out), (master_param -> master_param_outs)
 
+- op : layer_norm
+  args : (Tensor x, Tensor scale, Tensor bias, float epsilon = 1e-5, int begin_norm_axis = 1)
+  output : Tensor(out), Tensor(mean), Tensor(variance)
+  infer_meta :
+    func : LayerNormInferMeta
+  kernel :
+    func : layer_norm
+    data_type : x
+  backward : layer_norm_grad
+  intermediate : mean, variance
+  optional : scale, bias
+
 - op : leaky_relu
   args : (Tensor x, float negative_slope = 0.02f)
   output : Tensor
diff --git a/paddle/phi/infermeta/ternary.cc b/paddle/phi/infermeta/ternary.cc
index 646767703154e..95a9c9546746c 100644
--- a/paddle/phi/infermeta/ternary.cc
+++ b/paddle/phi/infermeta/ternary.cc
@@ -529,6 +529,12 @@ void LayerNormInferMeta(const MetaTensor& x,
                         MetaTensor* variance,
                         MetaConfig config) {
   auto x_dim = x.dims();
+  PADDLE_ENFORCE_GT(begin_norm_axis,
+                    0,
+                    phi::errors::InvalidArgument(
+                        "'begin_norm_axis' in Op(LayerNorm) should be"
+                        "greater than zero. But received [%d].",
+                        begin_norm_axis));
   PADDLE_ENFORCE_LT(
       begin_norm_axis,
       x_dim.size(),
@@ -588,6 +594,13 @@ void LayerNormInferMeta(const MetaTensor& x,
             right));
   }
 
+  PADDLE_ENFORCE_EQ(epsilon >= 0.0f && epsilon <= 0.001f,
+                    true,
+                    phi::errors::InvalidArgument(
+                        "'epsilon' in Op(LayerNorm) should be between"
+                        "0.0 and 0.001, But received [%s].",
+                        epsilon));
+
   phi::DataType x_dtype = x.dtype();
   out->set_dims(x_dim);
   out->set_dtype(x_dtype);
diff --git a/paddle/phi/ops/compat/layer_norm_sig.cc b/paddle/phi/ops/compat/layer_norm_sig.cc
deleted file mode 100644
index d2e75a700d2b8..0000000000000
--- a/paddle/phi/ops/compat/layer_norm_sig.cc
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/core/compat/op_utils.h"
-
-namespace phi {
-
-KernelSignature LayerNormOpArgumentMapping(const ArgumentMappingContext& ctx) {
-  return KernelSignature("layer_norm",
-                         {"X", "Scale", "Bias"},
-                         {"epsilon", "begin_norm_axis"},
-                         {"Y", "Mean", "Variance"});
-}
-
-KernelSignature LayerNormGradOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature("layer_norm_grad",
-                         {"X", "Scale", "Bias", "Mean", "Variance", "Y@GRAD"},
-                         {"epsilon", "begin_norm_axis"},
-                         {"X@GRAD", "Scale@GRAD", "Bias@GRAD"});
-}
-
-}  // namespace phi
-
-PD_REGISTER_ARG_MAPPING_FN(layer_norm, phi::LayerNormOpArgumentMapping);
-PD_REGISTER_ARG_MAPPING_FN(layer_norm_grad,
-                           phi::LayerNormGradOpArgumentMapping);
diff --git a/python/paddle/nn/functional/norm.py b/python/paddle/nn/functional/norm.py
index ba57e53cf69f4..5cf69fb42b6c5 100644
--- a/python/paddle/nn/functional/norm.py
+++ b/python/paddle/nn/functional/norm.py
@@ -332,7 +332,7 @@ def layer_norm(
         )
 
     if in_dygraph_mode():
-        out, _, _ = _C_ops.layer_norm(x, weight, bias, epsilon, begin_norm_axis)
+        out = _C_ops.layer_norm(x, weight, bias, epsilon, begin_norm_axis)
         return out
 
     else:
diff --git a/test/cpp/fluid/fused/CMakeLists.txt b/test/cpp/fluid/fused/CMakeLists.txt
index 4bbcfbb93d2f8..ff239f2f0c6a1 100644
--- a/test/cpp/fluid/fused/CMakeLists.txt
+++ b/test/cpp/fluid/fused/CMakeLists.txt
@@ -13,7 +13,7 @@ if(WITH_GPU OR WITH_ROCM)
       DEPS tensor
            op_registry
            dropout_op
-           layer_norm_op
+           generated_op
            device_context
            generator
            memory)
@@ -23,7 +23,7 @@ if(WITH_GPU OR WITH_ROCM)
       DEPS tensor
            op_registry
            dropout_op
-           layer_norm_op
+           generated_op
            device_context
            generator
            memory)
@@ -33,7 +33,7 @@ if(WITH_GPU OR WITH_ROCM)
       DEPS tensor
            op_registry
            dropout_op
-           layer_norm_op
+           generated_op
            device_context
            generator
            memory)
diff --git a/test/prim/composite_ops/test_composite_layer_norm.py b/test/prim/composite_ops/test_composite_layer_norm.py
index cca9dcfc4c52a..f56dbea155683 100644
--- a/test/prim/composite_ops/test_composite_layer_norm.py
+++ b/test/prim/composite_ops/test_composite_layer_norm.py
@@ -234,27 +234,27 @@ def compare_forward(self):
         b_p = paddle.to_tensor(b)
 
         expect = expect_forward(x_p, n_shape, w_p, b_p)
-        actual = self.cal_composite(x, n_shape, w, b)
-
-        assert expect[0].numpy().dtype == actual[0].dtype
-        for i in range(3):
-            np.testing.assert_allclose(
-                expect[i].numpy(),
-                actual[i],
-                rtol=attrs.get_rtol("forward"),
-                atol=attrs.get_atol("forward"),
-            )
+        actual, _a_mean, _a_var = self.cal_composite(x, n_shape, w, b)
+
+        assert expect.numpy().dtype == actual.dtype
+        np.testing.assert_allclose(
+            expect.numpy(),
+            actual,
+            rtol=attrs.get_rtol("forward"),
+            atol=attrs.get_atol("forward"),
+        )
 
         expect_2 = expect_forward(x_p, n_shape, None, None)
-        actual_2 = self.cal2_composite(x, n_shape, None, None)
-        assert expect_2[0].numpy().dtype == actual_2[0].dtype
-        for i in range(3):
-            np.testing.assert_allclose(
-                expect_2[i].numpy(),
-                actual_2[i],
-                rtol=attrs.get_rtol("forward"),
-                atol=attrs.get_atol("forward"),
-            )
+        actual_2, _a_mean_2, _a_var_2 = self.cal2_composite(
+            x, n_shape, None, None
+        )
+        assert expect_2.numpy().dtype == actual_2.dtype
+        np.testing.assert_allclose(
+            expect_2.numpy(),
+            actual_2,
+            rtol=attrs.get_rtol("forward"),
+            atol=attrs.get_atol("forward"),
+        )
 
     def test_forward(self):
         for j in self.dtypes:
diff --git a/tools/enforce/count_enforce_by_file.sh b/tools/enforce/count_enforce_by_file.sh
index c1e2903c092ce..fafc3516904d8 100644
--- a/tools/enforce/count_enforce_by_file.sh
+++ b/tools/enforce/count_enforce_by_file.sh
@@ -51,7 +51,6 @@ if [ "$1" != "" ]; then
 fi
 
 FILE_WHITE_LIST="\
-    layer_norm_op.cc \
     box_clip_op.cc \
     box_clip_op.h \
     random_crop_op.h \

From 1ac28b6be5d711a031b344d8ac1e8a2bff8dd3b7 Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Thu, 18 May 2023 11:48:35 +0800
Subject: [PATCH 390/405] Fix typos in executor_statistics.cc (#53917)

---
 .../framework/new_executor/executor_statistics.cc  | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/paddle/fluid/framework/new_executor/executor_statistics.cc b/paddle/fluid/framework/new_executor/executor_statistics.cc
index c1ba3b193f1de..0449fa53e1912 100644
--- a/paddle/fluid/framework/new_executor/executor_statistics.cc
+++ b/paddle/fluid/framework/new_executor/executor_statistics.cc
@@ -181,8 +181,8 @@ void StatisticsEngine::InitStdEvents() {
   names_.push_back("CplusplusEnd");
   name2idx_["RunOp"] = names_.size();
   names_.push_back("RunOp");
-  name2idx_["LuanchKernel"] = names_.size();
-  names_.push_back("LuanchKernel");
+  name2idx_["LaunchKernel"] = names_.size();
+  names_.push_back("LaunchKernel");
   name2idx_["OpCompute"] = names_.size();
   names_.push_back("OpCompute");
   name2idx_["OpInfershape"] = names_.size();
@@ -227,7 +227,7 @@ void StatisticsEngine::InitInnerthreadPriorityForStdEvents() {
 
 void StatisticsEngine::InitInterthreadPriorityForStdEvents() {
   int prio = 0;
-  priorities_[name2idx_["LuanchKernel"]].interthread_priority = ++prio;
+  priorities_[name2idx_["LaunchKernel"]].interthread_priority = ++prio;
   priorities_[name2idx_["AllocateDeviceMem"]].interthread_priority = ++prio;
   priorities_[name2idx_["FreeDeviceMem"]].interthread_priority = ++prio;
   priorities_[name2idx_["ThreadpoolAddTask"]].interthread_priority = ++prio;
@@ -459,11 +459,11 @@ int StatisticsEngine::Stat(const platform::NodeTrees& trees) {
   python_end.total_time = totol.total_time - cplusplus_end.total_time;
   python_end.count = cplusplus_end.count + 1;
 
-  auto& luanch_kernel = statistics_[name2idx_["LuanchKernel"]];
+  auto& launch_kernel = statistics_[name2idx_["LaunchKernel"]];
   const auto& op_compute = statistics_[name2idx_["OpCompute"]];
   const auto& allocate = statistics_[name2idx_["AllocateDeviceMem"]];
-  luanch_kernel.total_time = op_compute.total_time - allocate.total_time;
-  luanch_kernel.count = op_compute.count;
+  launch_kernel.total_time = op_compute.total_time - allocate.total_time;
+  launch_kernel.count = op_compute.count;
 
   if (executor_type_ != ExecutorType::EXECUTOR &&
       statistics_[name2idx_["ThreadpoolAddTask"]].count == 0) {
@@ -536,7 +536,7 @@ int StatisticsEngine::MergeInnerthreadEvents(
       if (names_[evt.evt_idx] == "Total") {
         evt.evt_idx = name2idx_["PythonEnd"];
       } else if (names_[evt.evt_idx] == "OpCompute") {
-        evt.evt_idx = name2idx_["LuanchKernel"];
+        evt.evt_idx = name2idx_["LaunchKernel"];
       }
     }
   }

From 2d0c694811b0d1f1a28b006d59836876cf613739 Mon Sep 17 00:00:00 2001
From: HongyuJia <jiahongyu@baidu.com>
Date: Thu, 18 May 2023 12:17:50 +0800
Subject: [PATCH 391/405] [CustomOp Unittest] Fix XPU unittest, discard static
 backward (#53899)

---
 test/custom_op/test_custom_relu_op_xpu_setup.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test/custom_op/test_custom_relu_op_xpu_setup.py b/test/custom_op/test_custom_relu_op_xpu_setup.py
index db8d56c742911..967b976533ebf 100644
--- a/test/custom_op/test_custom_relu_op_xpu_setup.py
+++ b/test/custom_op/test_custom_relu_op_xpu_setup.py
@@ -46,9 +46,7 @@ def custom_relu_static(
     with static.scope_guard(static.Scope()):
         with static.program_guard(static.Program()):
             x = static.data(name='X', shape=[None, 8], dtype=dtype)
-            x.stop_gradient = False
             out = func(x) if use_func else paddle.nn.functional.relu(x)
-            static.append_backward(out)
 
             exe = static.Executor()
             exe.run(static.default_startup_program())

From 7b1695afe89b88307d88dfcca3f771b3d384ef5e Mon Sep 17 00:00:00 2001
From: xiongkun <xiongkun03@baidu.com>
Date: Thu, 18 May 2023 13:48:52 +0800
Subject: [PATCH 392/405] [Dy2static-Fallback] add set_eval_frame function in
 pybind. (#52006)

* [Dy2static-Fallback] add set_eval_frame function in pybind.
1. add set_eval_frame function in pybind.

* add unittest for eval frame hooker.

* [support py38]

* fix-GeneratorExit error in eval frame hooker

* support python == 3.9

* support 3.10

* fix some comments
---
 paddle/fluid/pybind/jit.cc                | 227 ++++++++++++++++++++++
 paddle/fluid/pybind/jit.h                 |  94 ++++++++-
 paddle/fluid/pybind/pybind.cc             |   1 +
 test/dygraph_to_static/test_eval_frame.py |  56 ++++++
 4 files changed, 377 insertions(+), 1 deletion(-)
 create mode 100644 test/dygraph_to_static/test_eval_frame.py

diff --git a/paddle/fluid/pybind/jit.cc b/paddle/fluid/pybind/jit.cc
index a9c844093d1a5..274c4d795e29e 100644
--- a/paddle/fluid/pybind/jit.cc
+++ b/paddle/fluid/pybind/jit.cc
@@ -14,20 +14,234 @@ limitations under the License. */
 
 #include "paddle/fluid/pybind/jit.h"
 
+#include <Python.h>
+#include <code.h>
+#include <frameobject.h>
+#include <object.h>
+#include <pystate.h>
+
 #include "paddle/fluid/framework/variable.h"
 #include "paddle/fluid/imperative/layer.h"
 #include "paddle/fluid/platform/place.h"
 
+#include "glog/logging.h"
 #include "paddle/fluid/jit/function.h"
 #include "paddle/fluid/jit/function_schema.h"
 #include "paddle/fluid/jit/layer.h"
 #include "paddle/fluid/jit/serializer.h"
+#include "paddle/utils/pybind.h"
 
 namespace py = pybind11;
 
 namespace paddle {
 namespace pybind {
 
+#define unlikely(x) __builtin_expect((x), 0)
+
+// Use static variable to save customed eval hook.
+static Py_tss_t eval_frame_callback_key = {0, 0};
+
+inline static PyObject *eval_frame_callback_get(void) {
+  void *result = PyThread_tss_get(&eval_frame_callback_key);
+  if (unlikely(result == NULL)) {
+    Py_RETURN_NONE;
+  } else {
+    return reinterpret_cast<PyObject *>(result);
+  }
+}
+
+inline static void eval_frame_callback_set(PyObject *obj) {
+  PyThread_tss_set(&eval_frame_callback_key, obj);
+}
+
+// call python default eval frame to interpret current frame.
+inline static PyObject *eval_frame_default(PyThreadState *tstate,
+                                           PyFrameObject *frame,
+                                           int throw_flag) {
+#if PY_VERSION_HEX >= 0x03090000
+  if (tstate == NULL) {
+    tstate = PyThreadState_GET();
+  }
+  return _PyEval_EvalFrameDefault(tstate, frame, throw_flag);
+#else
+  return _PyEval_EvalFrameDefault(frame, throw_flag);
+#endif
+}
+
+// Start a new frame and run code in this frame.
+// Execute a piece of code by default frame-hook.
+inline static PyObject *eval_custom_code(PyThreadState *tstate,
+                                         PyFrameObject *frame,
+                                         PyCodeObject *code,
+                                         int throw_flag) {
+  Py_ssize_t ncells = 0;
+  Py_ssize_t nfrees = 0;
+  Py_ssize_t nlocals_new = code->co_nlocals;
+  Py_ssize_t nlocals_old = frame->f_code->co_nlocals;
+
+  if ((code->co_flags & CO_NOFREE) == 0) {
+    ncells = PyTuple_GET_SIZE(code->co_cellvars);
+    nfrees = PyTuple_GET_SIZE(code->co_freevars);
+  }
+
+  PyFrameObject *shadow = PyFrame_New(tstate, code, frame->f_globals, NULL);
+  if (shadow == NULL) {
+    return NULL;
+  }
+
+  PyObject **fastlocals_old = frame->f_localsplus;
+  PyObject **fastlocals_new = shadow->f_localsplus;
+
+  for (Py_ssize_t i = 0; i < nlocals_old; i++) {
+    Py_XINCREF(fastlocals_old[i]);
+    fastlocals_new[i] = fastlocals_old[i];
+  }
+
+  for (Py_ssize_t i = 0; i < ncells + nfrees; i++) {
+    Py_XINCREF(fastlocals_old[nlocals_old + i]);
+    fastlocals_new[nlocals_new + i] = fastlocals_old[nlocals_old + i];
+  }
+
+  PyObject *result = eval_frame_default(tstate, shadow, throw_flag);
+  Py_DECREF(shadow);
+  return result;
+}
+
+static PyObject *_custom_eval_frame(PyThreadState *tstate,
+                                    PyFrameObject *frame,
+                                    int throw_flag,
+                                    PyObject *callback) {
+  // https://peps.python.org/pep-0558/#fast-locals-proxy-implementation-details
+  // https://devguide.python.org/internals/interpreter/#all-sorts-of-variables
+  if (PyFrame_FastToLocalsWithError(frame) < 0) {
+    return NULL;
+  }
+
+  // NOTE:(xiongkun): Handle GeneratorExit exception: (Spend a day)
+  // In Python, gen close is also a Python function call that will enter this
+  // function with GeneratorExit set, which will cause the PyObject_CallObject
+  // raise SystemError. So we disable the custom behavior for GeneratorExit. def
+  // func():
+  //     iter = iter([1, 2, 3])
+  //     for i in iter:
+  //         return i # <--- Early return, cause a GeneratorExit thrown,
+  //                  # <--- which Cause the PyObject_CallObject raise
+  //                  SystemError.
+  if (PyErr_ExceptionMatches(PyExc_GeneratorExit)) {
+    return eval_frame_default(tstate, frame, throw_flag);
+  }
+
+  // We don't run the current custom_eval_frame behavior for guards.
+  // So we temporarily set the callback to Py_None to drive the correct behavior
+  // in the shim.
+  eval_frame_callback_set(Py_None);
+
+  PyObject *args = Py_BuildValue("(O)", frame);
+  PyObject *result = PyObject_CallObject(callback, args);
+  // result: GuardedCode
+  if (result == NULL) {
+    // internal exception
+    return NULL;
+  } else if (result != Py_None) {
+    //  NOTE: Cache is not supported now
+    PyCodeObject *code = reinterpret_cast<PyCodeObject *>(
+        PyObject_GetAttrString(result, "code"));
+    // Re-enable custom behavior
+    eval_frame_callback_set(callback);
+    return eval_custom_code(tstate, frame, code, throw_flag);
+  } else {
+    // Re-enable custom behavior
+    eval_frame_callback_set(callback);
+    return eval_frame_default(tstate, frame, throw_flag);
+  }
+}
+
+static PyObject *_custom_eval_frame_shim(PyThreadState *tstate,
+                                         PyFrameObject *frame,
+                                         int throw_flag) {
+  PyObject *callback = eval_frame_callback_get();
+
+  if (callback == Py_None) {
+    return eval_frame_default(tstate, frame, throw_flag);
+  }
+
+  return _custom_eval_frame(tstate, frame, throw_flag, callback);
+}
+
+#if PY_VERSION_HEX >= 0x03090000
+static PyObject *custom_eval_frame_shim(PyThreadState *tstate,
+                                        PyFrameObject *frame,
+                                        int throw_flag) {
+  return _custom_eval_frame_shim(tstate, frame, throw_flag);
+}
+#else
+static PyObject *custom_eval_frame_shim(PyFrameObject *frame, int throw_flag) {
+  PyThreadState *tstate = PyThreadState_GET();
+  return _custom_eval_frame_shim(tstate, frame, throw_flag);
+}
+#endif
+
+static PyObject *set_eval_frame(PyObject *new_callback, PyThreadState *tstate) {
+  // Change the eval frame callback and return the old one
+  //  - None: disables: diable custom callback.
+  //  - Python callable(): enables custom callback.
+  //  NOTE: Cache is not supported now
+  PyObject *old_callback = eval_frame_callback_get();
+
+#if PY_VERSION_HEX >= 0x03090000
+  auto *old_eval_frame = _PyInterpreterState_GetEvalFrameFunc(tstate->interp);
+#else
+  // Function pointer.
+  _PyFrameEvalFunction old_eval_frame = tstate->interp->eval_frame;
+#endif
+
+  // NOTE: multi-threading is not supported now
+  if (old_callback != Py_None && new_callback == Py_None) {
+    if (old_eval_frame != &_PyEval_EvalFrameDefault) {
+      VLOG(7) << "set _PyEval_EvalFrameDefault";
+#if PY_VERSION_HEX >= 0x03090000
+      _PyInterpreterState_SetEvalFrameFunc(tstate->interp,
+                                           &_PyEval_EvalFrameDefault);
+#else
+      tstate->interp->eval_frame = &_PyEval_EvalFrameDefault;
+#endif
+    }
+  } else if (old_callback == Py_None && new_callback != Py_None) {
+    if (old_eval_frame != &custom_eval_frame_shim) {
+      VLOG(7) << "set custom_eval_frame_shim";
+#if PY_VERSION_HEX >= 0x03090000
+      _PyInterpreterState_SetEvalFrameFunc(tstate->interp,
+                                           &custom_eval_frame_shim);
+#else
+      tstate->interp->eval_frame = &custom_eval_frame_shim;
+#endif
+    }
+  }
+
+  Py_INCREF(new_callback);
+  eval_frame_callback_set(new_callback);
+
+  return old_callback;
+}
+
+static PyObject *set_eval_frame_py(PyObject *callback) {
+  if (callback != Py_None && !PyCallable_Check(callback)) {
+    VLOG(7) << "callback is not a callable or none, invalid arguments.";
+    RETURN_PY_NONE
+  }
+  return set_eval_frame(callback, PyThreadState_GET());
+}
+
+PyMODINIT_FUNC PyInit__eval_frame(void) {
+  int result = PyThread_tss_create(&eval_frame_callback_key);
+  VLOG(7) << "Set PyThread_tss_create return: " << result;
+
+  Py_INCREF(Py_None);
+  eval_frame_callback_set(Py_None);
+
+  return NULL;
+}
+
 PyTypeObject *g_jit_function_pytype = nullptr;
 using Variable = paddle::framework::Variable;
 
@@ -58,5 +272,18 @@ void BindJit(pybind11::module *m) {
          });
 }
 
+void BindEvalFrame(pybind11::module *m) {
+  PyInit__eval_frame();
+  m->def(
+      "set_eval_frame",
+      [](const py::object &py_func) {
+        VLOG(5) << "start call set_eval_frame_py.";
+        auto ret = set_eval_frame_py(py_func.ptr());
+        auto obj = py::reinterpret_borrow<py::object>(ret);
+        return obj;
+      },
+      py::arg("callback"));
+}
+
 }  // namespace pybind
 }  // namespace paddle
diff --git a/paddle/fluid/pybind/jit.h b/paddle/fluid/pybind/jit.h
index 72bb56c110be5..2d1a2e08d1e89 100644
--- a/paddle/fluid/pybind/jit.h
+++ b/paddle/fluid/pybind/jit.h
@@ -22,10 +22,102 @@ limitations under the License. */
 #include "pybind11/pybind11.h"
 #include "pybind11/stl.h"
 
+// see https://bugs.python.org/issue35886
+// If py_version==3.8.*, we need to redefine _PyEvalFrameFunc and the
+// related functions and structs.
+
+#if PY_VERSION_HEX >= 0x03080000 && PY_VERSION_HEX < 0x3090000
+
+typedef PyObject *(*_PyFrameEvalFunction)(struct _frame *, int);
+
+struct _warnings_runtime_state {
+  /* Both 'filters' and 'onceregistry' can be set in warnings.py;
+     get_warnings_attr() will reset these variables accordingly. */
+  PyObject *filters;        /* List */
+  PyObject *once_registry;  /* Dict */
+  PyObject *default_action; /* String */
+  long filters_version;     // NOLINT
+};
+
+struct _is {
+  struct _is *next;
+  struct _ts *tstate_head;
+
+  int64_t id;
+  int64_t id_refcount;
+  int requires_idref;
+  PyThread_type_lock id_mutex;
+
+  int finalizing;
+
+  PyObject *modules;
+  PyObject *modules_by_index;
+  PyObject *sysdict;
+  PyObject *builtins;
+  PyObject *importlib;
+
+  /* Used in Python/sysmodule.c. */
+  int check_interval;
+
+  /* Used in Modules/_threadmodule.c. */
+  long num_threads;  // NOLINT
+  /* Support for runtime thread stack size tuning.
+     A value of 0 means using the platform's default stack size
+     or the size specified by the THREAD_STACK_SIZE macro. */
+  /* Used in Python/thread.c. */
+  size_t pythread_stacksize;
+
+  PyObject *codec_search_path;
+  PyObject *codec_search_cache;
+  PyObject *codec_error_registry;
+  int codecs_initialized;
+
+  /* fs_codec.encoding is initialized to NULL.
+     Later, it is set to a non-NULL string by _PyUnicode_InitEncodings(). */
+  struct {
+    char *encoding; /* Filesystem encoding (encoded to UTF-8) */
+    char *errors;   /* Filesystem errors (encoded to UTF-8) */
+    _Py_error_handler error_handler;
+  } fs_codec;
+
+  PyConfig config;
+#ifdef HAVE_DLOPEN
+  int dlopenflags;
+#endif
+
+  PyObject *dict; /* Stores per-interpreter state */
+
+  PyObject *builtins_copy;
+  PyObject *import_func;
+  /* Initialized to PyEval_EvalFrameDefault(). */
+  _PyFrameEvalFunction eval_frame;
+
+  Py_ssize_t co_extra_user_count;
+  freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS];
+
+#ifdef HAVE_FORK
+  PyObject *before_forkers;
+  PyObject *after_forkers_parent;
+  PyObject *after_forkers_child;
+#endif
+  /* AtExit module */
+  void (*pyexitfunc)(PyObject *);
+  PyObject *pyexitmodule;
+
+  uint64_t tstate_next_unique_id;
+
+  struct _warnings_runtime_state warnings;
+
+  PyObject *audit_hooks;
+};
+
+#endif
+
 namespace paddle {
 namespace pybind {
 
-void BindJit(pybind11::module* m);
+void BindJit(pybind11::module *m);
+void BindEvalFrame(pybind11::module *m);
 
 }  // namespace pybind
 }  // namespace paddle
diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc
index 34bb746c48ed6..0a01e9e52f5b3 100644
--- a/paddle/fluid/pybind/pybind.cc
+++ b/paddle/fluid/pybind/pybind.cc
@@ -676,6 +676,7 @@ PYBIND11_MODULE(libpaddle, m) {
   BindCudaStream(&m);
   BindXpuStream(&m);
   BindJit(&m);
+  BindEvalFrame(&m);
   BindCustomDevicePy(&m);
 
   // Not used, just make sure cpu_info.cc is linked.
diff --git a/test/dygraph_to_static/test_eval_frame.py b/test/dygraph_to_static/test_eval_frame.py
new file mode 100644
index 0000000000000..5c63517ee2fea
--- /dev/null
+++ b/test/dygraph_to_static/test_eval_frame.py
@@ -0,0 +1,56 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import collections
+import unittest
+
+import paddle
+
+
+class TestEvalFrame(unittest.TestCase):
+    def setUp(self):
+        self.x = paddle.to_tensor(2).astype('int')
+
+    def tearDown(self):
+        pass
+
+    def test_eval_frame(self):
+        CustomCode = collections.namedtuple("CustomCode", ["code"])
+
+        def mul(a, b):
+            return a * b
+
+        code = CustomCode(mul.__code__)
+
+        def callback(frame_obj):
+            # Do your callback function here and return a object with `.code`
+            if frame_obj.f_code.co_name == "add":
+                return code
+            return CustomCode(code=frame_obj.f_code)  # do nothing.
+
+        def add(a, b):
+            return a + b
+
+        x = 1
+        y = 2
+
+        paddle.fluid.core.set_eval_frame(callback)
+        assert add(x, y) == 2, "should be 2"
+        paddle.fluid.core.set_eval_frame(None)
+        assert add(x, y) == 3, "should be 3"
+
+
+if __name__ == "__main__":
+    unittest.main()

From acb5039a65f55234b061445f77df10477336505b Mon Sep 17 00:00:00 2001
From: tianshuo78520a <707759223@qq.com>
Date: Thu, 18 May 2023 14:04:15 +0800
Subject: [PATCH 393/405] Del test_async_read_write in CPU (#53882)

* fix

* fix
---
 test/legacy_test/CMakeLists.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/legacy_test/CMakeLists.txt b/test/legacy_test/CMakeLists.txt
index 85204105a534b..452386f635215 100644
--- a/test/legacy_test/CMakeLists.txt
+++ b/test/legacy_test/CMakeLists.txt
@@ -18,6 +18,10 @@ if(NOT WITH_COVERAGE)
   list(REMOVE_ITEM TEST_OPS test_hapi_hub)
 endif()
 
+if(NOT WITH_GPU)
+  list(REMOVE_ITEM TEST_OPS test_async_read_write)
+endif()
+
 foreach(src ${TEST_OPS})
   py_test(${src} SRCS ${src}.py)
 endforeach()

From 2782b29179ed3498f36a06bb35a28cf3cb61da20 Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Thu, 18 May 2023 14:14:29 +0800
Subject: [PATCH 394/405] Fix typos in elementwise dir (#53907)

---
 paddle/fluid/operators/elementwise/elementwise_add_op.cc      | 2 +-
 paddle/fluid/operators/elementwise/elementwise_div_op.cc      | 2 +-
 paddle/fluid/operators/elementwise/elementwise_floordiv_op.cc | 2 +-
 .../fluid/operators/elementwise/elementwise_heaviside_op.cc   | 2 +-
 paddle/fluid/operators/elementwise/elementwise_max_op.cc      | 4 ++--
 paddle/fluid/operators/elementwise/elementwise_min_op.cc      | 4 ++--
 paddle/fluid/operators/elementwise/elementwise_mod_op.cc      | 2 +-
 paddle/fluid/operators/elementwise/elementwise_mul_op.cc      | 2 +-
 paddle/fluid/operators/elementwise/elementwise_op.h           | 4 ++--
 paddle/fluid/operators/elementwise/elementwise_pow_op.cc      | 2 +-
 paddle/fluid/operators/elementwise/elementwise_sub_op.cc      | 2 +-
 11 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/paddle/fluid/operators/elementwise/elementwise_add_op.cc b/paddle/fluid/operators/elementwise/elementwise_add_op.cc
index 0b522c1d6b35b..91ed63b3e2efd 100644
--- a/paddle/fluid/operators/elementwise/elementwise_add_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_add_op.cc
@@ -46,7 +46,7 @@ class ElementwiseAddOpMaker : public ElementwiseOpMaker {
         "should be int32, int64, float32, float64.");
   }
 
-  std::string GetOpFuntionality() const override {
+  std::string GetOpFunctionality() const override {
     return "Add two tensors element-wise";
   }
 };
diff --git a/paddle/fluid/operators/elementwise/elementwise_div_op.cc b/paddle/fluid/operators/elementwise/elementwise_div_op.cc
index 88856658d3175..7321157f9c28f 100644
--- a/paddle/fluid/operators/elementwise/elementwise_div_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_div_op.cc
@@ -44,7 +44,7 @@ class ElementwiseDivOpMaker : public ElementwiseOpMaker {
         "should be int32, int64, float32, float64.");
   }
 
-  std::string GetOpFuntionality() const override {
+  std::string GetOpFunctionality() const override {
     return "Divide two tensors element-wise";
   }
 };
diff --git a/paddle/fluid/operators/elementwise/elementwise_floordiv_op.cc b/paddle/fluid/operators/elementwise/elementwise_floordiv_op.cc
index 27c12c300bd9f..72011de26af6b 100644
--- a/paddle/fluid/operators/elementwise/elementwise_floordiv_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_floordiv_op.cc
@@ -48,7 +48,7 @@ class ElementwiseFloorDivOpMaker : public ElementwiseOpMaker {
         "should be int32, int64.");
   }
 
-  std::string GetOpFuntionality() const override {
+  std::string GetOpFunctionality() const override {
     return "Floor divide two tensors element-wise";
   }
 };
diff --git a/paddle/fluid/operators/elementwise/elementwise_heaviside_op.cc b/paddle/fluid/operators/elementwise/elementwise_heaviside_op.cc
index 14e5e10743d3f..c13c6165c6bf4 100644
--- a/paddle/fluid/operators/elementwise/elementwise_heaviside_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_heaviside_op.cc
@@ -36,7 +36,7 @@ class ElementwiseHeavisideOpMaker : public ElementwiseOpMaker {
              "which is the value when X = 0. Its dtype should be same as X.");
   }
 
-  std::string GetOpFuntionality() const override {
+  std::string GetOpFunctionality() const override {
     return "Computes the Heaviside step function determined by Y "
            "for each element in X.";
   }
diff --git a/paddle/fluid/operators/elementwise/elementwise_max_op.cc b/paddle/fluid/operators/elementwise/elementwise_max_op.cc
index a945456a226ef..ead7b122be2b4 100644
--- a/paddle/fluid/operators/elementwise/elementwise_max_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_max_op.cc
@@ -44,7 +44,7 @@ class ElementwiseMaxOpMaker : public ElementwiseOpMaker {
     AddInput("Y", "The second tensor holding the elements to be compared.");
   }
 
-  std::string GetOpFuntionality() const override {
+  std::string GetOpFunctionality() const override {
     return "Compare two tensors and returns a new tensor containing the "
            "element-wise maxima.";
   }
@@ -63,7 +63,7 @@ class ElementwiseFMaxOpMaker : public ElementwiseOpMaker {
     AddInput("Y", "The second tensor holding the elements to be compared.");
   }
 
-  std::string GetOpFuntionality() const override {
+  std::string GetOpFunctionality() const override {
     return "Compare two tensors and returns a new tensor containing the "
            "element-wise maxima. If the element of one tensor is nan, "
            "return the element value of the other tensor, if both are nan, "
diff --git a/paddle/fluid/operators/elementwise/elementwise_min_op.cc b/paddle/fluid/operators/elementwise/elementwise_min_op.cc
index 9fd70754888bd..cfc93e791fda4 100644
--- a/paddle/fluid/operators/elementwise/elementwise_min_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_min_op.cc
@@ -41,7 +41,7 @@ class ElementwiseMinOpMaker : public ElementwiseOpMaker {
     AddInput("Y", "The second tensor holding the elements to be compared.");
   }
 
-  std::string GetOpFuntionality() const override {
+  std::string GetOpFunctionality() const override {
     return "Compare two tensors and returns a new tensor containing the "
            "element-wise minima.";
   }
@@ -60,7 +60,7 @@ class ElementwiseFMinOpMaker : public ElementwiseOpMaker {
     AddInput("Y", "The second tensor holding the elements to be compared.");
   }
 
-  std::string GetOpFuntionality() const override {
+  std::string GetOpFunctionality() const override {
     return "Compare two tensors and returns a new tensor containing the "
            "element-wise minima. If the element of one tensor is nan, "
            "return the element value of the other tensor, if both are nan, "
diff --git a/paddle/fluid/operators/elementwise/elementwise_mod_op.cc b/paddle/fluid/operators/elementwise/elementwise_mod_op.cc
index 4caeaf4180f29..5a98351b4460c 100644
--- a/paddle/fluid/operators/elementwise/elementwise_mod_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_mod_op.cc
@@ -46,7 +46,7 @@ class ElementwiseModOpMaker : public ElementwiseOpMaker {
              "should be int32, int64, float32 or float64.");
   }
 
-  std::string GetOpFuntionality() const override {
+  std::string GetOpFunctionality() const override {
     return "Mod two tensors element-wise";
   }
 };
diff --git a/paddle/fluid/operators/elementwise/elementwise_mul_op.cc b/paddle/fluid/operators/elementwise/elementwise_mul_op.cc
index 0fc1cab739105..9967d0fd95c4a 100644
--- a/paddle/fluid/operators/elementwise/elementwise_mul_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_mul_op.cc
@@ -44,7 +44,7 @@ class ElementwiseMulOpMaker : public ElementwiseOpMaker {
         "should be int32, int64, float32, float64.");
   }
 
-  std::string GetOpFuntionality() const override {
+  std::string GetOpFunctionality() const override {
     return "Multiply two tensors element-wise";
   }
 };
diff --git a/paddle/fluid/operators/elementwise/elementwise_op.h b/paddle/fluid/operators/elementwise/elementwise_op.h
index 2f8db0e737a21..67954e3fa2648 100644
--- a/paddle/fluid/operators/elementwise/elementwise_op.h
+++ b/paddle/fluid/operators/elementwise/elementwise_op.h
@@ -225,7 +225,7 @@ class ElementwiseOpMaker : public framework::OpProtoAndCheckerMaker {
   }
   virtual void AddOpComment() { AddComment(GetCommentExamples()); }
 
-  virtual std::string GetOpFuntionality() const { return ""; }
+  virtual std::string GetOpFunctionality() const { return ""; }
 
   virtual std::string GetName() const = 0;
   virtual std::string GetEquation() const = 0;
@@ -269,7 +269,7 @@ For example:
 
 )DOC",
                            GetName(),
-                           GetOpFuntionality(),
+                           GetOpFunctionality(),
                            GetEquation());
   }
 };
diff --git a/paddle/fluid/operators/elementwise/elementwise_pow_op.cc b/paddle/fluid/operators/elementwise/elementwise_pow_op.cc
index d2c047f0e4bbd..34f9e6f61cbfc 100644
--- a/paddle/fluid/operators/elementwise/elementwise_pow_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_pow_op.cc
@@ -76,7 +76,7 @@ class ElementwisePowOpMaker : public ElementwiseOpMaker {
 
   void AddInputY() override { AddInput("Y", "(Variable), The exponents."); }
 
-  std::string GetOpFuntionality() const override {
+  std::string GetOpFunctionality() const override {
     return "First tensor elements raised to powers from the second tensor, "
            "element-wise.";
   }
diff --git a/paddle/fluid/operators/elementwise/elementwise_sub_op.cc b/paddle/fluid/operators/elementwise/elementwise_sub_op.cc
index 0c8c9bc6e804c..87c4e4977258f 100644
--- a/paddle/fluid/operators/elementwise/elementwise_sub_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_sub_op.cc
@@ -49,7 +49,7 @@ class ElementwiseSubOpMaker : public ElementwiseOpMaker {
         "should be int32, int64, float32, float64.");
   }
 
-  std::string GetOpFuntionality() const override {
+  std::string GetOpFunctionality() const override {
     return "Substract two tensors element-wise";
   }
 };

From a862debf8a88713ab7351596c20669bed4398854 Mon Sep 17 00:00:00 2001
From: Wang Xin <xinwang614@gmail.com>
Date: Thu, 18 May 2023 14:16:13 +0800
Subject: [PATCH 395/405] move sequence_mask op InferShape func (#53782)

* move sequence_mask op InferShape func

* add dtype infer
---
 .../sequence_ops/sequence_mask_op.cc          | 25 +++++++------------
 paddle/phi/infermeta/binary.cc                | 19 ++++++++++++++
 paddle/phi/infermeta/binary.h                 |  6 +++++
 3 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/paddle/fluid/operators/sequence_ops/sequence_mask_op.cc b/paddle/fluid/operators/sequence_ops/sequence_mask_op.cc
index f50fc2268c4e8..a6cd59e44dff0 100644
--- a/paddle/fluid/operators/sequence_ops/sequence_mask_op.cc
+++ b/paddle/fluid/operators/sequence_ops/sequence_mask_op.cc
@@ -12,7 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "paddle/fluid/framework/infershape_utils.h"
 #include "paddle/fluid/framework/op_registry.h"
+#include "paddle/phi/core/infermeta_utils.h"
+#include "paddle/phi/infermeta/binary.h"
 
 namespace paddle {
 namespace operators {
@@ -21,21 +24,6 @@ class SequenceMaskOp : public framework::OperatorWithKernel {
  public:
   using framework::OperatorWithKernel::OperatorWithKernel;
 
-  void InferShape(framework::InferShapeContext* ctx) const override {
-    OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "SequenceMask");
-    OP_INOUT_CHECK(ctx->HasOutput("Y"), "Output", "Y", "SequenceMask");
-
-    int maxlen = ctx->Attrs().Get<int>("maxlen");
-    auto dim = phi::vectorize<int>(ctx->GetInputDim("X"));
-
-    if (ctx->HasInputs("MaxLenTensor")) {
-      dim.push_back(-1);
-    } else {
-      dim.push_back(maxlen > 0 ? maxlen : -1);
-    }
-    ctx->SetOutputDim("Y", phi::make_ddim(dim));
-  }
-
  protected:
   phi::KernelKey GetExpectedKernelType(
       const framework::ExecutionContext& ctx) const override {
@@ -93,9 +81,14 @@ If maxlen < 0, maxlen = max(X)
 }  // namespace operators
 }  // namespace paddle
 
+DECLARE_INFER_SHAPE_FUNCTOR(sequence_mask,
+                            SequenceMaskInferShapeFunctor,
+                            PD_INFER_META(phi::SequenceMaskInferMeta));
+
 REGISTER_OPERATOR(
     sequence_mask,
     paddle::operators::SequenceMaskOp,
     paddle::operators::SequenceMaskOpMaker,
     paddle::framework::EmptyGradOpMaker<paddle::framework::OpDesc>,
-    paddle::framework::EmptyGradOpMaker<paddle::imperative::OpBase>);
+    paddle::framework::EmptyGradOpMaker<paddle::imperative::OpBase>,
+    SequenceMaskInferShapeFunctor);
diff --git a/paddle/phi/infermeta/binary.cc b/paddle/phi/infermeta/binary.cc
index e32b1fc241b48..dde953b5d9db1 100644
--- a/paddle/phi/infermeta/binary.cc
+++ b/paddle/phi/infermeta/binary.cc
@@ -23,6 +23,7 @@ limitations under the License. */
 #include "paddle/phi/common/type_traits.h"
 #include "paddle/phi/core/ddim.h"
 #include "paddle/phi/core/infermeta_utils.h"
+#include "paddle/phi/core/utils/data_type.h"
 #include "paddle/phi/infermeta/unary.h"
 #include "paddle/phi/kernels/cpu/conv_util.h"
 #include "paddle/phi/kernels/funcs/axis_utils.h"
@@ -2584,6 +2585,24 @@ void SearchsortedInferMeta(const MetaTensor& sorted_sequence,
   }
 }
 
+void SequenceMaskInferMeta(const MetaTensor& x,
+                           const MetaTensor& max_len_tensor,
+                           int maxlen,
+                           int out_dtype,
+                           MetaTensor* y) {
+  auto dim = phi::vectorize<int>(x.dims());
+
+  if (max_len_tensor) {
+    dim.push_back(-1);
+  } else {
+    dim.push_back(maxlen > 0 ? maxlen : -1);
+  }
+
+  y->set_dims(phi::make_ddim(dim));
+  auto out_phi_dtype = phi::TransToPhiDataType(out_dtype);
+  y->set_dtype(out_phi_dtype);
+}
+
 void SoftmaxMaskFuseInferMeta(const MetaTensor& x,
                               const MetaTensor& mask,
                               MetaTensor* out) {
diff --git a/paddle/phi/infermeta/binary.h b/paddle/phi/infermeta/binary.h
index 3a38e6b599021..ff03ecb8f8a75 100644
--- a/paddle/phi/infermeta/binary.h
+++ b/paddle/phi/infermeta/binary.h
@@ -399,6 +399,12 @@ void SearchsortedInferMeta(const MetaTensor& sorted_sequence,
                            bool right,
                            MetaTensor* out);
 
+void SequenceMaskInferMeta(const MetaTensor& x,
+                           const MetaTensor& max_len_tensor,
+                           int maxlen,
+                           int out_dtype,
+                           MetaTensor* y);
+
 void SoftmaxMaskFuseInferMeta(const MetaTensor& x,
                               const MetaTensor& mask,
                               MetaTensor* out);

From d8407c51b79d38c39461b83af1fe2ae926f2bc20 Mon Sep 17 00:00:00 2001
From: LoneRanger <836253168@qq.com>
Date: Thu, 18 May 2023 14:17:53 +0800
Subject: [PATCH 396/405] add fp16 and bf16 for trunc (#53876)

---
 paddle/phi/kernels/gpu/trunc_grad_kernel.cu   |  4 ++-
 paddle/phi/kernels/gpu/trunc_kernel.cu        | 18 ++++++++--
 .../fluid/tests/unittests/test_trunc_op.py    | 33 ++++++++++++++++++-
 3 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/paddle/phi/kernels/gpu/trunc_grad_kernel.cu b/paddle/phi/kernels/gpu/trunc_grad_kernel.cu
index 8a88383e6e4f0..40e1404cd900d 100644
--- a/paddle/phi/kernels/gpu/trunc_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/trunc_grad_kernel.cu
@@ -52,4 +52,6 @@ PD_REGISTER_KERNEL(trunc_grad,
                    float,
                    double,
                    int,
-                   int64_t) {}
+                   int64_t,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/paddle/phi/kernels/gpu/trunc_kernel.cu b/paddle/phi/kernels/gpu/trunc_kernel.cu
index dfc4f6589e9cf..bdbdb80a97f5c 100644
--- a/paddle/phi/kernels/gpu/trunc_kernel.cu
+++ b/paddle/phi/kernels/gpu/trunc_kernel.cu
@@ -17,6 +17,7 @@
 #include "paddle/phi/backends/gpu/gpu_context.h"
 #include "paddle/phi/backends/gpu/gpu_info.h"
 #include "paddle/phi/backends/gpu/gpu_primitives.h"
+#include "paddle/phi/common/amp_type_traits.h"
 #include "paddle/phi/core/kernel_registry.h"
 
 namespace phi {
@@ -27,7 +28,10 @@ template <typename T>
 class TruncFunctor {
  public:
   __device__ TruncFunctor(const T x) : x_(x) {}
-  __device__ T operator()() { return trunc(x_); }
+  __device__ T operator()() {
+    using MPType = typename phi::dtype::MPTypeTrait<T>::Type;
+    return static_cast<T>(trunc(static_cast<MPType>(x_)));
+  }
 
  public:
   const T x_;
@@ -78,5 +82,13 @@ void TruncKernel(const Context& dev_ctx,
 
 }  // namespace phi
 
-PD_REGISTER_KERNEL(
-    trunc, GPU, ALL_LAYOUT, phi::TruncKernel, float, double, int, int64_t) {}
+PD_REGISTER_KERNEL(trunc,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::TruncKernel,
+                   float,
+                   double,
+                   int,
+                   int64_t,
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/python/paddle/fluid/tests/unittests/test_trunc_op.py b/python/paddle/fluid/tests/unittests/test_trunc_op.py
index 7a05a47a2458a..404403fca0453 100644
--- a/python/paddle/fluid/tests/unittests/test_trunc_op.py
+++ b/python/paddle/fluid/tests/unittests/test_trunc_op.py
@@ -15,9 +15,10 @@
 import unittest
 
 import numpy as np
-from eager_op_test import OpTest
+from eager_op_test import OpTest, convert_float_to_uint16
 
 import paddle
+from paddle.fluid import core
 
 paddle.enable_static()
 
@@ -90,5 +91,35 @@ def test_errors(self):
             self.assertRaises(TypeError, paddle.trunc, x)
 
 
+class TestTruncFP16OP(TestTruncOp):
+    def init_dtype_type(self):
+        self.dtype = np.float16
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not complied with CUDA and not support the bfloat16",
+)
+class TestTruncBF16OP(OpTest):
+    def setUp(self):
+        self.python_api = paddle.trunc
+        self.op_type = "trunc"
+        self.dtype = np.uint16
+        np.random.seed(2021)
+        x = np.random.random((20, 20)).astype("float32")
+        out = np.trunc(x)
+        self.inputs = {'X': convert_float_to_uint16(x)}
+        self.outputs = {'Out': convert_float_to_uint16(out)}
+
+    def test_check_output(self):
+        place = core.CUDAPlace(0)
+        self.check_output_with_place(place)
+
+    def test_check_grad(self):
+        place = core.CUDAPlace(0)
+        self.check_grad_with_place(place, ['X'], 'Out', numeric_grad_delta=1e-5)
+
+
 if __name__ == "__main__":
     unittest.main()

From e916e80c3615e6197209f1329ba2431f5baae300 Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Thu, 18 May 2023 14:25:06 +0800
Subject: [PATCH 397/405] Fix typos, test=document_fix (#53927)

---
 paddle/fluid/dialect/pd_type.h   |  2 +-
 paddle/ir/attribute_base.h       | 10 +++++-----
 paddle/ir/storage_manager.cc     |  8 ++++----
 paddle/ir/storage_manager.h      |  4 ++--
 paddle/ir/type_base.h            | 12 ++++++------
 test/cpp/ir/ir_attribute_test.cc |  2 +-
 6 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/paddle/fluid/dialect/pd_type.h b/paddle/fluid/dialect/pd_type.h
index 2040828a146ac..0d4f8aa27b423 100644
--- a/paddle/fluid/dialect/pd_type.h
+++ b/paddle/fluid/dialect/pd_type.h
@@ -22,7 +22,7 @@ namespace dialect {
 #define GET_PADDLE_TYPE_LIST paddle::dialect::DenseTensorType
 
 ///
-/// \brief Define built-in parameteric types.
+/// \brief Define built-in parametric types.
 ///
 class DenseTensorType : public ir::Type {
  public:
diff --git a/paddle/ir/attribute_base.h b/paddle/ir/attribute_base.h
index b7c36a38458f2..eaf0deb899f34 100644
--- a/paddle/ir/attribute_base.h
+++ b/paddle/ir/attribute_base.h
@@ -92,10 +92,10 @@ struct AttributeManager;
 /// \brief AttributeStorage is used to store all information of a Attribute. A
 /// Attribute object contains a AttributeStorage. For non-parameter attribute,
 /// the information includes: TypeId, so AttributeStorage only needs to include
-/// AbstractAttribute; For parameteric attribute, in addition to
-/// AbstractAttribute/TypeId, parameteric information needs to be included. So
-/// that, non-parameteric attribute can be constructed by AttributeStorage
-/// directly but parameteric attribute should be constructed by Derived
+/// AbstractAttribute; For parametric attribute, in addition to
+/// AbstractAttribute/TypeId, parametric information needs to be included. So
+/// that, non-parametric attribute can be constructed by AttributeStorage
+/// directly but parametric attribute should be constructed by Derived
 /// AttributeStorage.
 ///
 class AttributeStorage : public StorageManager::StorageBase {
@@ -144,7 +144,7 @@ class AttributeStorage : public StorageManager::StorageBase {
 struct AttributeManager {
   ///
   /// \brief Get a unique instance of Attribute T from IrContext. Note: For a
-  /// parameteric attribute, if not found in IrContext, it will try to create a
+  /// parametric attribute, if not found in IrContext, it will try to create a
   /// new instance and register it to IrContext; for a parameterless attribute,
   /// only search.
   ///
diff --git a/paddle/ir/storage_manager.cc b/paddle/ir/storage_manager.cc
index 632c130fae67b..a1fe0c1d3e0ca 100644
--- a/paddle/ir/storage_manager.cc
+++ b/paddle/ir/storage_manager.cc
@@ -19,7 +19,7 @@
 
 namespace ir {
 // This is a structure for creating, caching, and looking up Storage of
-// parameteric types.
+// parametric types.
 struct ParametricStorageManager {
   using StorageBase = StorageManager::StorageBase;
 
@@ -41,7 +41,7 @@ struct ParametricStorageManager {
       auto pr = parametric_instances_.equal_range(hash_value);
       while (pr.first != pr.second) {
         if (equal_func(pr.first->second)) {
-          VLOG(4) << "Found a cached parameteric storage of: [param_hash="
+          VLOG(4) << "Found a cached parametric storage of: [param_hash="
                   << hash_value << ", storage_ptr=" << pr.first->second << "].";
           return pr.first->second;
         }
@@ -50,7 +50,7 @@ struct ParametricStorageManager {
     }
     StorageBase *storage = constructor();
     parametric_instances_.emplace(hash_value, storage);
-    VLOG(4) << "No cache found, construct and cache a new parameteric storage "
+    VLOG(4) << "No cache found, construct and cache a new parametric storage "
                "of: [param_hash="
             << hash_value << ", storage_ptr=" << storage << "].";
     return storage;
@@ -94,7 +94,7 @@ StorageManager::StorageBase *StorageManager::GetParameterlessStorageImpl(
 
 void StorageManager::RegisterParametricStorageImpl(TypeId type_id) {
   std::lock_guard<ir::SpinLock> guard(parametric_instance_lock_);
-  VLOG(4) << "Register a parameteric storage of: [TypeId_hash="
+  VLOG(4) << "Register a parametric storage of: [TypeId_hash="
           << std::hash<ir::TypeId>()(type_id) << "].";
   parametric_instance_.emplace(type_id,
                                std::make_unique<ParametricStorageManager>());
diff --git a/paddle/ir/storage_manager.h b/paddle/ir/storage_manager.h
index f79855525d506..be83df3fcb101 100644
--- a/paddle/ir/storage_manager.h
+++ b/paddle/ir/storage_manager.h
@@ -33,7 +33,7 @@ struct ParametricStorageManager;
 /// Storage class must be a derived class of StorageManager::StorageBase.
 /// There are two types of Storage class:
 /// One is a parameterless type, which can directly obtain an instance through
-/// the get method; The other is a parameteric type, which needs to comply with
+/// the get method; The other is a parametric type, which needs to comply with
 /// the following conditions: (1) Need to define a type alias called ParamKey,
 /// it serves as the unique identifier for the Storage class; (2) Need to
 /// provide a hash method on the ParamKey for storage and access; (3) Need to
@@ -134,7 +134,7 @@ class StorageManager {
   void RegisterParameterlessStorageImpl(
       TypeId type_id, std::function<StorageBase *()> constructor);
 
-  // This map is a mapping between type id and parameteric type storage.
+  // This map is a mapping between type id and parametric type storage.
   std::unordered_map<TypeId, std::unique_ptr<ParametricStorageManager>>
       parametric_instance_;
 
diff --git a/paddle/ir/type_base.h b/paddle/ir/type_base.h
index c78cd54c9b392..3ecde46d2b678 100644
--- a/paddle/ir/type_base.h
+++ b/paddle/ir/type_base.h
@@ -24,7 +24,7 @@ class Dialect;
 ///
 /// \brief Abstract the properties and behaviors common to all Type classes into
 /// an AbstractType class. There are two types in Type system:
-/// non-parameter/parameterless type and parameteric-type. The common attributes
+/// non-parameter/parameterless type and parametric-type. The common attributes
 /// of all types is TypeId (and possibly others). Therefore, construct a class
 /// with TypeId as its member.
 ///
@@ -95,10 +95,10 @@ struct TypeManager;
 ///
 /// \brief TypeStorage is used to store all information of a Type. A Type object
 /// contains a TypeStorage. For non-parameter type, the information includes:
-/// TypeId, so TypeStorage only needs to include AbstractType; For parameteric
-/// type, in addition to AbstractType/TypeId, parameteric information needs to
-/// be included. So that, non-parameteric type can be constructed by TypeStorage
-/// directly but parameteric type should be constructed by Derived TypeStorage.
+/// TypeId, so TypeStorage only needs to include AbstractType; For parametric
+/// type, in addition to AbstractType/TypeId, parametric information needs to
+/// be included. So that, non-parametric type can be constructed by TypeStorage
+/// directly but parametric type should be constructed by Derived TypeStorage.
 ///
 class TypeStorage : public StorageManager::StorageBase {
   friend StorageManager;
@@ -144,7 +144,7 @@ class TypeStorage : public StorageManager::StorageBase {
 struct TypeManager {
   ///
   /// \brief Get a unique instance of Type T from IrContext. Note: For a
-  /// parameteric_type, if not found in IrContext, it will try to create a new
+  /// parametric_type, if not found in IrContext, it will try to create a new
   /// instance and register it to IrContext; for a parameterless type, only
   /// search.
   ///
diff --git a/test/cpp/ir/ir_attribute_test.cc b/test/cpp/ir/ir_attribute_test.cc
index 8aec0bec4b64d..d361a1c20d09a 100644
--- a/test/cpp/ir/ir_attribute_test.cc
+++ b/test/cpp/ir/ir_attribute_test.cc
@@ -45,7 +45,7 @@ TEST(attribute_test, attribute_base) {
 
 TEST(attribute_test, built_in_attribute) {
   ir::IrContext *ctx = ir::IrContext::Instance();
-  // Test 1: Test the parameteric built-in attribute of IrContext.
+  // Test 1: Test the parametric built-in attribute of IrContext.
   std::string str_tmp = "string_a";
   ir::Attribute string_attr_1 = ir::StrAttribute::get(ctx, str_tmp);
   ir::Attribute string_attr_2 = ir::StrAttribute::get(ctx, str_tmp);

From 117e951b5eecd181b874db4bcdbb3f7977b5d690 Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Thu, 18 May 2023 14:25:34 +0800
Subject: [PATCH 398/405] Fix typos (#53912)

---
 paddle/fluid/eager/to_static/run_program_op_node.h | 10 +++++-----
 paddle/fluid/framework/scope.h                     |  4 ++--
 paddle/fluid/pybind/pybind.cc                      |  2 +-
 python/paddle/jit/dy2static/partial_program.py     |  2 +-
 python/paddle/nn/functional/distance.py            |  2 +-
 python/paddle/nn/functional/norm.py                |  2 +-
 6 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/paddle/fluid/eager/to_static/run_program_op_node.h b/paddle/fluid/eager/to_static/run_program_op_node.h
index b4deb4e4ac306..548af27a8ac4f 100644
--- a/paddle/fluid/eager/to_static/run_program_op_node.h
+++ b/paddle/fluid/eager/to_static/run_program_op_node.h
@@ -443,11 +443,11 @@ inline void RunProgramAPI(
       VLOG(4) << "don't require any grad, set this scope can reused";
       VLOG(4) << "is_test: " << is_test
               << ", require_any_grad: " << require_any_grad;
-      global_inner_scope->SetCanReuesd(true);
+      global_inner_scope->SetCanReused(true);
       details::GcScope(global_inner_scope);
     } else {
       VLOG(4) << "not test, set this scope can not reused";
-      global_inner_scope->SetCanReuesd(false);
+      global_inner_scope->SetCanReused(false);
     }
   }
 
@@ -582,7 +582,7 @@ inline void RunProgramGradAPI(
                                                    *backward_global_block,
                                                    global_inner_scope);
     VLOG(4) << "after backward gc all vars";
-    global_inner_scope->SetCanReuesd(true);
+    global_inner_scope->SetCanReused(true);
     details::GcScope(global_inner_scope);
   }
 }
@@ -599,9 +599,9 @@ class GradNodeRunProgram : public egr::GradNodeBase {
       // Normally out_scope_vec.size() == 1. for safty, we add for-loop here.
       for (size_t i = 0; i < out_scope_vec->size(); ++i) {
         paddle::framework::Scope *global_inner_scope = out_scope_vec->at(i);
-        global_inner_scope->SetCanReuesd(true);
+        global_inner_scope->SetCanReused(true);
         details::GcScope(global_inner_scope);
-        VLOG(4) << "global_inner_scope SetCanReuesd";
+        VLOG(4) << "global_inner_scope SetCanReused";
       }
     }
   }
diff --git a/paddle/fluid/framework/scope.h b/paddle/fluid/framework/scope.h
index b87a294878051..b0b418c85a303 100644
--- a/paddle/fluid/framework/scope.h
+++ b/paddle/fluid/framework/scope.h
@@ -122,9 +122,9 @@ class Scope {
   std::string Rename(const std::string& origin_name) const;
 
   // only for dygraph_to_static
-  bool CanReuesd() const { return can_reused_; }
+  bool CanReused() const { return can_reused_; }
 
-  void SetCanReuesd(bool can_reused) { can_reused_ = can_reused; }
+  void SetCanReused(bool can_reused) { can_reused_ = can_reused; }
 
  protected:
   struct KeyHasher {
diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc
index 0a01e9e52f5b3..05eccd45c8fa6 100644
--- a/paddle/fluid/pybind/pybind.cc
+++ b/paddle/fluid/pybind/pybind.cc
@@ -1210,7 +1210,7 @@ All parameter, weight, gradient are variables in Paddle.
            Delete all sub-scopes of the current scope.
            )DOC")
       .def("_kids", &Scope::kids)
-      .def_property("_can_reuesd", &Scope::CanReuesd, &Scope::SetCanReuesd);
+      .def_property("_can_reused", &Scope::CanReused, &Scope::SetCanReused);
 
   m.def(
       "Scope",
diff --git a/python/paddle/jit/dy2static/partial_program.py b/python/paddle/jit/dy2static/partial_program.py
index ad2e62b9e0461..042977988d4b3 100644
--- a/python/paddle/jit/dy2static/partial_program.py
+++ b/python/paddle/jit/dy2static/partial_program.py
@@ -259,7 +259,7 @@ def _get_scope(self, program_id=None, use_scope_cache=False):
                 return scope
             else:
                 for scope in self._scope_cache[program_id]:
-                    if scope._can_reuesd:
+                    if scope._can_reused:
                         return scope
                 scope = core.Scope()
                 self._scope_cache[program_id].append(scope)
diff --git a/python/paddle/nn/functional/distance.py b/python/paddle/nn/functional/distance.py
index e8e209be18a94..067a49816dd22 100644
--- a/python/paddle/nn/functional/distance.py
+++ b/python/paddle/nn/functional/distance.py
@@ -70,7 +70,7 @@ def pairwise_distance(x, y, p=2.0, epsilon=1e-6, keepdim=False, name=None):
     """
     if in_dygraph_mode():
         sub = _C_ops.subtract(x, y)
-        # p_norm op has not uesd epsilon, so change it to the following.
+        # p_norm op has not used epsilon, so change it to the following.
         if epsilon != 0.0:
             epsilon = paddle.fluid.dygraph.base.to_variable(
                 [epsilon], dtype=sub.dtype
diff --git a/python/paddle/nn/functional/norm.py b/python/paddle/nn/functional/norm.py
index 5cf69fb42b6c5..e1c3d985a68db 100644
--- a/python/paddle/nn/functional/norm.py
+++ b/python/paddle/nn/functional/norm.py
@@ -127,7 +127,7 @@ def batch_norm(
     """
     Applies Batch Normalization as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift .
 
-    nn.functional.batch_norm is uesd for nn.BatchNorm1D, nn.BatchNorm2D, nn.BatchNorm3D. Please use above API for BatchNorm.
+    nn.functional.batch_norm is used for nn.BatchNorm1D, nn.BatchNorm2D, nn.BatchNorm3D. Please use above API for BatchNorm.
 
     Parameters:
         x(Tesnor): input value. It's data type should be float32, float64.

From 0bed22031a8a19f130d34510383fa890e0a262e7 Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Thu, 18 May 2023 14:31:22 +0800
Subject: [PATCH 399/405] Add segment_pool tests (#53785)

---
 paddle/phi/backends/gpu/gpu_primitives.h      | 102 ++++++++++++
 paddle/phi/kernels/funcs/segment_pooling.cu   |   4 +
 .../kernels/gpu/segment_pool_grad_kernel.cu   |   3 +-
 paddle/phi/kernels/gpu/segment_pool_kernel.cu |   3 +-
 .../fluid/tests/unittests/test_segment_ops.py | 148 ++++++++++++++++--
 python/paddle/geometric/math.py               |   8 +-
 6 files changed, 245 insertions(+), 23 deletions(-)

diff --git a/paddle/phi/backends/gpu/gpu_primitives.h b/paddle/phi/backends/gpu/gpu_primitives.h
index 252ed90e44114..a77527c081650 100644
--- a/paddle/phi/backends/gpu/gpu_primitives.h
+++ b/paddle/phi/backends/gpu/gpu_primitives.h
@@ -445,6 +445,57 @@ CUDA_ATOMIC_WRAPPER(Max, phi::dtype::float16) {
 }
 #endif
 
+inline static __device__ uint32_t bf16_max_to_low_half(uint32_t val, float x) {
+  phi::dtype::bfloat16 low_half;
+  // The bfloat16 in lower 16bits
+  low_half.x = static_cast<uint16_t>(val & 0xFFFFu);
+  low_half =
+      static_cast<phi::dtype::bfloat16>(max(static_cast<float>(low_half), x));
+  return (val & 0xFFFF0000u) | low_half.x;
+}
+
+inline static __device__ uint32_t bf16_max_to_high_half(uint32_t val, float x) {
+  phi::dtype::bfloat16 high_half;
+  // The bfloat16 in higher 16bits
+  high_half.x = static_cast<uint16_t>(val >> 16);
+  high_half =
+      static_cast<phi::dtype::bfloat16>(max(static_cast<float>(high_half), x));
+  return (val & 0xFFFFu) | (static_cast<uint32_t>(high_half.x) << 16);
+}
+
+CUDA_ATOMIC_WRAPPER(Max, phi::dtype::bfloat16) {
+  if (*address >= val) {
+    return *address;
+  }
+  uint32_t *address_as_ui = reinterpret_cast<uint32_t *>(
+      reinterpret_cast<char *>(address) -
+      (reinterpret_cast<uintptr_t>(address) & 0x02));
+  float val_f = static_cast<float>(val);
+  uint32_t old = *address_as_ui;
+  uint32_t assumed;
+  if (((uintptr_t)address & 0x02) == 0) {
+    // The bfloat16 value stay at lower 16 bits of the address.
+    do {
+      assumed = old;
+      old = atomicCAS(
+          address_as_ui, assumed, bf16_max_to_low_half(assumed, val_f));
+    } while (old != assumed);
+    phi::dtype::bfloat16 ret;
+    ret.x = old & 0xFFFFu;
+    return ret;
+  } else {
+    // The bfloat16 value stay at higher 16 bits of the address.
+    do {
+      assumed = old;
+      old = atomicCAS(
+          address_as_ui, assumed, bf16_max_to_high_half(assumed, val_f));
+    } while (old != assumed);
+    phi::dtype::bfloat16 ret;
+    ret.x = old >> 16;
+    return ret;
+  }
+}
+
 // For atomicMin
 USE_CUDA_ATOMIC(Min, int);
 USE_CUDA_ATOMIC(Min, unsigned int);
@@ -580,6 +631,57 @@ CUDA_ATOMIC_WRAPPER(Min, phi::dtype::float16) {
 }
 #endif
 
+inline static __device__ uint32_t bf16_min_to_low_half(uint32_t val, float x) {
+  phi::dtype::bfloat16 low_half;
+  // The bfloat16 in lower 16bits
+  low_half.x = static_cast<uint16_t>(val & 0xFFFFu);
+  low_half =
+      static_cast<phi::dtype::bfloat16>(min(static_cast<float>(low_half), x));
+  return (val & 0xFFFF0000u) | low_half.x;
+}
+
+inline static __device__ uint32_t bf16_min_to_high_half(uint32_t val, float x) {
+  phi::dtype::bfloat16 high_half;
+  // The bfloat16 in higher 16bits
+  high_half.x = static_cast<uint16_t>(val >> 16);
+  high_half =
+      static_cast<phi::dtype::bfloat16>(min(static_cast<float>(high_half), x));
+  return (val & 0xFFFFu) | (static_cast<uint32_t>(high_half.x) << 16);
+}
+
+CUDA_ATOMIC_WRAPPER(Min, phi::dtype::bfloat16) {
+  if (*address <= val) {
+    return *address;
+  }
+  uint32_t *address_as_ui = reinterpret_cast<uint32_t *>(
+      reinterpret_cast<char *>(address) -
+      (reinterpret_cast<uintptr_t>(address) & 0x02));
+  float val_f = static_cast<float>(val);
+  uint32_t old = *address_as_ui;
+  uint32_t assumed;
+  if (((uintptr_t)address & 0x02) == 0) {
+    // The bfloat16 value stay at lower 16 bits of the address.
+    do {
+      assumed = old;
+      old = atomicCAS(
+          address_as_ui, assumed, bf16_min_to_low_half(assumed, val_f));
+    } while (old != assumed);
+    phi::dtype::bfloat16 ret;
+    ret.x = old & 0xFFFFu;
+    return ret;
+  } else {
+    // The bfloat16 value stay at higher 16 bits of the address.
+    do {
+      assumed = old;
+      old = atomicCAS(
+          address_as_ui, assumed, bf16_min_to_high_half(assumed, val_f));
+    } while (old != assumed);
+    phi::dtype::bfloat16 ret;
+    ret.x = old >> 16;
+    return ret;
+  }
+}
+
 #ifdef PADDLE_WITH_CUDA
 /*
  * One thead block deals with elementwise atomicAdd for vector of len.
diff --git a/paddle/phi/kernels/funcs/segment_pooling.cu b/paddle/phi/kernels/funcs/segment_pooling.cu
index 2624b5850e1b2..0b6df55bdeff1 100644
--- a/paddle/phi/kernels/funcs/segment_pooling.cu
+++ b/paddle/phi/kernels/funcs/segment_pooling.cu
@@ -451,6 +451,8 @@ template class SegmentPoolFunctor<GPU, int64_t, int>;
 template class SegmentPoolFunctor<GPU, int64_t, int64_t>;
 template class SegmentPoolFunctor<GPU, float16, int>;
 template class SegmentPoolFunctor<GPU, float16, int64_t>;
+template class SegmentPoolFunctor<GPU, phi::dtype::bfloat16, int>;
+template class SegmentPoolFunctor<GPU, phi::dtype::bfloat16, int64_t>;
 
 template class SegmentPoolGradFunctor<GPU, float, int>;
 template class SegmentPoolGradFunctor<GPU, float, int64_t>;
@@ -462,6 +464,8 @@ template class SegmentPoolGradFunctor<GPU, int64_t, int>;
 template class SegmentPoolGradFunctor<GPU, int64_t, int64_t>;
 template class SegmentPoolGradFunctor<GPU, float16, int>;
 template class SegmentPoolGradFunctor<GPU, float16, int64_t>;
+template class SegmentPoolGradFunctor<GPU, phi::dtype::bfloat16, int>;
+template class SegmentPoolGradFunctor<GPU, phi::dtype::bfloat16, int64_t>;
 
 }  // namespace funcs
 }  // namespace phi
diff --git a/paddle/phi/kernels/gpu/segment_pool_grad_kernel.cu b/paddle/phi/kernels/gpu/segment_pool_grad_kernel.cu
index 86800ed840528..0b73580d5c94b 100644
--- a/paddle/phi/kernels/gpu/segment_pool_grad_kernel.cu
+++ b/paddle/phi/kernels/gpu/segment_pool_grad_kernel.cu
@@ -27,4 +27,5 @@ PD_REGISTER_KERNEL(segment_pool_grad,
                    double,
                    int,
                    int64_t,
-                   phi::dtype::float16) {}
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/paddle/phi/kernels/gpu/segment_pool_kernel.cu b/paddle/phi/kernels/gpu/segment_pool_kernel.cu
index 9da2ef519a42b..526c46e32496c 100644
--- a/paddle/phi/kernels/gpu/segment_pool_kernel.cu
+++ b/paddle/phi/kernels/gpu/segment_pool_kernel.cu
@@ -27,4 +27,5 @@ PD_REGISTER_KERNEL(segment_pool,
                    double,
                    int,
                    int64_t,
-                   phi::dtype::float16) {}
+                   phi::dtype::float16,
+                   phi::dtype::bfloat16) {}
diff --git a/python/paddle/fluid/tests/unittests/test_segment_ops.py b/python/paddle/fluid/tests/unittests/test_segment_ops.py
index 763b814f4b143..ab71a515c4a33 100644
--- a/python/paddle/fluid/tests/unittests/test_segment_ops.py
+++ b/python/paddle/fluid/tests/unittests/test_segment_ops.py
@@ -15,7 +15,7 @@
 import unittest
 
 import numpy as np
-from eager_op_test import OpTest
+from eager_op_test import OpTest, convert_float_to_uint16
 
 import paddle
 from paddle.fluid import core
@@ -84,7 +84,10 @@ def segment_pool_split(X, SegmentIds, pooltype):
 
 class TestSegmentOps(OpTest):
     def set_data(self):
-        x = np.random.uniform(-1, 1, self.shape).astype(self.dtype)
+        if self.dtype == np.uint16:
+            x = np.random.uniform(-1, 1, self.shape).astype(self.np_dtype)
+        else:
+            x = np.random.uniform(-1, 1, self.shape).astype(self.dtype)
         segment_ids = self.set_segment(len(x), len(x) // 5 + 1)
         return x, segment_ids
 
@@ -110,10 +113,14 @@ def setUp(self):
         x, segment_ids = self.set_data()
         result = self.compute(x, segment_ids)
         self.inputs = {
-            'X': x.astype(self.dtype),
+            'X': x,
             'SegmentIds': segment_ids.astype(np.int64),
         }
-        self.outputs = {'Out': result.astype(self.dtype)}
+        if self.dtype == np.uint16:
+            self.outputs = {'Out': result.astype(self.np_dtype)}
+        else:
+            self.outputs = {'Out': result.astype(self.dtype)}
+        self.convert_bf16()
 
     def test_check_output(self):
         self.check_output()
@@ -121,6 +128,12 @@ def test_check_output(self):
     def test_check_grad(self):
         self.check_grad(["X"], "Out")
 
+    def convert_bf16(self):
+        if self.dtype == np.uint16:
+            self.inputs['X'] = convert_float_to_uint16(self.inputs['X'])
+            self.outputs['Out'] = convert_float_to_uint16(self.outputs['Out'])
+            self.place = core.CUDAPlace(0)
+
 
 class TestSegmentSum2(TestSegmentOps):
     def prepare(self):
@@ -141,23 +154,16 @@ def setUp(self):
 
 class TestSegmentMax(TestSegmentOps):
     def compute(self, x, segment_ids):
-        return compute_segment_min_max(x, segment_ids, pooltype="MAX")
+        result, self.gradient = compute_segment_min_max(
+            x, segment_ids, pooltype="MAX"
+        )
+        return result
 
     def prepare(self):
         super().prepare()
         self.shape = [40, 20]
         self.attrs = {'pooltype': "MAX"}
 
-    def setUp(self):
-        self.prepare()
-        x, segment_ids = self.set_data()
-        result, self.gradient = self.compute(x, segment_ids)
-        self.inputs = {
-            'X': x.astype(self.dtype),
-            'SegmentIds': segment_ids.astype(np.int32),
-        }
-        self.outputs = {'Out': result.astype(self.dtype)}
-
     def test_check_grad(self):
         self.check_grad(["X"], "Out", user_defined_grads=[self.gradient])
 
@@ -170,7 +176,10 @@ def prepare(self):
 
 class TestSegmentMin(TestSegmentMax):
     def compute(self, x, segment_ids):
-        return compute_segment_min_max(x, segment_ids, pooltype="MIN")
+        result, self.gradient = compute_segment_min_max(
+            x, segment_ids, pooltype="MIN"
+        )
+        return result
 
     def prepare(self):
         super().prepare()
@@ -197,12 +206,17 @@ def setUp(self):
         x, segment_ids = self.set_data()
         result = self.compute(x, segment_ids)
         self.inputs = {'X': x, 'SegmentIds': segment_ids}
+        if self.dtype == np.uint16:
+            astype = self.np_dtype
+        else:
+            astype = self.dtype
         self.outputs = {
             'Out': result,
             'SummedIds': compute_segment_sum(
-                np.ones([len(x), 1]).astype(self.dtype), segment_ids
+                np.ones([len(x), 1]).astype(astype), segment_ids
             ),
         }
+        self.convert_bf16()
 
 
 class TestSegmentMean2(TestSegmentMean):
@@ -213,6 +227,106 @@ def prepare(self):
         self.attrs = {'pooltype': "MEAN"}
 
 
+class TestSegmentSumFP16Op(TestSegmentOps):
+    def prepare(self):
+        super().prepare()
+        self.dtype = np.float16
+
+
+class TestSegmentMaxFP16Op(TestSegmentMax):
+    def prepare(self):
+        super().prepare()
+        self.dtype = np.float16
+
+
+class TestSegmentMinFP16Op(TestSegmentMin):
+    def prepare(self):
+        super().prepare()
+        self.dtype = np.float16
+
+
+class TestSegmentMeanFP16Op(TestSegmentMean):
+    def prepare(self):
+        super().prepare()
+        self.dtype = np.float16
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support bfloat16",
+)
+class TestSegmentSumBF16Op(TestSegmentOps):
+    def prepare(self):
+        super().prepare()
+        self.dtype = np.uint16
+        self.np_dtype = np.float32
+
+    def test_check_output(self):
+        self.check_output_with_place(self.place)
+
+    def test_check_grad(self):
+        self.check_grad_with_place(self.place, ["X"], "Out")
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support bfloat16",
+)
+class TestSegmentMaxBF16Op(TestSegmentMax):
+    def prepare(self):
+        super().prepare()
+        self.dtype = np.uint16
+        self.np_dtype = np.float32
+
+    def test_check_output(self):
+        self.check_output_with_place(self.place)
+
+    def test_check_grad(self):
+        self.check_grad_with_place(
+            self.place, ["X"], "Out", user_defined_grads=[self.gradient]
+        )
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support bfloat16",
+)
+class TestSegmentMinBF16Op(TestSegmentMin):
+    def prepare(self):
+        super().prepare()
+        self.dtype = np.uint16
+        self.np_dtype = np.float32
+
+    def test_check_output(self):
+        self.check_output_with_place(self.place)
+
+    def test_check_grad(self):
+        self.check_grad_with_place(
+            self.place, ["X"], "Out", user_defined_grads=[self.gradient]
+        )
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support bfloat16",
+)
+class TestSegmentMeanBF16Op(TestSegmentMean):
+    def prepare(self):
+        super().prepare()
+        self.dtype = np.uint16
+        self.np_dtype = np.float32
+
+    def test_check_output(self):
+        self.check_output_with_place(self.place)
+
+    def test_check_grad(self):
+        self.check_grad_with_place(self.place, ["X"], "Out")
+
+
 class API_SegmentOpsTest(unittest.TestCase):
     def test_static(self):
         with paddle.static.program_guard(paddle.static.Program()):
diff --git a/python/paddle/geometric/math.py b/python/paddle/geometric/math.py
index fdec045ec1317..9e07e1c77a7c0 100644
--- a/python/paddle/geometric/math.py
+++ b/python/paddle/geometric/math.py
@@ -56,7 +56,7 @@ def segment_sum(data, segment_ids, name=None):
         check_variable_and_dtype(
             data,
             "X",
-            ("float32", "float64", "int32", "int64", "float16"),
+            ("float32", "float64", "int32", "int64", "float16", "uint16"),
             "segment_pool",
         )
         check_variable_and_dtype(
@@ -114,7 +114,7 @@ def segment_mean(data, segment_ids, name=None):
         check_variable_and_dtype(
             data,
             "X",
-            ("float32", "float64", "int32", "int64", "float16"),
+            ("float32", "float64", "int32", "int64", "float16", "uint16"),
             "segment_pool",
         )
         check_variable_and_dtype(
@@ -170,7 +170,7 @@ def segment_min(data, segment_ids, name=None):
         check_variable_and_dtype(
             data,
             "X",
-            ("float32", "float64", "int32", "int64", "float16"),
+            ("float32", "float64", "int32", "int64", "float16", "uint16"),
             "segment_pool",
         )
         check_variable_and_dtype(
@@ -226,7 +226,7 @@ def segment_max(data, segment_ids, name=None):
         check_variable_and_dtype(
             data,
             "X",
-            ("float32", "float64", "int32", "int64", "float16"),
+            ("float32", "float64", "int32", "int64", "float16", "uint16"),
             "segment_pool",
         )
         check_variable_and_dtype(

From 26da689dead90f2333ee7d5f9511fc7011ecfc1e Mon Sep 17 00:00:00 2001
From: huangjiyi <43315610+huangjiyi@users.noreply.github.com>
Date: Thu, 18 May 2023 14:33:22 +0800
Subject: [PATCH 400/405] move fusion_group kernel to phi (#53781)

---
 .../framework/ir/fusion_group/CMakeLists.txt  |   4 +-
 .../ir/fusion_group/code_generator_tester.cc  |   4 +-
 .../ir/fusion_group/fusion_group_pass.cc      |  18 ++-
 paddle/fluid/operators/fused/CMakeLists.txt   |   2 +-
 .../fluid/operators/fused/fusion_group_op.cc  |   2 +-
 .../fluid/operators/fused/fusion_group_op.h   |  99 -----------------
 paddle/fluid/platform/CMakeLists.txt          |   6 +-
 paddle/fluid/platform/device_code_test.cc     |  44 ++++----
 paddle/phi/backends/CMakeLists.txt            |   4 +
 .../platform => phi/backends}/device_code.cc  |  58 +++++-----
 .../platform => phi/backends}/device_code.h   |  34 +++---
 paddle/phi/kernels/CMakeLists.txt             |   5 +
 .../kernels/fusion/gpu/fusion_group_kernel.cu | 104 ++++++++++++++++++
 .../ops/compat/fusion_group_sig.cc}           |  26 +++--
 test/cpp/fluid/fused/fusion_group_op_test.cc  |  11 +-
 15 files changed, 216 insertions(+), 205 deletions(-)
 delete mode 100644 paddle/fluid/operators/fused/fusion_group_op.h
 rename paddle/{fluid/platform => phi/backends}/device_code.cc (91%)
 rename paddle/{fluid/platform => phi/backends}/device_code.h (77%)
 create mode 100644 paddle/phi/kernels/fusion/gpu/fusion_group_kernel.cu
 rename paddle/{fluid/operators/fused/fusion_group_op.cu.cc => phi/ops/compat/fusion_group_sig.cc} (51%)

diff --git a/paddle/fluid/framework/ir/fusion_group/CMakeLists.txt b/paddle/fluid/framework/ir/fusion_group/CMakeLists.txt
index 7df678fbdd7e3..5e05108b666f2 100644
--- a/paddle/fluid/framework/ir/fusion_group/CMakeLists.txt
+++ b/paddle/fluid/framework/ir/fusion_group/CMakeLists.txt
@@ -6,13 +6,13 @@ if(WITH_GPU OR WITH_ROCM)
   cc_test(
     test_code_generator
     SRCS code_generator_tester.cc
-    DEPS code_generator device_code lod_tensor graph_viz_pass)
+    DEPS code_generator phi_backends lod_tensor graph_viz_pass)
 endif()
 
 cc_library(
   fusion_group_pass
   SRCS fusion_group_pass.cc elementwise_group_detector.cc
-  DEPS subgraph_detector fuse_pass_base code_generator device_code)
+  DEPS subgraph_detector fuse_pass_base code_generator phi_backends)
 cc_test(
   test_fusion_group_pass
   SRCS fusion_group_pass_tester.cc
diff --git a/paddle/fluid/framework/ir/fusion_group/code_generator_tester.cc b/paddle/fluid/framework/ir/fusion_group/code_generator_tester.cc
index 762922ba88129..06593733e6a27 100644
--- a/paddle/fluid/framework/ir/fusion_group/code_generator_tester.cc
+++ b/paddle/fluid/framework/ir/fusion_group/code_generator_tester.cc
@@ -20,8 +20,8 @@ limitations under the License. */
 #include "paddle/fluid/framework/ir/fusion_group/code_generator.h"
 #include "paddle/fluid/framework/ir/fusion_group/operation.h"
 #include "paddle/fluid/framework/ir/pass_tester_helper.h"
-#include "paddle/fluid/platform/device_code.h"
 #include "paddle/fluid/platform/float16.h"
+#include "paddle/phi/backends/device_code.h"
 
 namespace phi {
 class DenseTensor;
@@ -182,7 +182,7 @@ void TestMainImpl(std::string func_name,
                     std::type_index(typeid(paddle::platform::float16));
 
   paddle::platform::CUDAPlace place = paddle::platform::CUDAPlace(0);
-  paddle::platform::CUDADeviceCode device_code(place, func_name, code_str);
+  phi::GPUDeviceCode device_code(place, func_name, code_str);
 #ifdef PADDLE_WITH_HIP
   device_code.Compile(true);
 #else
diff --git a/paddle/fluid/framework/ir/fusion_group/fusion_group_pass.cc b/paddle/fluid/framework/ir/fusion_group/fusion_group_pass.cc
index 3ed66b576d883..1f8f3b63598ce 100644
--- a/paddle/fluid/framework/ir/fusion_group/fusion_group_pass.cc
+++ b/paddle/fluid/framework/ir/fusion_group/fusion_group_pass.cc
@@ -19,12 +19,10 @@ limitations under the License. */
 #include "paddle/fluid/framework/ir/graph_pattern_detector.h"
 #include "paddle/fluid/framework/ir/pass_tester_helper.h"
 #include "paddle/fluid/framework/op_proto_maker.h"
-#include "paddle/fluid/platform/device_code.h"
-namespace paddle {
-namespace platform {
+#include "paddle/phi/backends/device_code.h"
+namespace phi {
 class DeviceCodePool;
-}  // namespace platform
-}  // namespace paddle
+}  // namespace phi
 
 namespace paddle {
 namespace framework {
@@ -36,7 +34,7 @@ void FusionGroupPass::ApplyImpl(ir::Graph* graph) const {
   FusePassBase::Init("fusion_group_pass", graph);
   if (Get<bool>("use_gpu")) {
     // TODO(liuyiqun): open this check.
-    // if (!platform::CUDADeviceCode::IsAvailable()) {
+    // if (!phi::GPUDeviceCode::IsAvailable()) {
     //   LOG(WARNING)
     //       << "Disable fusion_group because CUDA Driver or NVRTC is not
     //       avaiable.";
@@ -54,7 +52,7 @@ void FusionGroupPass::ApplyImpl(ir::Graph* graph) const {
 int FusionGroupPass::DetectFusionGroup(Graph* graph, int type) const {
   // TODO(liuyiqun): supported different places
   platform::CUDAPlace place = platform::CUDAPlace(0);
-  int index = platform::DeviceCodePool::Init({place}).size(place);
+  int index = phi::DeviceCodePool::Init({place}).size(place);
 
   std::vector<std::vector<Node*>> subgraphs =
       fusion_group::ElementwiseGroupDetector()(graph);
@@ -88,11 +86,11 @@ bool FusionGroupPass::GenerateCode(fusion_group::SubGraph* subgraph) const {
 
   // TODO(liuyiqun): supported different places
   platform::CUDAPlace place = platform::CUDAPlace(0);
-  std::unique_ptr<platform::CUDADeviceCode> device_code(
-      new platform::CUDADeviceCode(place, subgraph->GetFuncName(), code_str));
+  std::unique_ptr<phi::GPUDeviceCode> device_code(
+      new phi::GPUDeviceCode(place, subgraph->GetFuncName(), code_str));
   bool is_compiled = device_code->Compile();
   if (is_compiled) {
-    platform::DeviceCodePool& pool = platform::DeviceCodePool::Init({place});
+    phi::DeviceCodePool& pool = phi::DeviceCodePool::Init({place});
     pool.Set(std::move(device_code));
   }
   return is_compiled;
diff --git a/paddle/fluid/operators/fused/CMakeLists.txt b/paddle/fluid/operators/fused/CMakeLists.txt
index 261d1905246c8..ebb6e747f16be 100755
--- a/paddle/fluid/operators/fused/CMakeLists.txt
+++ b/paddle/fluid/operators/fused/CMakeLists.txt
@@ -73,7 +73,7 @@ if(WITH_GPU OR WITH_ROCM)
   op_library(fused_gate_attention_op)
   # fusion_group
   if(NOT APPLE AND NOT WIN32)
-    op_library(fusion_group_op DEPS device_code)
+    op_library(fusion_group_op)
   endif()
   # fused_bn_add_activation
   # HIP not support bn act fuse in MIOPEN
diff --git a/paddle/fluid/operators/fused/fusion_group_op.cc b/paddle/fluid/operators/fused/fusion_group_op.cc
index 362819d97ff95..48a82f2ac50ab 100644
--- a/paddle/fluid/operators/fused/fusion_group_op.cc
+++ b/paddle/fluid/operators/fused/fusion_group_op.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/operators/fused/fusion_group_op.h"
+#include "paddle/fluid/framework/op_registry.h"
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/fused/fusion_group_op.h b/paddle/fluid/operators/fused/fusion_group_op.h
deleted file mode 100644
index 657648874399a..0000000000000
--- a/paddle/fluid/operators/fused/fusion_group_op.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#pragma once
-
-#include <string>
-#include <vector>
-
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/platform/device_code.h"
-
-namespace paddle {
-namespace operators {
-
-template <typename DeviceContext>
-static void MutableMultiTypeData(std::vector<phi::DenseTensor*>* var,
-                                 const std::vector<int>& data_type,
-                                 const DeviceContext& dev_ctx,
-                                 const platform::Place& place) {
-  for (size_t i = 0; i < var->size(); i++) {
-    if (data_type[i] == framework::proto::VarType::FP32) {
-      dev_ctx.template Alloc<float>((*var)[i],
-                                    (*var)[i]->numel() * sizeof(float));
-    } else if (data_type[i] == framework::proto::VarType::FP16) {
-      dev_ctx.template Alloc<paddle::platform::float16>(
-          (*var)[i], (*var)[i]->numel() * sizeof(paddle::platform::float16));
-    } else if (data_type[i] == framework::proto::VarType::FP64) {
-      dev_ctx.template Alloc<double>((*var)[i],
-                                     (*var)[i]->numel() * sizeof(double));
-    }
-  }
-}
-
-template <typename T, typename DeviceContext>
-class FusionGroupKernel : public framework::OpKernel<T> {
- public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
-    auto ins = ctx.MultiInput<phi::DenseTensor>("Inputs");
-    auto outs = ctx.MultiOutput<phi::DenseTensor>("Outs");
-    int type = ctx.Attr<int>("type");
-    const auto& outs_dtype = ctx.Attr<std::vector<int>>("outs_dtype");
-    const auto& inputs_dtype = ctx.Attr<std::vector<int>>("inputs_dtype");
-
-    size_t num_ins = ins.size();
-    size_t num_outs = outs.size();
-
-    auto place = ctx.GetPlace();
-    auto& dev_ctx = ctx.template device_context<DeviceContext>();
-
-    MutableMultiTypeData(&outs, outs_dtype, dev_ctx, place);
-
-    std::string func_name = ctx.Attr<std::string>("func_name");
-    platform::DeviceCode* dev_code =
-        platform::DeviceCodePool::Instance().Get(place, func_name);
-    VLOG(3) << "func_name: " << func_name;
-
-    if (type == 0) {
-      size_t n = ins[0]->numel();
-      std::vector<void*> args;
-      args.push_back(&n);
-      std::vector<const void*> ptrs(num_ins + num_outs);
-      for (size_t i = 0; i < num_ins; ++i) {
-        if (inputs_dtype[i] == framework::proto::VarType::FP16) {
-          ptrs[i] = ins[i]->data<paddle::platform::float16>();
-        } else if (inputs_dtype[i] == framework::proto::VarType::FP32) {
-          ptrs[i] = ins[i]->data<float>();
-        } else if (inputs_dtype[i] == framework::proto::VarType::FP64) {
-          ptrs[i] = ins[i]->data<double>();
-        }
-        args.push_back(&ptrs[i]);
-      }
-      for (size_t j = 0; j < num_outs; ++j) {
-        if (outs_dtype[j] == framework::proto::VarType::FP16) {
-          ptrs[num_ins + j] = outs[j]->data<paddle::platform::float16>();
-        } else if (outs_dtype[j] == framework::proto::VarType::FP32) {
-          ptrs[num_ins + j] = outs[j]->data<float>();
-        } else if (outs_dtype[j] == framework::proto::VarType::FP64) {
-          ptrs[num_ins + j] = outs[j]->data<double>();
-        }
-        args.push_back(&ptrs[num_ins + j]);
-      }
-      dev_code->Launch(n, &args);
-    }
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
diff --git a/paddle/fluid/platform/CMakeLists.txt b/paddle/fluid/platform/CMakeLists.txt
index ffebe1012b35f..0907b05622d81 100644
--- a/paddle/fluid/platform/CMakeLists.txt
+++ b/paddle/fluid/platform/CMakeLists.txt
@@ -356,15 +356,11 @@ if(WITH_ROCM)
 endif()
 
 if(NOT APPLE AND NOT WIN32)
-  cc_library(
-    device_code
-    SRCS device_code.cc
-    DEPS device_context)
   if(WITH_GPU OR WITH_ROCM)
     cc_test(
       device_code_test
       SRCS device_code_test.cc
-      DEPS device_code lod_tensor)
+      DEPS phi_backends lod_tensor)
   endif()
 endif()
 
diff --git a/paddle/fluid/platform/device_code_test.cc b/paddle/fluid/platform/device_code_test.cc
index c4ac44603dd2f..6b58453f03ea8 100644
--- a/paddle/fluid/platform/device_code_test.cc
+++ b/paddle/fluid/platform/device_code_test.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/platform/device_code.h"
+#include "paddle/phi/backends/device_code.h"
 
 #include <utility>
 
@@ -47,14 +47,13 @@ void saxpy_kernel(float a, float *x, float* y, float* z, size_t n) {
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 TEST(DeviceCode, cuda) {
-  if (!paddle::platform::dynload::HasNVRTC() ||
-      !paddle::platform::dynload::HasCUDADriver()) {
+  if (!phi::dynload::HasNVRTC() || !phi::dynload::HasCUDADriver()) {
     return;
   }
 
   paddle::framework::InitDevices({0});
-  paddle::platform::CUDAPlace place = paddle::platform::CUDAPlace(0);
-  paddle::platform::CUDADeviceCode code(place, "saxpy_kernel", saxpy_code);
+  phi::GPUPlace place = phi::GPUPlace(0);
+  phi::GPUDeviceCode code(place, "saxpy_kernel", saxpy_code);
 
   phi::DenseTensor cpu_x;
   phi::DenseTensor cpu_y;
@@ -63,8 +62,12 @@ TEST(DeviceCode, cuda) {
   float scale = 2;
   auto dims =
       phi::make_ddim({static_cast<int64_t>(256), static_cast<int64_t>(1024)});
-  cpu_x.mutable_data<float>(dims, paddle::platform::CPUPlace());
-  cpu_y.mutable_data<float>(dims, paddle::platform::CPUPlace());
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* cpu_ctx = reinterpret_cast<phi::CPUContext*>(pool.Get(phi::CPUPlace()));
+  cpu_x.Resize(dims);
+  cpu_ctx->template Alloc<float>(&cpu_x);
+  cpu_y.Resize(dims);
+  cpu_ctx->template Alloc<float>(&cpu_y);
 
   size_t n = cpu_x.numel();
   for (size_t i = 0; i < n; ++i) {
@@ -78,9 +81,13 @@ TEST(DeviceCode, cuda) {
   phi::DenseTensor y;
   phi::DenseTensor z;
 
-  float* x_data = x.mutable_data<float>(dims, place);
-  float* y_data = y.mutable_data<float>(dims, place);
-  float* z_data = z.mutable_data<float>(dims, place);
+  auto* dev_ctx = reinterpret_cast<phi::GPUContext*>(pool.Get(place));
+  x.Resize(dims);
+  float* x_data = dev_ctx->template Alloc<float>(&x);
+  y.Resize(dims);
+  float* y_data = dev_ctx->template Alloc<float>(&y);
+  z.Resize(dims);
+  float* z_data = dev_ctx->template Alloc<float>(&z);
 
   paddle::framework::TensorCopySync(cpu_x, place, &x);
   paddle::framework::TensorCopySync(cpu_y, place, &y);
@@ -92,36 +99,33 @@ TEST(DeviceCode, cuda) {
   code.SetWorkloadPerThread(1);
   code.Launch(n, &args);
 
-  auto* dev_ctx = paddle::platform::DeviceContextPool::Instance().Get(place);
   dev_ctx->Wait();
 
-  paddle::framework::TensorCopySync(z, paddle::platform::CPUPlace(), &cpu_z);
+  paddle::framework::TensorCopySync(z, phi::CPUPlace(), &cpu_z);
   for (size_t i = 0; i < n; i++) {
     EXPECT_EQ(cpu_z.data<float>()[i], static_cast<float>(i) * scale + 0.5);
   }
 }
 
 TEST(DeviceCodePool, cuda) {
-  if (!paddle::platform::dynload::HasNVRTC() ||
-      !paddle::platform::dynload::HasCUDADriver()) {
+  if (!phi::dynload::HasNVRTC() || !phi::dynload::HasCUDADriver()) {
     return;
   }
 
   paddle::framework::InitDevices({0});
-  paddle::platform::CUDAPlace place = paddle::platform::CUDAPlace(0);
-  paddle::platform::DeviceCodePool& pool =
-      paddle::platform::DeviceCodePool::Init({place});
+  phi::GPUPlace place = phi::GPUPlace(0);
+  phi::DeviceCodePool& pool = phi::DeviceCodePool::Init({place});
   size_t num_device_codes_before = pool.size(place);
   EXPECT_EQ(num_device_codes_before, 0UL);
 
-  std::unique_ptr<paddle::platform::DeviceCode> code(
-      new paddle::platform::CUDADeviceCode(place, "saxpy_kernel", saxpy_code));
+  std::unique_ptr<phi::DeviceCode> code(
+      new phi::GPUDeviceCode(place, "saxpy_kernel", saxpy_code));
   LOG(INFO) << "origin ptr: " << code.get();
   pool.Set(std::move(code));
   size_t num_device_codes_after = pool.size(place);
   EXPECT_EQ(num_device_codes_after, 1UL);
 
-  paddle::platform::DeviceCode* code_get = pool.Get(place, "saxpy_kernel");
+  phi::DeviceCode* code_get = pool.Get(place, "saxpy_kernel");
   LOG(INFO) << "get ptr: " << code_get;
 }
 #endif
diff --git a/paddle/phi/backends/CMakeLists.txt b/paddle/phi/backends/CMakeLists.txt
index 54888cf8932af..bb4ccd9a46ed7 100644
--- a/paddle/phi/backends/CMakeLists.txt
+++ b/paddle/phi/backends/CMakeLists.txt
@@ -14,6 +14,10 @@ if(WITH_XBYAK)
   list(APPEND BACKENDS_DEPS xbyak)
 endif()
 
+if(NOT APPLE AND NOT WIN32)
+  list(APPEND BACKENDS_SRCS device_code.cc)
+endif()
+
 if(WITH_GPU OR WITH_ROCM)
   list(APPEND BACKENDS_SRCS gpu/gpu_context.cc gpu/gpu_info.cc
        gpu/gpu_resources.cc)
diff --git a/paddle/fluid/platform/device_code.cc b/paddle/phi/backends/device_code.cc
similarity index 91%
rename from paddle/fluid/platform/device_code.cc
rename to paddle/phi/backends/device_code.cc
index 2a410182b48a4..eb2934d1b4842 100644
--- a/paddle/fluid/platform/device_code.cc
+++ b/paddle/phi/backends/device_code.cc
@@ -12,20 +12,22 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/platform/device_code.h"
+#include "paddle/phi/backends/device_code.h"
 
+#include <glog/logging.h>
 #include <sys/stat.h>
 
 #include <algorithm>
 #include <set>
 #include <utility>
 
-#include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/flags.h"
 
 PHI_DECLARE_string(cuda_dir);
 
-namespace paddle {
-namespace platform {
+namespace phi {
 
 DeviceCodePool* DeviceCodePool::pool = nullptr;
 
@@ -35,7 +37,7 @@ void DeviceCodePool::Set(std::unique_ptr<DeviceCode>&& code) {
 
   auto iter = device_codes_.find(place);
   if (iter == device_codes_.end()) {
-    PADDLE_THROW(platform::errors::NotFound(
+    PADDLE_THROW(phi::errors::NotFound(
         "Place %s is not supported for runtime compiling.", place));
   }
 
@@ -43,18 +45,18 @@ void DeviceCodePool::Set(std::unique_ptr<DeviceCode>&& code) {
   codes_map.emplace(name, std::move(code));
 }
 
-platform::DeviceCode* DeviceCodePool::Get(const platform::Place& place,
-                                          const std::string& name) {
+DeviceCode* DeviceCodePool::Get(const phi::Place& place,
+                                const std::string& name) {
   auto iter = device_codes_.find(place);
   if (iter == device_codes_.end()) {
-    PADDLE_THROW(platform::errors::NotFound(
+    PADDLE_THROW(phi::errors::NotFound(
         "Place %s is not supported for runtime compiling.", place));
   }
 
   auto& codes_map = iter->second;
   auto code_iter = codes_map.find(name);
   if (code_iter == codes_map.end()) {
-    PADDLE_THROW(platform::errors::NotFound(
+    PADDLE_THROW(phi::errors::NotFound(
         "Device code named %s for place %s does not exist.",
         name.c_str(),
         place));
@@ -63,7 +65,7 @@ platform::DeviceCode* DeviceCodePool::Get(const platform::Place& place,
   return code_iter->second.get();
 }
 
-DeviceCodePool::DeviceCodePool(const std::vector<platform::Place>& places) {
+DeviceCodePool::DeviceCodePool(const std::vector<phi::Place>& places) {
   PADDLE_ENFORCE_GT(places.size(),
                     0,
                     errors::InvalidArgument(
@@ -75,11 +77,11 @@ DeviceCodePool::DeviceCodePool(const std::vector<platform::Place>& places) {
     set.insert(p);
   }
   for (auto& p : set) {
-    if (is_gpu_place(p)) {
+    if (p.GetType() == phi::AllocationType::GPU) {
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
       device_codes_.emplace(p, DeviceCodeMap());
 #else
-      PADDLE_THROW(platform::errors::PreconditionNotMet(
+      PADDLE_THROW(phi::errors::PreconditionNotMet(
           "CUDAPlace or HIPPlace is not supported, please re-compile with "
           "WITH_GPU=ON or WITH_ROCM=ON."));
 #endif
@@ -87,7 +89,7 @@ DeviceCodePool::DeviceCodePool(const std::vector<platform::Place>& places) {
   }
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-  CUDADeviceCode::CheckAvailableStatus();
+  GPUDeviceCode::CheckAvailableStatus();
 #endif
 }
 
@@ -114,8 +116,8 @@ static bool CheckCUDADriverResult(CUresult result,
   return true;
 }
 
-bool CUDADeviceCode::available_ = false;
-void CUDADeviceCode::CheckAvailableStatus() {
+bool GPUDeviceCode::available_ = false;
+void GPUDeviceCode::CheckAvailableStatus() {
   available_ = false;
   if (!dynload::HasNVRTC() || !dynload::HasCUDADriver()) {
     LOG_FIRST_N(WARNING, 1)
@@ -215,12 +217,12 @@ static std::string FindCUDAIncludePath() {
   return "";
 }
 
-CUDADeviceCode::CUDADeviceCode(const Place& place,
-                               const std::string& name,
-                               const std::string& kernel) {
-  if (!is_gpu_place(place)) {
-    PADDLE_THROW(platform::errors::PermissionDenied(
-        "CUDADeviceCode can only launch on GPU place."));
+GPUDeviceCode::GPUDeviceCode(const Place& place,
+                             const std::string& name,
+                             const std::string& kernel) {
+  if (place.GetType() != phi::AllocationType::GPU) {
+    PADDLE_THROW(phi::errors::PermissionDenied(
+        "GPUDeviceCode can only launch on GPU place."));
   }
 
   place_ = place;
@@ -232,7 +234,7 @@ CUDADeviceCode::CUDADeviceCode(const Place& place,
 #endif
 }
 
-bool CUDADeviceCode::Compile(bool include_path) {
+bool GPUDeviceCode::Compile(bool include_path) {
   is_compiled_ = false;
   if (!dynload::HasNVRTC() || !dynload::HasCUDADriver()) {
     LOG_FIRST_N(WARNING, 1)
@@ -403,7 +405,7 @@ bool CUDADeviceCode::Compile(bool include_path) {
   return true;
 }
 
-void CUDADeviceCode::Launch(const size_t n, std::vector<void*>* args) const {
+void GPUDeviceCode::Launch(const size_t n, std::vector<void*>* args) const {
   PADDLE_ENFORCE_EQ(
       is_compiled_,
       true,
@@ -454,8 +456,8 @@ void CUDADeviceCode::Launch(const size_t n, std::vector<void*>* args) const {
 }
 
 #ifdef PADDLE_WITH_HIP
-bool CUDADeviceCode::CheckNVRTCResult(hiprtcResult result,
-                                      std::string function) {
+bool GPUDeviceCode::CheckNVRTCResult(hiprtcResult result,
+                                     std::string function) {
   if (result != HIPRTC_SUCCESS) {
     LOG_FIRST_N(WARNING, 1)
         << "Call " << function << " for < " << name_
@@ -463,8 +465,7 @@ bool CUDADeviceCode::CheckNVRTCResult(hiprtcResult result,
     return false;
   }
 #else
-bool CUDADeviceCode::CheckNVRTCResult(nvrtcResult result,
-                                      std::string function) {
+bool GPUDeviceCode::CheckNVRTCResult(nvrtcResult result, std::string function) {
   if (result != NVRTC_SUCCESS) {
     LOG_FIRST_N(WARNING, 1)
         << "Call " << function << " for < " << name_
@@ -476,5 +477,4 @@ bool CUDADeviceCode::CheckNVRTCResult(nvrtcResult result,
 }
 #endif
 
-}  // namespace platform
-}  // namespace paddle
+}  // namespace phi
diff --git a/paddle/fluid/platform/device_code.h b/paddle/phi/backends/device_code.h
similarity index 77%
rename from paddle/fluid/platform/device_code.h
rename to paddle/phi/backends/device_code.h
index d4928f55c67c5..8debb4dc9c45e 100644
--- a/paddle/fluid/platform/device_code.h
+++ b/paddle/phi/backends/device_code.h
@@ -20,18 +20,18 @@ limitations under the License. */
 #include <unordered_map>
 #include <vector>
 
-#include "paddle/fluid/platform/device_context.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/enforce.h"
 #ifdef PADDLE_WITH_CUDA
-#include "paddle/fluid/platform/dynload/cuda_driver.h"
-#include "paddle/fluid/platform/dynload/nvrtc.h"
+#include "paddle/phi/backends/dynload/cuda_driver.h"
+#include "paddle/phi/backends/dynload/nvrtc.h"
 #endif
 #ifdef PADDLE_WITH_HIP
-#include "paddle/fluid/platform/dynload/hiprtc.h"
-#include "paddle/fluid/platform/dynload/rocm_driver.h"
+#include "paddle/phi/backends/dynload/hiprtc.h"
+#include "paddle/phi/backends/dynload/rocm_driver.h"
 #endif
 
-namespace paddle {
-namespace platform {
+namespace phi {
 
 class DeviceCode {
  public:
@@ -49,11 +49,11 @@ class DeviceCode {
 };
 
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-class CUDADeviceCode : public DeviceCode {
+class GPUDeviceCode : public DeviceCode {
  public:
-  explicit CUDADeviceCode(const Place& place,
-                          const std::string& name,
-                          const std::string& kernel);
+  explicit GPUDeviceCode(const Place& place,
+                         const std::string& name,
+                         const std::string& kernel);
   bool Compile(bool include_path = false) override;
   void Launch(const size_t n, std::vector<void*>* args) const override;
 
@@ -94,7 +94,7 @@ class DeviceCodePool {
   using DeviceCodeMap =
       std::unordered_map<std::string, std::unique_ptr<DeviceCode>>;
 
-  explicit DeviceCodePool(const std::vector<platform::Place>& places);
+  explicit DeviceCodePool(const std::vector<Place>& places);
 
   static DeviceCodePool& Instance() {
     PADDLE_ENFORCE_NOT_NULL(
@@ -104,7 +104,7 @@ class DeviceCodePool {
     return *pool;
   }
 
-  static DeviceCodePool& Init(const std::vector<platform::Place>& places) {
+  static DeviceCodePool& Init(const std::vector<Place>& places) {
     if (pool == nullptr) {
       pool = new DeviceCodePool(places);
     }
@@ -113,10 +113,9 @@ class DeviceCodePool {
 
   void Set(std::unique_ptr<DeviceCode>&& code);
 
-  platform::DeviceCode* Get(const platform::Place& place,
-                            const std::string& name);
+  DeviceCode* Get(const Place& place, const std::string& name);
 
-  size_t size(const platform::Place& place) const {
+  size_t size(const Place& place) const {
     auto iter = device_codes_.find(place);
     if (iter == device_codes_.end()) {
       return 0;
@@ -130,5 +129,4 @@ class DeviceCodePool {
   DISABLE_COPY_AND_ASSIGN(DeviceCodePool);
 };
 
-}  // namespace platform
-}  // namespace paddle
+}  // namespace phi
diff --git a/paddle/phi/kernels/CMakeLists.txt b/paddle/phi/kernels/CMakeLists.txt
index 932091b087c09..8589785a97e85 100644
--- a/paddle/phi/kernels/CMakeLists.txt
+++ b/paddle/phi/kernels/CMakeLists.txt
@@ -153,6 +153,11 @@ if(WITH_CUTLASS)
   list(APPEND kernel_cu ${cutlass_cu})
 endif()
 
+if(APPLE OR WIN32)
+  list(REMOVE_ITEM kernel_cu
+       "${CMAKE_CURRENT_SOURCE_DIR}/fusion/gpu/fusion_group_kernel.cu")
+endif()
+
 if(WITH_MKLDNN)
   file(
     GLOB
diff --git a/paddle/phi/kernels/fusion/gpu/fusion_group_kernel.cu b/paddle/phi/kernels/fusion/gpu/fusion_group_kernel.cu
new file mode 100644
index 0000000000000..eca9bcf18cf0e
--- /dev/null
+++ b/paddle/phi/kernels/fusion/gpu/fusion_group_kernel.cu
@@ -0,0 +1,104 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "glog/logging.h"
+
+#include "paddle/phi/backends/device_code.h"
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/utils/data_type.h"
+
+namespace phi {
+namespace fusion {
+
+template <typename DeviceContext>
+static void MutableMultiTypeData(std::vector<phi::DenseTensor*>* var,
+                                 const std::vector<int>& data_type,
+                                 const DeviceContext& dev_ctx) {
+  for (size_t i = 0; i < var->size(); i++) {
+    if (data_type[i] == phi::TransToProtoVarType(phi::DataType::FLOAT32)) {
+      dev_ctx.template Alloc<float>((*var)[i],
+                                    (*var)[i]->numel() * sizeof(float));
+    } else if (data_type[i] ==
+               phi::TransToProtoVarType(phi::DataType::FLOAT16)) {
+      dev_ctx.template Alloc<phi::dtype::float16>(
+          (*var)[i], (*var)[i]->numel() * sizeof(phi::dtype::float16));
+    } else if (data_type[i] ==
+               phi::TransToProtoVarType(phi::DataType::FLOAT64)) {
+      dev_ctx.template Alloc<double>((*var)[i],
+                                     (*var)[i]->numel() * sizeof(double));
+    }
+  }
+}
+
+template <typename T, typename Context>
+void FusionGroupKernel(const Context& dev_ctx,
+                       const std::vector<const DenseTensor*>& ins,
+                       const std::vector<int>& outs_dtype,
+                       const std::vector<int>& inputs_dtype,
+                       const std::string& func_name,
+                       int type,
+                       std::vector<DenseTensor*> outs) {
+  size_t num_ins = ins.size();
+  size_t num_outs = outs.size();
+
+  MutableMultiTypeData(&outs, outs_dtype, dev_ctx);
+
+  phi::DeviceCode* dev_code =
+      phi::DeviceCodePool::Instance().Get(dev_ctx.GetPlace(), func_name);
+  VLOG(3) << "func_name: " << func_name;
+
+  if (type == 0) {
+    size_t n = ins[0]->numel();
+    std::vector<void*> args;
+    args.push_back(&n);
+    std::vector<const void*> ptrs(num_ins + num_outs);
+    for (size_t i = 0; i < num_ins; ++i) {
+      if (inputs_dtype[i] == phi::TransToProtoVarType(phi::DataType::FLOAT16)) {
+        ptrs[i] = ins[i]->data<phi::dtype::float16>();
+      } else if (inputs_dtype[i] ==
+                 phi::TransToProtoVarType(phi::DataType::FLOAT32)) {
+        ptrs[i] = ins[i]->data<float>();
+      } else if (inputs_dtype[i] ==
+                 phi::TransToProtoVarType(phi::DataType::FLOAT64)) {
+        ptrs[i] = ins[i]->data<double>();
+      }
+      args.push_back(&ptrs[i]);
+    }
+    for (size_t j = 0; j < num_outs; ++j) {
+      if (outs_dtype[j] == phi::TransToProtoVarType(phi::DataType::FLOAT16)) {
+        ptrs[num_ins + j] = outs[j]->data<phi::dtype::float16>();
+      } else if (outs_dtype[j] ==
+                 phi::TransToProtoVarType(phi::DataType::FLOAT32)) {
+        ptrs[num_ins + j] = outs[j]->data<float>();
+      } else if (outs_dtype[j] ==
+                 phi::TransToProtoVarType(phi::DataType::FLOAT64)) {
+        ptrs[num_ins + j] = outs[j]->data<double>();
+      }
+      args.push_back(&ptrs[num_ins + j]);
+    }
+    dev_code->Launch(n, &args);
+  }
+}
+
+}  // namespace fusion
+}  // namespace phi
+
+PD_REGISTER_KERNEL(fusion_group,
+                   GPU,
+                   ALL_LAYOUT,
+                   phi::fusion::FusionGroupKernel,
+                   float,
+                   double,
+                   phi::dtype::float16) {}
diff --git a/paddle/fluid/operators/fused/fusion_group_op.cu.cc b/paddle/phi/ops/compat/fusion_group_sig.cc
similarity index 51%
rename from paddle/fluid/operators/fused/fusion_group_op.cu.cc
rename to paddle/phi/ops/compat/fusion_group_sig.cc
index 8ec5bdd9643f0..666e6f77d218f 100644
--- a/paddle/fluid/operators/fused/fusion_group_op.cu.cc
+++ b/paddle/phi/ops/compat/fusion_group_sig.cc
@@ -1,4 +1,4 @@
-/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -12,16 +12,18 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/operators/fused/fusion_group_op.h"
+#include "paddle/phi/core/compat/op_utils.h"
 
-#include "paddle/fluid/platform/float16.h"
+namespace phi {
 
-namespace ops = paddle::operators;
-namespace plat = paddle::platform;
-PD_REGISTER_STRUCT_KERNEL(fusion_group,
-                          GPU,
-                          ALL_LAYOUT,
-                          ops::FusionGroupKernel,
-                          float,
-                          double,
-                          plat::float16) {}
+KernelSignature FusionGroupOpArgumentMapping(
+    const ArgumentMappingContext& ctx) {
+  return KernelSignature("fusion_group",
+                         {"Inputs"},
+                         {"outs_dtype", "inputs_dtype", "func_name", "type"},
+                         {"Outs"});
+}
+
+}  // namespace phi
+
+PD_REGISTER_ARG_MAPPING_FN(fusion_group, phi::FusionGroupOpArgumentMapping);
diff --git a/test/cpp/fluid/fused/fusion_group_op_test.cc b/test/cpp/fluid/fused/fusion_group_op_test.cc
index e46d6a7c28e95..17128a1add492 100644
--- a/test/cpp/fluid/fused/fusion_group_op_test.cc
+++ b/test/cpp/fluid/fused/fusion_group_op_test.cc
@@ -17,8 +17,8 @@ limitations under the License. */
 #include "paddle/fluid/framework/op_proto_maker.h"
 #include "paddle/fluid/framework/op_registry.h"
 #include "paddle/fluid/framework/program_desc.h"
-#include "paddle/fluid/platform/device_code.h"
 #include "paddle/fluid/platform/init.h"
+#include "paddle/phi/backends/device_code.h"
 
 namespace paddle {
 namespace operators {
@@ -93,11 +93,10 @@ framework::OpDesc* CreateFusionGroupOp(
 void PrepareDeviceCode(platform::Place place,
                        std::string func_name,
                        std::string cuda_kernel_str) {
-  paddle::platform::DeviceCodePool& pool =
-      paddle::platform::DeviceCodePool::Init({place});
+  phi::DeviceCodePool& pool = phi::DeviceCodePool::Init({place});
 
-  std::unique_ptr<paddle::platform::DeviceCode> code(
-      new paddle::platform::CUDADeviceCode(place, func_name, cuda_kernel_str));
+  std::unique_ptr<phi::DeviceCode> code(
+      new phi::GPUDeviceCode(place, func_name, cuda_kernel_str));
   code->Compile();
   pool.Set(std::move(code));
 }
@@ -183,7 +182,7 @@ void TestMain(const std::vector<std::string>& input_names,
 }
 
 TEST(FusionGroupOp, elementwise) {
-  if (!platform::dynload::HasNVRTC() || !platform::dynload::HasCUDADriver()) {
+  if (!phi::dynload::HasNVRTC() || !phi::dynload::HasCUDADriver()) {
     return;
   }
 

From fb4a6ecfd8216f3cd07c9a99cb29e997e57b8197 Mon Sep 17 00:00:00 2001
From: Hulek <jakub.hulek01@gmail.com>
Date: Thu, 18 May 2023 08:56:55 +0200
Subject: [PATCH 401/405] Fused elementwises kernels and ops (#51427)

* Fused elementwises kernels and ops

* change fuse pass name

* adjust .pbtxt files

* adjust quantization attributes

* add missing arguments and fix others, review fixed

* simplify fused kernel registration

* fix elementwise unit tests

* reuse one fused elementwise op

* adjust proto

* Add supported datatypes

* Change 'Scale' to 'scale' in tests, change some tests to onednn

* Revert breaking changes

* Fix unit tests

* Delete obsolete test cases

* Delete commented out code

* Fix codestyle

* delete temporary condition

* fix conflicts and delete duplicate fusing

* Fix code after merge

* Move tests to new directory

* fix tests volatility

* Rename test_elementwise_add_onednn_op.py to test_elementwise_add_mkldnn_op.py

* Update CMakeLists.txt add mkldnn op test

---------

Co-authored-by: Silv3S <slawomir.siwek@intel.com>
---
 paddle/fluid/framework/ir/CMakeLists.txt      |   2 +-
 .../framework/ir/graph_pattern_detector.cc    |   8 +-
 .../framework/ir/mkldnn/cpu_quantize_pass.cc  |  12 +-
 .../ir/mkldnn/cpu_quantize_pass_tester.cc     |  34 ++--
 .../ir/mkldnn/cpu_quantize_placement_pass.cc  |  10 +-
 ...cc => elementwise_act_onednn_fuse_pass.cc} |   7 +-
 ...s.h => elementwise_act_onednn_fuse_pass.h} |   0
 .../framework/ir/mkldnn/mkldnn_pass_util.h    |   2 +
 .../mkldnn/operator_scale_onednn_fuse_pass.cc |   4 +
 .../operator_unsqueeze2_onednn_fuse_pass.cc   |   1 -
 .../inference/api/paddle_pass_builder.cc      |   2 +-
 .../operators/compat/elementwise_add.pbtxt    |   8 -
 .../operators/compat/elementwise_div.pbtxt    |  10 +-
 .../operators/compat/elementwise_mul.pbtxt    |  10 --
 .../operators/compat/elementwise_pow.pbtxt    |  10 +-
 .../operators/compat/elementwise_sub.pbtxt    |  10 +-
 .../compat/fused_elementwise_add.pbtxt        |  50 ++++++
 .../compat/fused_elementwise_sub.pbtxt        |  50 ++++++
 .../elementwise/elementwise_sub_op.cc         |   4 +-
 .../operators/fused/fused_elementwise_op.cc   |  15 ++
 paddle/phi/api/yaml/op_compat.yaml            |  26 +--
 .../fusion/onednn/fused_elementwise_kernel.cc |  20 +++
 .../phi/kernels/onednn/elementwise_kernel.cc  |  29 +---
 .../phi/ops/compat/fused_elementwise_sig.cc   |  36 +++++
 python/paddle/incubate/autograd/primrules.py  |  54 +------
 test/cpp/fluid/feed_forward_test.cu           |  10 +-
 .../test_mkldnn_elt_act_fuse_pass.py          | 152 +++++++++---------
 ...nn_elementwise_add_activation_fuse_pass.py |   6 +-
 .../test_trt_convert_multihead_matmul.py      |   6 +-
 test/mkldnn/CMakeLists.txt                    |   2 +
 test/mkldnn/test_elementwise_add_mkldnn_op.py |  70 ++------
 ...p.py => test_elementwise_mul_onednn_op.py} |  64 ++------
 ...p.py => test_elementwise_sub_onednn_op.py} |  28 ++--
 33 files changed, 367 insertions(+), 385 deletions(-)
 rename paddle/fluid/framework/ir/mkldnn/{elt_act_mkldnn_fuse_pass.cc => elementwise_act_onednn_fuse_pass.cc} (94%)
 rename paddle/fluid/framework/ir/mkldnn/{elt_act_mkldnn_fuse_pass.h => elementwise_act_onednn_fuse_pass.h} (100%)
 create mode 100644 paddle/fluid/operators/compat/fused_elementwise_add.pbtxt
 create mode 100644 paddle/fluid/operators/compat/fused_elementwise_sub.pbtxt
 rename test/mkldnn/{test_elementwise_mul_mkldnn_op.py => test_elementwise_mul_onednn_op.py} (66%)
 rename test/mkldnn/{test_elementwise_sub_mkldnn_op.py => test_elementwise_sub_onednn_op.py} (91%)

diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt
index 8130c91a31373..f116255c17bb9 100755
--- a/paddle/fluid/framework/ir/CMakeLists.txt
+++ b/paddle/fluid/framework/ir/CMakeLists.txt
@@ -180,7 +180,7 @@ if(WITH_MKLDNN)
   pass_library(softplus_activation_onednn_fuse_pass inference DIR mkldnn)
   pass_library(shuffle_channel_mkldnn_detect_pass inference DIR mkldnn)
   pass_library(fc_act_mkldnn_fuse_pass inference DIR mkldnn)
-  pass_library(elt_act_mkldnn_fuse_pass inference DIR mkldnn)
+  pass_library(elementwise_act_onednn_fuse_pass inference DIR mkldnn)
   pass_library(matmul_elementwise_add_mkldnn_fuse_pass inference DIR mkldnn)
   pass_library(matmul_activation_mkldnn_fuse_pass inference DIR mkldnn)
   pass_library(layer_norm_onednn_optimization_pass inference DIR mkldnn)
diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc
index af23d69bd4be3..03f1d5bc40498 100644
--- a/paddle/fluid/framework/ir/graph_pattern_detector.cc
+++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc
@@ -2260,7 +2260,8 @@ PDNode *patterns::OpRequant::operator()() {
   auto any_op = pattern->NewNode(any_op_repr())
                     ->assert_is_op()
                     ->assert_more([&](Node *node) {
-                      return node->Op()->HasAttr("Scale_out") ? true : false;
+                      return (node->Op()->HasAttr("Scale_out") ||
+                              node->Op()->HasAttr("scale_out"));
                     });
   auto requant_in = pattern->NewNode(requant_in_repr())
                         ->assert_is_op_input("requantize", "Input");
@@ -2288,7 +2289,10 @@ PDNode *patterns::RequantOp::operator()() {
                     ->assert_more([&](Node *node) {
                       return (node->Op()->HasAttr("Scale_in") ||
                               node->Op()->HasAttr("Scale_x") ||
-                              node->Op()->HasAttr("Scale_y"));
+                              node->Op()->HasAttr("Scale_y") ||
+                              node->Op()->HasAttr("scale_in") ||
+                              node->Op()->HasAttr("scale_x") ||
+                              node->Op()->HasAttr("scale_y"));
                     });
 
   requant_op->LinksFrom({requant_in}).LinksTo({requant_out});
diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass.cc b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass.cc
index 2c7d93278ab62..7376e87155187 100644
--- a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass.cc
+++ b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass.cc
@@ -1045,14 +1045,14 @@ void CPUQuantizePass::QuantizeElementwise(
                   "X",
                   input_x_scale,
                   is_x_unsigned,
-                  "Scale_x");
+                  "scale_x");
     QuantizeInput(g,
                   elementwise_op,
                   elementwise_y,
                   "Y",
                   input_y_scale,
                   is_y_unsigned,
-                  "Scale_y");
+                  "scale_y");
 
     bool is_output_unsigned{false};
     auto output_scale =
@@ -1064,7 +1064,7 @@ void CPUQuantizePass::QuantizeElementwise(
                      "Out",
                      output_scale,
                      is_output_unsigned,
-                     "Scale_out");
+                     "scale_out");
 
     ++quantize_elementwise_count;
   };
@@ -1314,9 +1314,9 @@ void CPUQuantizePass::ApplyImpl(ir::Graph* graph) const {
   QuantizeImmutable(graph, "nearest_interp", "X");
   QuantizeImmutable(graph, "nearest_interp_v2", "X");
   QuantizeImmutable(graph, "split", "X");
-  QuantizeElementwise(graph, "elementwise_add");
-  QuantizeElementwise(graph, "elementwise_mul");
-  QuantizeElementwise(graph, "elementwise_sub");
+  QuantizeElementwise(graph, "fused_elementwise_add");
+  QuantizeElementwise(graph, "fused_elementwise_mul");
+  QuantizeElementwise(graph, "fused_elementwise_sub");
   QuantizeFusionGru(graph);
   QuantizeMultiGru(graph);
   QuantizeFusionLSTM(graph);
diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass_tester.cc
index 427b4278ceb0a..5ffc2db337ada 100644
--- a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass_tester.cc
+++ b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_pass_tester.cc
@@ -97,14 +97,15 @@ void SetOp(ProgramDesc* prog,
     op->SetAttr("Scale_x", 1.0f);
     op->SetAttr("Scale_y", 1.0f);
     op->SetAttr("Scale_out", 1.0f);
-  } else if (type == "elementwise_add" || type == "elementwise_mul" ||
-             type == "elementwise_sub") {
+  } else if (type == "fused_elementwise_add" ||
+             type == "fused_elementwise_sub" ||
+             type == "fused_elementwise_mul") {
     op->SetInput("X", {inputs[0]});
     if (inputs.size() > 1) op->SetInput("Y", {inputs[1]});
     op->SetOutput("Out", {outputs[0]});
-    op->SetAttr("Scale_x", 1.0f);
-    op->SetAttr("Scale_y", 1.0f);
-    op->SetAttr("Scale_out", 1.0f);
+    op->SetAttr("scale_x", 1.0f);
+    op->SetAttr("scale_y", 1.0f);
+    op->SetAttr("scale_out", 1.0f);
   } else if (type == "fusion_gru") {
     op->SetInput("X", {inputs[0]});
     op->SetInput("Bias", {inputs[1]});
@@ -178,16 +179,19 @@ void CheckScales(const OpDesc* op, float scale, float shift) {
               scale);
     scale_names.push_back("Scale_in");
     scale_names.push_back("Scale_out");
-  } else if (type == "fused_matmul" || type == "elementwise_add" ||
-             type == "elementwise_mul" || type == "elementwise_sub") {
+  } else if (type == "fused_matmul") {
     scale_names.push_back("Scale_x");
     scale_names.push_back("Scale_y");
     scale_names.push_back("Scale_out");
-    if (type == "fused_matmul") {
-      auto const& names = op->InputNames();
-      if (std::find(names.begin(), names.end(), "ResidualData") != names.end())
-        scale_names.push_back("Scale_in_eltwise");
-    }
+    auto const& names = op->InputNames();
+    if (std::find(names.begin(), names.end(), "ResidualData") != names.end())
+      scale_names.push_back("Scale_in_eltwise");
+  } else if (type == "fused_elementwise_add" ||
+             type == "fused_elementwise_sub" ||
+             type == "fused_elementwise_mul") {
+    scale_names.push_back("scale_x");
+    scale_names.push_back("scale_y");
+    scale_names.push_back("scale_out");
   } else if (type == "fusion_gru" || type == "fusion_lstm") {
     EXPECT_EQ(op->GetAttrIfExists<float>("Shift_data"), shift);
     EXPECT_EQ(op->GetAttrIfExists<std::vector<float>>("Scale_weights")[0],
@@ -710,9 +714,9 @@ void TestElementwiseUnsignedAndSignedInput(
 }
 
 const std::vector<std::vector<std::string>> elementwises = {
-    {"elementwise_add", "ElementwiseAdd"},
-    {"elementwise_mul", "ElementwiseMul"},
-    {"elementwise_sub", "ElementwiseSub"}};
+    {"fused_elementwise_add", "FusedElementwiseAdd"},
+    {"fused_elementwise_mul", "FusedElementwiseMul"},
+    {"fused_elementwise_sub", "FusedElementwiseSub"}};
 
 class TestElementwises
     : public testing::TestWithParam<std::vector<std::string>> {};
diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_placement_pass.cc b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_placement_pass.cc
index 0fd1f13463c0d..2071f284126b7 100644
--- a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_placement_pass.cc
+++ b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_placement_pass.cc
@@ -32,6 +32,9 @@ void CPUQuantizePlacementPass::ApplyImpl(ir::Graph* graph) const {
                                        "fused_conv2d",
                                        "fused_conv3d",
                                        "fused_matmul",
+                                       "fused_elementwise_add",
+                                       "fused_elementwise_mul",
+                                       "fused_elementwise_sub",
                                        "elementwise_add",
                                        "elementwise_mul",
                                        "elementwise_sub",
@@ -87,12 +90,7 @@ void CPUQuantizePlacementPass::ApplyImpl(ir::Graph* graph) const {
       return;
     }
 
-    // Remove this condition when all fused_elementwise ops are merged
-    if (!(op->Op()->Type() == "elementwise_add" ||
-          op->Op()->Type() == "elementwise_sub" ||
-          op->Op()->Type() == "elementwise_mul")) {
-      ConvertToFusedOp(op->Op());
-    }
+    ConvertToFusedOp(op->Op());
     op->Op()->SetAttr("mkldnn_data_type", std::string("int8"));
   };
   gpd(graph, handler);
diff --git a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/elementwise_act_onednn_fuse_pass.cc
similarity index 94%
rename from paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc
rename to paddle/fluid/framework/ir/mkldnn/elementwise_act_onednn_fuse_pass.cc
index 4b3f6a95d6dd1..b6e84145aebff 100644
--- a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.cc
+++ b/paddle/fluid/framework/ir/mkldnn/elementwise_act_onednn_fuse_pass.cc
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h"
+#include "paddle/fluid/framework/ir/mkldnn/elementwise_act_onednn_fuse_pass.h"
 
 #include "paddle/fluid/framework/ir/graph_pattern_detector.h"
 #include "paddle/fluid/framework/ir/mkldnn/activation_onednn_fuse_pass.h"
@@ -62,6 +62,7 @@ void ElementwiseActivationOneDNNPass::FuseElementwiseAct(
     GET_IR_NODE_FROM_SUBGRAPH(
         activation_out, activation_out, elementwise_act_pattern);
 
+    ConvertToFusedOp(elementwise->Op());
     SetActivationAttrs(elementwise->Op(), activation->Op(), act_type);
     elementwise->Op()->SetOutput("Out", {activation_out->Name()});
 
@@ -84,9 +85,9 @@ void ElementwiseActivationOneDNNPass::FuseElementwiseAct(
 }  // namespace framework
 }  // namespace paddle
 
-REGISTER_PASS(elt_act_mkldnn_fuse_pass,
+REGISTER_PASS(elementwise_act_onednn_fuse_pass,
               paddle::framework::ir::ElementwiseActivationOneDNNPass);
-REGISTER_PASS_CAPABILITY(elt_act_mkldnn_fuse_pass)
+REGISTER_PASS_CAPABILITY(elementwise_act_onednn_fuse_pass)
     .AddCombination(
         paddle::framework::compatible::OpVersionComparatorCombination()
             .LE("elementwise_add", 1)
diff --git a/paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/mkldnn/elementwise_act_onednn_fuse_pass.h
similarity index 100%
rename from paddle/fluid/framework/ir/mkldnn/elt_act_mkldnn_fuse_pass.h
rename to paddle/fluid/framework/ir/mkldnn/elementwise_act_onednn_fuse_pass.h
diff --git a/paddle/fluid/framework/ir/mkldnn/mkldnn_pass_util.h b/paddle/fluid/framework/ir/mkldnn/mkldnn_pass_util.h
index 8438bf79fd6c7..a9c6d0e0b0bd1 100644
--- a/paddle/fluid/framework/ir/mkldnn/mkldnn_pass_util.h
+++ b/paddle/fluid/framework/ir/mkldnn/mkldnn_pass_util.h
@@ -148,6 +148,8 @@ inline void ConvertToFusedOp(OpDesc* op) {
   const std::map<std::string, std::string> fused_ops = {
       {"conv2d", "fused_conv2d"},
       {"depthwise_conv2d", "fused_conv2d"},
+      {"elementwise_add", "fused_elementwise_add"},
+      {"elementwise_sub", "fused_elementwise_sub"},
       {"elementwise_mul", "fused_elementwise_mul"},
       {"elementwise_div", "fused_elementwise_div"},
       {"matmul", "fused_matmul"},
diff --git a/paddle/fluid/framework/ir/mkldnn/operator_scale_onednn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/operator_scale_onednn_fuse_pass.cc
index 7c42ae2551585..086de37ab4299 100644
--- a/paddle/fluid/framework/ir/mkldnn/operator_scale_onednn_fuse_pass.cc
+++ b/paddle/fluid/framework/ir/mkldnn/operator_scale_onednn_fuse_pass.cc
@@ -31,6 +31,8 @@ void FuseOperatorScaleOneDNNPass::ApplyImpl(Graph *graph) const {
       "fused_matmul",
       "matmul",
       "matmul_v2",
+      "fused_elementwise_add",
+      "fused_elementwise_sub",
       "fused_elementwise_mul",
       "fused_elementwise_div",
       "elementwise_add",
@@ -119,6 +121,8 @@ REGISTER_PASS_CAPABILITY(operator_scale_onednn_fuse_pass)
             .EQ("fused_matmul", 0)
             .LE("matmul", 1)
             .EQ("matmul_v2", 0)
+            .EQ("fused_elementwise_add", 0)
+            .EQ("fused_elementwise_sub", 0)
             .EQ("fused_elementwise_mul", 0)
             .EQ("fused_elementwise_div", 0)
             .LE("elementwise_add", 1)
diff --git a/paddle/fluid/framework/ir/mkldnn/operator_unsqueeze2_onednn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/operator_unsqueeze2_onednn_fuse_pass.cc
index 538a39374d807..7726206e84ea4 100644
--- a/paddle/fluid/framework/ir/mkldnn/operator_unsqueeze2_onednn_fuse_pass.cc
+++ b/paddle/fluid/framework/ir/mkldnn/operator_unsqueeze2_onednn_fuse_pass.cc
@@ -16,7 +16,6 @@
 
 #include "paddle/fluid/framework/ir/mkldnn/mkldnn_pass_util.h"
 #include "paddle/fluid/framework/op_version_registry.h"
-#include "paddle/phi/backends/onednn/onednn_reuse.h"
 #include "paddle/utils/string/pretty_log.h"
 
 namespace paddle {
diff --git a/paddle/fluid/inference/api/paddle_pass_builder.cc b/paddle/fluid/inference/api/paddle_pass_builder.cc
index f9123a111771f..4cbbb0e23ed2c 100644
--- a/paddle/fluid/inference/api/paddle_pass_builder.cc
+++ b/paddle/fluid/inference/api/paddle_pass_builder.cc
@@ -368,7 +368,7 @@ void CpuPassStrategy::EnableMKLDNN() {
              "batch_norm_act_fuse_pass",              //
              "softplus_activation_onednn_fuse_pass",  //
              "shuffle_channel_mkldnn_detect_pass",    //
-             "elt_act_mkldnn_fuse_pass",              //
+             "elementwise_act_onednn_fuse_pass",      //
              "layer_norm_onednn_optimization_pass",   //
              "operator_scale_onednn_fuse_pass",       //
              "operator_unsqueeze2_onednn_fuse_pass",  //
diff --git a/paddle/fluid/operators/compat/elementwise_add.pbtxt b/paddle/fluid/operators/compat/elementwise_add.pbtxt
index 5b55f3981c77d..ee2f1d7c6861e 100644
--- a/paddle/fluid/operators/compat/elementwise_add.pbtxt
+++ b/paddle/fluid/operators/compat/elementwise_add.pbtxt
@@ -22,14 +22,6 @@ extra {
   attrs {
     name: "Out0_threshold"
     type: FLOAT
-  }  
-  attrs {
-    name: "x_data_format"
-    type: STRING
-  }
-  attrs {
-    name: "y_data_format"
-    type: STRING
   }
   attrs {
     name: "Scale_x"
diff --git a/paddle/fluid/operators/compat/elementwise_div.pbtxt b/paddle/fluid/operators/compat/elementwise_div.pbtxt
index a73d2072029a3..0b46bfb748401 100644
--- a/paddle/fluid/operators/compat/elementwise_div.pbtxt
+++ b/paddle/fluid/operators/compat/elementwise_div.pbtxt
@@ -16,11 +16,7 @@ def {
 }
 extra {
   attrs {
-    name: "x_data_format"
-    type: STRING
-  }
-  attrs {
-    name: "y_data_format"
+    name: "act"
     type: STRING
   }
   attrs {
@@ -35,8 +31,4 @@ extra {
     name: "Scale_out"
     type: FLOAT
   }
-  attrs {
-    name: "act"
-    type: STRING
-  }
 }
diff --git a/paddle/fluid/operators/compat/elementwise_mul.pbtxt b/paddle/fluid/operators/compat/elementwise_mul.pbtxt
index 22289e2689c10..94cec09ace8f3 100644
--- a/paddle/fluid/operators/compat/elementwise_mul.pbtxt
+++ b/paddle/fluid/operators/compat/elementwise_mul.pbtxt
@@ -13,16 +13,6 @@ def {
     name: "axis"
     type: INT
   }
-}
-extra {
-  attrs {
-    name: "x_data_format"
-    type: STRING
-  }
-  attrs {
-    name: "y_data_format"
-    type: STRING
-  }
   attrs {
     name: "Scale_x"
     type: FLOAT
diff --git a/paddle/fluid/operators/compat/elementwise_pow.pbtxt b/paddle/fluid/operators/compat/elementwise_pow.pbtxt
index a2ab73f409b78..e9427104dbfa7 100644
--- a/paddle/fluid/operators/compat/elementwise_pow.pbtxt
+++ b/paddle/fluid/operators/compat/elementwise_pow.pbtxt
@@ -16,11 +16,7 @@ def {
 }
 extra {
   attrs {
-    name: "x_data_format"
-    type: STRING
-  }
-  attrs {
-    name: "y_data_format"
+    name: "act"
     type: STRING
   }
   attrs {
@@ -35,8 +31,4 @@ extra {
     name: "Scale_out"
     type: FLOAT
   }
-  attrs {
-    name: "act"
-    type: STRING
-  }
 }
diff --git a/paddle/fluid/operators/compat/elementwise_sub.pbtxt b/paddle/fluid/operators/compat/elementwise_sub.pbtxt
index 9f38601f585ea..f73b92fb95f87 100644
--- a/paddle/fluid/operators/compat/elementwise_sub.pbtxt
+++ b/paddle/fluid/operators/compat/elementwise_sub.pbtxt
@@ -16,11 +16,7 @@ def {
 }
 extra {
   attrs {
-    name: "x_data_format"
-    type: STRING
-  }
-  attrs {
-    name: "y_data_format"
+    name: "act"
     type: STRING
   }
   attrs {
@@ -35,8 +31,4 @@ extra {
     name: "Scale_out"
     type: FLOAT
   }
-  attrs {
-    name: "act"
-    type: STRING
-  }
 }
diff --git a/paddle/fluid/operators/compat/fused_elementwise_add.pbtxt b/paddle/fluid/operators/compat/fused_elementwise_add.pbtxt
new file mode 100644
index 0000000000000..6c589b7244ca4
--- /dev/null
+++ b/paddle/fluid/operators/compat/fused_elementwise_add.pbtxt
@@ -0,0 +1,50 @@
+type: "fused_elementwise_add"
+def {
+  inputs {
+    name: "X"
+  }
+  inputs {
+    name: "Y"
+  }
+  outputs {
+    name: "Out"
+  }
+  attrs {
+    name: "axis"
+    type: INT
+  }
+}
+extra {
+  attrs {
+    name: "fuse_activation"
+    type: STRING
+  }
+  attrs {
+    name: "fuse_alpha"
+    type: FLOAT
+  }
+  attrs {
+    name: "fuse_beta"
+    type: FLOAT
+  }
+  attrs {
+    name: "fused_output_scale"
+    type: FLOAT
+  }
+  attrs {
+    name: "fused_unsqueeze2_axes"
+    type: INTS
+  }
+  attrs {
+    name: "scale_x"
+    type: FLOAT
+  }
+  attrs {
+    name: "scale_y"
+    type: FLOAT
+  }
+  attrs {
+    name: "scale_out"
+    type: FLOAT
+  }
+}
diff --git a/paddle/fluid/operators/compat/fused_elementwise_sub.pbtxt b/paddle/fluid/operators/compat/fused_elementwise_sub.pbtxt
new file mode 100644
index 0000000000000..b9d4acad8cbc7
--- /dev/null
+++ b/paddle/fluid/operators/compat/fused_elementwise_sub.pbtxt
@@ -0,0 +1,50 @@
+type: "fused_elementwise_sub"
+def {
+  inputs {
+    name: "X"
+  }
+  inputs {
+    name: "Y"
+  }
+  outputs {
+    name: "Out"
+  }
+  attrs {
+    name: "axis"
+    type: INT
+  }
+}
+extra {
+  attrs {
+    name: "fuse_activation"
+    type: STRING
+  }
+  attrs {
+    name: "fuse_alpha"
+    type: FLOAT
+  }
+  attrs {
+    name: "fuse_beta"
+    type: FLOAT
+  }
+  attrs {
+    name: "fused_output_scale"
+    type: FLOAT
+  }
+  attrs {
+    name: "fused_unsqueeze2_axes"
+    type: INTS
+  }
+  attrs {
+    name: "scale_x"
+    type: FLOAT
+  }
+  attrs {
+    name: "scale_y"
+    type: FLOAT
+  }
+  attrs {
+    name: "scale_out"
+    type: FLOAT
+  }
+}
diff --git a/paddle/fluid/operators/elementwise/elementwise_sub_op.cc b/paddle/fluid/operators/elementwise/elementwise_sub_op.cc
index 87c4e4977258f..5860f7a4009db 100644
--- a/paddle/fluid/operators/elementwise/elementwise_sub_op.cc
+++ b/paddle/fluid/operators/elementwise/elementwise_sub_op.cc
@@ -170,9 +170,9 @@ REGISTER_OPERATOR(elementwise_sub_grad_grad,
 
 REGISTER_OP_VERSION(elementwise_sub)
     .AddCheckpoint(
-        R"ROC(Register elementwise_sub for adding the attribute of Scale_y)ROC",
+        R"ROC(Register elementwise_sub for adding the attribute of scale_y)ROC",
         paddle::framework::compatible::OpVersionDesc().NewAttr(
-            "Scale_y",
+            "scale_y",
             "In order to support the function of scaling the input Y when "
             "using the operator of elementwise_sub.",
             1.0f));
diff --git a/paddle/fluid/operators/fused/fused_elementwise_op.cc b/paddle/fluid/operators/fused/fused_elementwise_op.cc
index 4b21ada97d07c..e6c2743e9385d 100644
--- a/paddle/fluid/operators/fused/fused_elementwise_op.cc
+++ b/paddle/fluid/operators/fused/fused_elementwise_op.cc
@@ -62,6 +62,21 @@ class FusedElementwiseOpMaker : public framework::OpProtoAndCheckerMaker {
 }  // namespace paddle
 
 namespace ops = paddle::operators;
+REGISTER_OPERATOR(
+    fused_elementwise_add,
+    ops::ElementwiseOp,
+    ops::FusedElementwiseOpMaker,
+    ops::ElementwiseOpInferVarType,
+    paddle::framework::EmptyGradOpMaker<paddle::framework::OpDesc>,
+    paddle::framework::EmptyGradOpMaker<paddle::imperative::OpBase>);
+
+REGISTER_OPERATOR(
+    fused_elementwise_sub,
+    ops::ElementwiseOp,
+    ops::FusedElementwiseOpMaker,
+    ops::ElementwiseOpInferVarType,
+    paddle::framework::EmptyGradOpMaker<paddle::framework::OpDesc>,
+    paddle::framework::EmptyGradOpMaker<paddle::imperative::OpBase>);
 
 REGISTER_OPERATOR(
     fused_elementwise_mul,
diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml
index edf08f5bb8fe5..4c9ac1729b62d 100755
--- a/paddle/phi/api/yaml/op_compat.yaml
+++ b/paddle/phi/api/yaml/op_compat.yaml
@@ -92,7 +92,7 @@
 - op : add (elementwise_add)
   backward : add_grad (elementwise_add_grad)
   extra :
-    attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
+    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
 - op : add_n (sum)
@@ -661,7 +661,7 @@
   outputs :
     out: Out
   extra :
-    attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
+    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
 - op : dot
@@ -733,7 +733,7 @@
 - op : elementwise_pow
   backward : elementwise_pow_grad
   extra :
-    attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
+    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
 - op : elu
@@ -922,19 +922,19 @@
 
 - op : floor_divide (elementwise_floordiv)
   extra :
-    attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
+    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
 - op : fmax (elementwise_fmax)
   backward : fmax_grad (elementwise_fmax_grad)
   extra :
-    attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
+    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
 - op : fmin (elementwise_fmin)
   backward : fmin_grad (elementwise_fmin_grad)
   extra :
-    attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
+    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
 - op : fold
@@ -1036,7 +1036,7 @@
 
 - op : grad_add
   extra :
-    attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
+    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
 - op : greater_equal
@@ -1114,7 +1114,7 @@
 - op : heaviside (elementwise_heaviside)
   backward : heaviside_grad (elementwise_heaviside_grad)
   extra :
-    attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
+    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
 - op : histogram
@@ -1522,13 +1522,13 @@
 - op : maximum (elementwise_max)
   backward : maximum_grad (elementwise_max_grad)
   extra :
-    attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
+    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
 - op : maximum (elementwise_min)
   backward : maximum_grad (elementwise_min_grad)
   extra :
-    attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
+    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
 - op : maxout
@@ -1642,7 +1642,7 @@
   outputs :
     out : Out
   extra :
-    attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
+    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
 - op : mv
@@ -1897,7 +1897,7 @@
 
 - op : remainder (elementwise_mod)
   extra :
-    attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
+    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
 - op : renorm
@@ -2260,7 +2260,7 @@
   outputs :
     out : Out
   extra :
-    attrs : [bool use_mkldnn = false, str x_data_format = "", str y_data_format = "", str mkldnn_data_type = "float32",
+    attrs : [bool use_mkldnn = false, str mkldnn_data_type = "float32",
              bool use_quantizer = false, float Scale_x = 1.0f, float Scale_y = 1.0f, float Scale_out = 1.0f]
 
 - op : sum (reduce_sum)
diff --git a/paddle/phi/kernels/fusion/onednn/fused_elementwise_kernel.cc b/paddle/phi/kernels/fusion/onednn/fused_elementwise_kernel.cc
index b38485543e578..d19bca440fbbf 100644
--- a/paddle/phi/kernels/fusion/onednn/fused_elementwise_kernel.cc
+++ b/paddle/phi/kernels/fusion/onednn/fused_elementwise_kernel.cc
@@ -151,12 +151,32 @@ void FusedElementwiseKernel(const OneDNNContext& dev_ctx,
                                          out);                     \
   }
 
+DEFINE_ONEDNN_ELEMENTWISE_KERNEL(FusedAdd, dnnl::algorithm::binary_add)
+DEFINE_ONEDNN_ELEMENTWISE_KERNEL(FusedSubtract, dnnl::algorithm::binary_sub)
 DEFINE_ONEDNN_ELEMENTWISE_KERNEL(FusedMultiply, dnnl::algorithm::binary_mul)
 DEFINE_ONEDNN_ELEMENTWISE_KERNEL(FusedDivide, dnnl::algorithm::binary_div)
 
 }  // namespace fusion
 }  // namespace phi
 
+PD_REGISTER_KERNEL(fused_elementwise_add,
+                   OneDNN,
+                   ONEDNN,
+                   phi::fusion::FusedAddKernel,
+                   float,
+                   phi::dtype::bfloat16,
+                   int8_t,
+                   uint8_t) {}
+
+PD_REGISTER_KERNEL(fused_elementwise_sub,
+                   OneDNN,
+                   ONEDNN,
+                   phi::fusion::FusedSubtractKernel,
+                   float,
+                   phi::dtype::bfloat16,
+                   int8_t,
+                   uint8_t) {}
+
 PD_REGISTER_KERNEL(fused_elementwise_mul,
                    OneDNN,
                    ONEDNN,
diff --git a/paddle/phi/kernels/onednn/elementwise_kernel.cc b/paddle/phi/kernels/onednn/elementwise_kernel.cc
index d20fef81a2ebf..02b148e54f547 100644
--- a/paddle/phi/kernels/onednn/elementwise_kernel.cc
+++ b/paddle/phi/kernels/onednn/elementwise_kernel.cc
@@ -30,26 +30,6 @@ void ElementwiseKernel(const OneDNNContext& dev_ctx,
                        DenseTensor* out) {
   const auto& onednn_engine = dev_ctx.GetEngine();
 
-  float scale_x = dev_ctx.HasDnnAttr("Scale_x")
-                      ? PADDLE_GET_CONST(float, dev_ctx.GetDnnAttr("Scale_x"))
-                      : 1.0f;
-  float scale_y = dev_ctx.HasDnnAttr("Scale_y")
-                      ? PADDLE_GET_CONST(float, dev_ctx.GetDnnAttr("Scale_y"))
-                      : 1.0f;
-  float scale_out =
-      dev_ctx.HasDnnAttr("Scale_out")
-          ? PADDLE_GET_CONST(float, dev_ctx.GetDnnAttr("Scale_out"))
-          : 1.0f;
-
-  dnnl::post_ops post_operations;
-  funcs::AppendActivation(dev_ctx, post_operations);
-  if (dev_ctx.HasDnnAttr("fused_output_scale")) {
-    float scale_alpha =
-        PADDLE_GET_CONST(float, dev_ctx.GetDnnAttr("fused_output_scale"));
-    post_operations.append_eltwise(
-        1.0, dnnl::algorithm::eltwise_linear, scale_alpha, 0.0f);
-  }
-
   auto* non_const_x = &x;
   auto* non_const_y = &y;
 
@@ -60,11 +40,10 @@ void ElementwiseKernel(const OneDNNContext& dev_ctx,
                                         non_const_x,
                                         non_const_y,
                                         out,
-                                        scale_x,
-                                        scale_y,
-                                        scale_out,
-                                        true,
-                                        post_operations);
+                                        1.0f,
+                                        1.0f,
+                                        1.0f,
+                                        true);
 
   // oneDNN's binary is optimized for broadcasting y into x, so in other case
   // we have to swap tensors to achieve optimal performance
diff --git a/paddle/phi/ops/compat/fused_elementwise_sig.cc b/paddle/phi/ops/compat/fused_elementwise_sig.cc
index a5b9da7dcfe38..5494546de9804 100644
--- a/paddle/phi/ops/compat/fused_elementwise_sig.cc
+++ b/paddle/phi/ops/compat/fused_elementwise_sig.cc
@@ -16,6 +16,38 @@
 
 namespace phi {
 
+KernelSignature FusedElementwiseAddOpArgumentMapping(
+    const ArgumentMappingContext& ctx) {
+  return KernelSignature("fused_elementwise_add",
+                         {"X", "Y"},
+                         {"axis",
+                          "fuse_activation",
+                          "fuse_alpha",
+                          "fuse_beta",
+                          "fused_output_scale",
+                          "fused_unsqueeze2_axes",
+                          "scale_x",
+                          "scale_y",
+                          "scale_out"},
+                         {"Out"});
+}
+
+KernelSignature FusedElementwiseSubOpArgumentMapping(
+    const ArgumentMappingContext& ctx) {
+  return KernelSignature("fused_elementwise_sub",
+                         {"X", "Y"},
+                         {"axis",
+                          "fuse_activation",
+                          "fuse_alpha",
+                          "fuse_beta",
+                          "fused_output_scale",
+                          "fused_unsqueeze2_axes",
+                          "scale_x",
+                          "scale_y",
+                          "scale_out"},
+                         {"Out"});
+}
+
 KernelSignature FusedElementwiseMulOpArgumentMapping(
     const ArgumentMappingContext& ctx) {
   return KernelSignature("fused_elementwise_mul",
@@ -50,6 +82,10 @@ KernelSignature FusedElementwiseDivOpArgumentMapping(
 
 }  // namespace phi
 
+PD_REGISTER_ARG_MAPPING_FN(fused_elementwise_add,
+                           phi::FusedElementwiseAddOpArgumentMapping);
+PD_REGISTER_ARG_MAPPING_FN(fused_elementwise_sub,
+                           phi::FusedElementwiseSubOpArgumentMapping);
 PD_REGISTER_ARG_MAPPING_FN(fused_elementwise_mul,
                            phi::FusedElementwiseMulOpArgumentMapping);
 PD_REGISTER_ARG_MAPPING_FN(fused_elementwise_div,
diff --git a/python/paddle/incubate/autograd/primrules.py b/python/paddle/incubate/autograd/primrules.py
index b6487c79305ec..0fd8a77dd7205 100644
--- a/python/paddle/incubate/autograd/primrules.py
+++ b/python/paddle/incubate/autograd/primrules.py
@@ -132,69 +132,21 @@ def linear_jvp(op, *args, **kwargs):
 def elementwise_add_orig2prim(op, x, y):
     if x.shape != y.shape:
         y = broadcast(y, shape=x.shape)
-    if op.attr('Scale_x') - 1.0 > 1e-5:
-        scale_x = fill_const(
-            shape=x.shape, dtype=x.dtype, value=op.attr('Scale_x')
-        )
-        x = mul(x, scale_x)
-    if op.attr('Scale_y') - 1.0 > 1e-5:
-        scale_y = fill_const(
-            shape=y.shape, dtype=y.dtype, value=op.attr('Scale_y')
-        )
-        y = mul(y, scale_y)
-    z = add(x, y)
-    if op.attr('Scale_out') - 1.0 > 1e-5:
-        scale_out = fill_const(
-            shape=z.shape, dtype=z.dtype, value=op.attr('Scale_out')
-        )
-        z = mul(z, scale_out)
-    return z
+    return add(x, y)
 
 
 @REGISTER_ORIG2PRIM('elementwise_sub')
 def elementwise_sub_orig2prim(op, x, y):
     if x.shape != y.shape:
         y = broadcast(y, shape=x.shape)
-    if op.attr('Scale_x') - 1.0 > 1e-5:
-        scale_x = fill_const(
-            shape=x.shape, dtype=x.dtype, value=op.attr('Scale_x')
-        )
-        x = mul(x, scale_x)
-    if op.attr('Scale_y') - 1.0 > 1e-5:
-        scale_y = fill_const(
-            shape=y.shape, dtype=y.dtype, value=op.attr('Scale_y')
-        )
-        y = mul(y, scale_y)
-    z = sub(x, y)
-    if op.attr('Scale_out') - 1.0 > 1e-5:
-        scale_out = fill_const(
-            shape=z.shape, dtype=z.dtype, value=op.attr('Scale_out')
-        )
-        z = mul(z, scale_out)
-    return z
+    return sub(x, y)
 
 
 @REGISTER_ORIG2PRIM('elementwise_mul')
 def elementwise_mul_orig2prim(op, x, y):
     if x.shape != y.shape:
         y = broadcast(y, shape=x.shape)
-    if op.attr('Scale_x') - 1.0 > 1e-5:
-        scale_x = fill_const(
-            shape=x.shape, dtype=x.dtype, value=op.attr('Scale_x')
-        )
-        x = mul(x, scale_x)
-    if op.attr('Scale_y') - 1.0 > 1e-5:
-        scale_y = fill_const(
-            shape=y.shape, dtype=y.dtype, value=op.attr('Scale_y')
-        )
-        y = mul(y, scale_y)
-    z = mul(x, y)
-    if op.attr('Scale_out') - 1.0 > 1e-5:
-        scale_out = fill_const(
-            shape=z.shape, dtype=z.dtype, value=op.attr('Scale_out')
-        )
-        z = mul(z, scale_out)
-    return z
+    return mul(x, y)
 
 
 @REGISTER_ORIG2PRIM('elementwise_div')
diff --git a/test/cpp/fluid/feed_forward_test.cu b/test/cpp/fluid/feed_forward_test.cu
index 08d91d1bb2af3..7febf20e77118 100644
--- a/test/cpp/fluid/feed_forward_test.cu
+++ b/test/cpp/fluid/feed_forward_test.cu
@@ -233,21 +233,13 @@ void GetElementwiseAddOpGrad(const std::vector<T> &dout_vec,
       dout_ptr, dout_vec.data(), size_z * sizeof(T), cudaMemcpyHostToDevice);
 
   int axis = -1;
-  bool use_mkldnn = false, use_quantizer = false;
+  bool use_mkldnn = false;
   std::string mkldnn_data_type = "float32";
-  std::string x_data_format = "", y_data_format = "";
-  float Scale_x = 1.0, Scale_y = 1.0, Scale_out = 1.0;
 
   framework::AttributeMap attrs;
   attrs.insert({"axis", axis});
   attrs.insert({"use_mkldnn", use_mkldnn});
-  attrs.insert({"x_data_format", x_data_format});
-  attrs.insert({"y_data_format", y_data_format});
-  attrs.insert({"use_quantizer", use_quantizer});
   attrs.insert({"mkldnn_data_type", mkldnn_data_type});
-  attrs.insert({"Scale_x", Scale_x});
-  attrs.insert({"Scale_y", Scale_y});
-  attrs.insert({"Scale_out", Scale_out});
 
   auto op = framework::OpRegistry::CreateOp(
       "elementwise_add_grad",
diff --git a/test/ir/inference/test_mkldnn_elt_act_fuse_pass.py b/test/ir/inference/test_mkldnn_elt_act_fuse_pass.py
index a6ba318e81cba..582d3efec9d3f 100644
--- a/test/ir/inference/test_mkldnn_elt_act_fuse_pass.py
+++ b/test/ir/inference/test_mkldnn_elt_act_fuse_pass.py
@@ -23,10 +23,10 @@
 from paddle.fluid.core import PassVersionChecker
 
 
-class ElementwiseActivationMkldnnFusePassTest(InferencePassTest):
+class ElementwiseActivationOneDNNFusePassTest(InferencePassTest):
     act_alpha = None
     act_beta = None
-    pass_name = 'elt_act_mkldnn_fuse_pass'
+    pass_name = 'elementwise_act_onednn_fuse_pass'
 
     def setUp(self):
         self.set_params()
@@ -65,24 +65,24 @@ def test_pass_compatible(self):
         self.assertTrue(PassVersionChecker.IsCompatible(self.pass_name))
 
 
-class ElementwiseActivationMkldnnFusePassTest_Add_Relu(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Add_Relu(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.add
         self.act = F.relu
 
 
-class ElementwiseActivationMkldnnFusePassTest_Add_Tanh(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Add_Tanh(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.add
         self.act = paddle.tanh
 
 
-class ElementwiseActivationMkldnnFusePassTest_Add_LeakyRelu(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Add_LeakyRelu(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.add
@@ -90,40 +90,40 @@ def set_params(self):
         self.act = paddle.nn.functional.leaky_relu
 
 
-class ElementwiseActivationMkldnnFusePassTest_Add_Swish(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Add_Swish(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.add
         self.act = paddle.nn.functional.swish
 
 
-class ElementwiseActivationMkldnnFusePassTest_Add_HardSwish(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Add_HardSwish(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.add
         self.act = paddle.nn.functional.hardswish
 
 
-class ElementwiseActivationMkldnnFusePassTest_Add_SQRT(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Add_SQRT(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.add
         self.act = paddle.sqrt
 
 
-class ElementwiseActivationMkldnnFusePassTest_Add_ABS(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Add_ABS(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.add
         self.act = paddle.abs
 
 
-class ElementwiseActivationMkldnnFusePassTest_Add_Clip(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Add_Clip(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.add
@@ -132,16 +132,16 @@ def set_params(self):
         self.act_beta = 10.0
 
 
-class ElementwiseActivationMkldnnFusePassTest_Add_Gelu(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Add_Gelu(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.add
         self.act = paddle.nn.functional.gelu
 
 
-class ElementwiseActivationMkldnnFusePassTest_Add_Gelu_Tanh(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Add_Gelu_Tanh(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.add
@@ -149,40 +149,40 @@ def set_params(self):
         self.act_alpha = True
 
 
-class ElementwiseActivationMkldnnFusePassTest_Add_Relu6(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Add_Relu6(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.add
         self.act = paddle.nn.functional.relu6
 
 
-class ElementwiseActivationMkldnnFusePassTest_Add_Sigmoid(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Add_Sigmoid(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.add
         self.act = paddle.nn.functional.sigmoid
 
 
-class ElementwiseActivationMkldnnFusePassTest_Sub_Relu(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Sub_Relu(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.subtract
         self.act = F.relu
 
 
-class ElementwiseActivationMkldnnFusePassTest_Sub_Tanh(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Sub_Tanh(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.subtract
         self.act = paddle.tanh
 
 
-class ElementwiseActivationMkldnnFusePassTest_Sub_LeakyRelu(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Sub_LeakyRelu(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.subtract
@@ -190,32 +190,32 @@ def set_params(self):
         self.act = paddle.nn.functional.leaky_relu
 
 
-class ElementwiseActivationMkldnnFusePassTest_Sub_Swish(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Sub_Swish(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.subtract
         self.act = paddle.nn.functional.swish
 
 
-class ElementwiseActivationMkldnnFusePassTest_Sub_HardSwish(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Sub_HardSwish(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.subtract
         self.act = paddle.nn.functional.hardswish
 
 
-class ElementwiseActivationMkldnnFusePassTest_Sub_ABS(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Sub_ABS(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.subtract
         self.act = paddle.abs
 
 
-class ElementwiseActivationMkldnnFusePassTest_Sub_Clip(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Sub_Clip(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.subtract
@@ -224,16 +224,16 @@ def set_params(self):
         self.act_beta = 10.0
 
 
-class ElementwiseActivationMkldnnFusePassTest_Sub_Gelu(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Sub_Gelu(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.subtract
         self.act = paddle.nn.functional.gelu
 
 
-class ElementwiseActivationMkldnnFusePassTest_Sub_Gelu_Tanh(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Sub_Gelu_Tanh(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.subtract
@@ -241,40 +241,40 @@ def set_params(self):
         self.act_alpha = True
 
 
-class ElementwiseActivationMkldnnFusePassTest_Sub_Relu6(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Sub_Relu6(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.subtract
         self.act = paddle.nn.functional.relu6
 
 
-class ElementwiseActivationMkldnnFusePassTest_Sub_Sigmoid(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Sub_Sigmoid(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.subtract
         self.act = paddle.nn.functional.sigmoid
 
 
-class ElementwiseActivationMkldnnFusePassTest_Mul_Relu(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Mul_Relu(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.multiply
         self.act = F.relu
 
 
-class ElementwiseActivationMkldnnFusePassTest_Mul_Tanh(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Mul_Tanh(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.multiply
         self.act = paddle.tanh
 
 
-class ElementwiseActivationMkldnnFusePassTest_Mul_LeakyRelu(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Mul_LeakyRelu(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.multiply
@@ -282,40 +282,40 @@ def set_params(self):
         self.act = paddle.nn.functional.leaky_relu
 
 
-class ElementwiseActivationMkldnnFusePassTest_Mul_Swish(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Mul_Swish(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.multiply
         self.act = paddle.nn.functional.swish
 
 
-class ElementwiseActivationMkldnnFusePassTest_Mul_HardSwish(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Mul_HardSwish(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.multiply
         self.act = paddle.nn.functional.hardswish
 
 
-class ElementwiseActivationMkldnnFusePassTest_Mul_SQRT(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Mul_SQRT(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.multiply
         self.act = paddle.sqrt
 
 
-class ElementwiseActivationMkldnnFusePassTest_Mul_ABS(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Mul_ABS(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.multiply
         self.act = paddle.abs
 
 
-class ElementwiseActivationMkldnnFusePassTest_Mul_Clip(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Mul_Clip(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.multiply
@@ -324,16 +324,16 @@ def set_params(self):
         self.act_beta = 10.0
 
 
-class ElementwiseActivationMkldnnFusePassTest_Mul_Gelu(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Mul_Gelu(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.multiply
         self.act = paddle.nn.functional.gelu
 
 
-class ElementwiseActivationMkldnnFusePassTest_Mul_Gelu_Tanh(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Mul_Gelu_Tanh(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.multiply
@@ -341,16 +341,16 @@ def set_params(self):
         self.act_alpha = True
 
 
-class ElementwiseActivationMkldnnFusePassTest_Mul_Relu6(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Mul_Relu6(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.multiply
         self.act = paddle.nn.functional.relu6
 
 
-class ElementwiseActivationMkldnnFusePassTest_Mul_Sigmoid(
-    ElementwiseActivationMkldnnFusePassTest
+class ElementwiseActivationOneDNNFusePassTest_Mul_Sigmoid(
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.multiply
@@ -358,7 +358,7 @@ def set_params(self):
 
 
 class ElementwiseScaleOneDNNFusePassTest_Add(
-    ElementwiseActivationMkldnnFusePassTest
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.add
@@ -367,7 +367,7 @@ def set_params(self):
 
 
 class ElementwiseScaleOneDNNFusePassTest_Sub(
-    ElementwiseActivationMkldnnFusePassTest
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.subtract
@@ -376,7 +376,7 @@ def set_params(self):
 
 
 class ElementwiseScaleOneDNNFusePassTest_Mul(
-    ElementwiseActivationMkldnnFusePassTest
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.multiply
@@ -385,7 +385,7 @@ def set_params(self):
 
 
 class ElementwiseScaleOneDNNFusePassTest_Div(
-    ElementwiseActivationMkldnnFusePassTest
+    ElementwiseActivationOneDNNFusePassTest
 ):
     def set_params(self):
         self.operand = paddle.divide
diff --git a/test/ir/inference/test_onednn_elementwise_add_activation_fuse_pass.py b/test/ir/inference/test_onednn_elementwise_add_activation_fuse_pass.py
index 93165f829b1c8..e4a4809971739 100644
--- a/test/ir/inference/test_onednn_elementwise_add_activation_fuse_pass.py
+++ b/test/ir/inference/test_onednn_elementwise_add_activation_fuse_pass.py
@@ -118,17 +118,17 @@ def sample_predictor_configs(self, program_config):
         config = self.create_inference_config(
             use_mkldnn=True,
             passes=[
-                'elt_act_mkldnn_fuse_pass',
+                'elementwise_act_onednn_fuse_pass',
                 'operator_scale_onednn_fuse_pass',
             ],
         )
-        yield config, ['elementwise_add'], (1e-5, 1e-5)
+        yield config, ['fused_elementwise_add'], (1e-5, 1e-5)
 
     def test(self):
         self.run_and_statis(
             quant=False,
             passes=[
-                'elt_act_mkldnn_fuse_pass',
+                'elementwise_act_onednn_fuse_pass',
                 'operator_scale_onednn_fuse_pass',
             ],
         )
diff --git a/test/ir/inference/test_trt_convert_multihead_matmul.py b/test/ir/inference/test_trt_convert_multihead_matmul.py
index dd64efa70311b..4ef391f48abe3 100644
--- a/test/ir/inference/test_trt_convert_multihead_matmul.py
+++ b/test/ir/inference/test_trt_convert_multihead_matmul.py
@@ -763,9 +763,9 @@ def generate_weight2():
                         },
                         "op_outputs": {"Out": ["elementwise_add1_output"]},
                         "op_attrs": {
-                            "Scale_out": 1.0,
-                            "Scale_x": 1.0,
-                            "Scale_y": 1.0,
+                            "scale_out": 1.0,
+                            "scale_x": 1.0,
+                            "scale_y": 1.0,
                             "axis": 2,
                             "Out": 1.0,
                         },
diff --git a/test/mkldnn/CMakeLists.txt b/test/mkldnn/CMakeLists.txt
index d3da5f2897da3..16030200222e8 100644
--- a/test/mkldnn/CMakeLists.txt
+++ b/test/mkldnn/CMakeLists.txt
@@ -24,6 +24,8 @@ endif()
 
 set_tests_properties(test_concat_mkldnn_op PROPERTIES TIMEOUT 120)
 set_tests_properties(test_conv3d_mkldnn_op PROPERTIES TIMEOUT 120)
+set_tests_properties(test_elementwise_mul_onednn_op PROPERTIES TIMEOUT 60)
+set_tests_properties(test_elementwise_add_mkldnn_op PROPERTIES TIMEOUT 60)
 if(WITH_MKLDNN AND NOT WIN32)
   set_tests_properties(test_onnx_format_quantization_mobilenetv1
                        PROPERTIES TIMEOUT 300)
diff --git a/test/mkldnn/test_elementwise_add_mkldnn_op.py b/test/mkldnn/test_elementwise_add_mkldnn_op.py
index 6e190b3183125..0ad1c40021d50 100644
--- a/test/mkldnn/test_elementwise_add_mkldnn_op.py
+++ b/test/mkldnn/test_elementwise_add_mkldnn_op.py
@@ -23,7 +23,7 @@
 )
 
 
-class TestMKLDNNElementwiseAddOp(TestElementwiseAddOp):
+class TestOneDNNElementwiseAddOp(TestElementwiseAddOp):
     def init_kernel_type(self):
         self.use_mkldnn = True
 
@@ -31,21 +31,21 @@ def init_dtype(self):
         self.dtype = np.float32
 
 
-class TestMKLDNNElementwiseAddOp2(TestMKLDNNElementwiseAddOp):
+class TestOneDNNElementwiseAddOp2(TestOneDNNElementwiseAddOp):
     def init_input_output(self):
         self.x = np.random.random((100,)).astype(self.dtype)
         self.y = np.random.random((100,)).astype(self.dtype)
         self.out = np.add(self.x, self.y)
 
 
-class TestMKLDNNElementwiseAddOp3(TestMKLDNNElementwiseAddOp):
+class TestOneDNNElementwiseAddOp3(TestOneDNNElementwiseAddOp):
     def init_input_output(self):
         self.x = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype)
         self.y = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype)
         self.out = np.add(self.x, self.y)
 
 
-class TestMKLDNNElementwiseAddOp4(TestMKLDNNElementwiseAddOp):
+class TestOneDNNElementwiseAddOp4(TestOneDNNElementwiseAddOp):
     def init_input_output(self):
         self.x = np.random.uniform(1, 2, [2, 3, 4, 32]).astype(self.dtype)
         self.y = np.random.uniform(1, 2, [4, 32]).astype(self.dtype)
@@ -59,21 +59,21 @@ def test_check_grad_ingore_y(self):
         pass
 
 
-class TestMKLDNNElementwiseAddOp5(TestMKLDNNElementwiseAddOp):
+class TestOneDNNElementwiseAddOp5(TestOneDNNElementwiseAddOp):
     def init_input_output(self):
         self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype)
         self.y = np.random.uniform(1, 2, [100]).astype(self.dtype)
         self.out = np.add(self.x, self.y)
 
 
-class TestMKLDNNElementwiseAddOpBroadcastXintoY(TestMKLDNNElementwiseAddOp):
+class TestOneDNNElementwiseAddOpBroadcastXintoY(TestOneDNNElementwiseAddOp):
     def init_input_output(self):
         self.x = np.random.uniform(1, 2, [2, 50, 1]).astype(self.dtype)
         self.y = np.random.uniform(1, 2, [2, 50, 160]).astype(self.dtype)
         self.out = np.add(self.x, self.y)
 
 
-class TestMKLDNNElementwiseAddOp_broadcast_3(TestMKLDNNElementwiseAddOp):
+class TestOneDNNElementwiseAddOp_broadcast_3(TestOneDNNElementwiseAddOp):
     def init_input_output(self):
         self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype)
         self.y = np.random.rand(10, 12).astype(self.dtype)
@@ -83,7 +83,7 @@ def init_axis(self):
         self.axis = 1
 
 
-class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestMKLDNNElementwiseAddOp):
+class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestOneDNNElementwiseAddOp):
     def init_input_output(self):
         self.x = np.random.rand(10, 12).astype(self.dtype)
         self.y = np.random.rand(2, 2, 10, 12).astype(self.dtype)
@@ -103,21 +103,21 @@ def test_check_grad_ingore_x(self):
         pass
 
 
-class TestMKLDNNElementwiseAddOpZeroDim(TestMKLDNNElementwiseAddOp):
+class TestOneDNNlementwiseAddOpZeroDim(TestOneDNNElementwiseAddOp):
     def init_input_output(self):
         self.x = np.random.random((100,)).astype(self.dtype)
         self.y = np.array(3.0).astype(self.dtype)
         self.out = np.add(self.x, self.y)
 
 
-class TestMKLDNNElementwiseAddOpZeroDim2(TestMKLDNNElementwiseAddOp):
+class TestOneDNNlementwiseAddOpZeroDim2(TestOneDNNElementwiseAddOp):
     def init_input_output(self):
         self.x = np.array(3.0).astype(self.dtype)
         self.y = np.random.random((100,)).astype(self.dtype)
         self.out = np.add(self.x, self.y)
 
 
-class TestMKLDNNElementwiseAddOpZeroDim3(TestMKLDNNElementwiseAddOp):
+class TestOneDNNlementwiseAddOpZeroDim3(TestOneDNNElementwiseAddOp):
     def init_input_output(self):
         self.x = np.array(3.0).astype(self.dtype)
         self.y = np.array(3.0).astype(self.dtype)
@@ -144,9 +144,9 @@ def init_input_output(self):
         self.out = np.add(self.x, self.y)
 
     def init_scales(self):
-        self.attrs['Scale_x'] = 1.0
-        self.attrs['Scale_y'] = 1.0
-        self.attrs['Scale_out'] = 1.0
+        self.attrs['scale_x'] = 1.0
+        self.attrs['scale_y'] = 1.0
+        self.attrs['scale_out'] = 1.0
 
     def test_check_output(self):
         # TODO(wangzhongpu): support mkldnn op in dygraph mode
@@ -163,48 +163,6 @@ def test_check_grad_ingore_y(self):
         pass
 
 
-class TestInt8Scales(TestInt8):
-    def quantize(self, tensor, dt="int8"):
-        max_int = 127.0 if dt == "int8" else 255.0
-        scale = max_int / np.abs(np.amax(tensor))
-        quantized = np.round(scale * tensor).astype(dt)
-        return scale, quantized
-
-    def init_input_output(self):
-        self.x_f = np.random.random((100,)).astype("float")
-        self.y_f = np.random.random((100,)).astype("float")
-        self.out_f = np.add(self.x_f, self.y_f)
-
-        self.scale_x, self.x = self.quantize(self.x_f)
-        self.scale_y, self.y = self.quantize(self.y_f)
-        self.scale_o, self.out = self.quantize(self.out_f)
-
-    def init_scales(self):
-        self.attrs['Scale_x'] = self.scale_x
-        self.attrs['Scale_y'] = self.scale_y
-        self.attrs['Scale_out'] = self.scale_o
-
-    def test_check_output(self):
-        # TODO(wangzhongpu): support mkldnn op in dygraph mode
-        self.init_scales()
-        int_atol = 1  # different quantization techniques
-        self.check_output(check_dygraph=(not self.use_mkldnn), atol=int_atol)
-
-
-class TestUint8Scales(TestInt8Scales):
-    def init_input_output(self):
-        self.x_f = np.random.random((100,)).astype("float")
-        self.y_f = np.random.random((100,)).astype("float")
-        self.out_f = np.add(self.x_f, self.y_f)
-
-        self.scale_x, self.x = self.quantize(self.x_f, "uint8")
-        self.scale_y, self.y = self.quantize(self.y_f, "uint8")
-        self.scale_o, self.out = self.quantize(self.out_f, "uint8")
-
-    def init_dtype(self):
-        self.dtype = np.uint8
-
-
 if __name__ == '__main__':
     enable_static()
     unittest.main()
diff --git a/test/mkldnn/test_elementwise_mul_mkldnn_op.py b/test/mkldnn/test_elementwise_mul_onednn_op.py
similarity index 66%
rename from test/mkldnn/test_elementwise_mul_mkldnn_op.py
rename to test/mkldnn/test_elementwise_mul_onednn_op.py
index 8d60282abcbdd..09ba96995d3f5 100644
--- a/test/mkldnn/test_elementwise_mul_mkldnn_op.py
+++ b/test/mkldnn/test_elementwise_mul_onednn_op.py
@@ -23,7 +23,7 @@
 )
 
 
-class TestMKLDNNElementwiseMulOp(ElementwiseMulOp):
+class TestOneDNNElementwiseMulOp(ElementwiseMulOp):
     def init_kernel_type(self):
         self.use_mkldnn = True
 
@@ -31,21 +31,21 @@ def init_dtype(self):
         self.dtype = np.float32
 
 
-class TestMKLDNNElementwiseMulOp2(TestMKLDNNElementwiseMulOp):
+class TestOneDNNElementwiseMulOp2(TestOneDNNElementwiseMulOp):
     def init_input_output(self):
         self.x = np.random.random((100,)).astype(self.dtype)
         self.y = np.random.random((100,)).astype(self.dtype)
         self.out = np.multiply(self.x, self.y)
 
 
-class TestMKLDNNElementwiseMulOp3(TestMKLDNNElementwiseMulOp):
+class TestOneDNNElementwiseMulOp3(TestOneDNNElementwiseMulOp):
     def init_input_output(self):
         self.x = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype)
         self.y = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype)
         self.out = np.multiply(self.x, self.y)
 
 
-class TestMKLDNNElementwiseMulOp4(TestMKLDNNElementwiseMulOp):
+class TestOneDNNElementwiseMulOp4(TestOneDNNElementwiseMulOp):
     def init_input_output(self):
         self.x = np.random.uniform(1, 2, [2, 3, 4, 32]).astype(self.dtype)
         self.y = np.random.uniform(1, 2, [4, 32]).astype(self.dtype)
@@ -59,7 +59,7 @@ def test_check_grad_ingore_y(self):
         pass
 
 
-class TestMKLDNNElementwiseMulOp5(TestMKLDNNElementwiseMulOp):
+class TestOneDNNElementwiseMulOp5(TestOneDNNElementwiseMulOp):
     def init_input_output(self):
         self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype)
         self.y = np.random.uniform(1, 2, [100]).astype(self.dtype)
@@ -76,7 +76,7 @@ def test_check_grad_ingore_x(self):
         pass
 
 
-class TestMKLDNNElementwiseMulOpZeroDim(TestMKLDNNElementwiseMulOp):
+class TestOneDNNElementwiseMulOpZeroDim(TestOneDNNElementwiseMulOp):
     def init_input_output(self):
         self.x = np.random.random((100,)).astype(self.dtype)
         self.y = np.array(3.0).astype(self.dtype)
@@ -92,7 +92,7 @@ def test_check_grad_ingore_x(self):
         pass
 
 
-class TestMKLDNNElementwiseMulOpZeroDim2(TestMKLDNNElementwiseMulOp):
+class TestOneDNNElementwiseMulOpZeroDim2(TestOneDNNElementwiseMulOp):
     def init_input_output(self):
         self.x = np.array(3.0).astype(self.dtype)
         self.y = np.random.random((100,)).astype(self.dtype)
@@ -108,7 +108,7 @@ def test_check_grad_ingore_x(self):
         pass
 
 
-class TestMKLDNNElementwiseMulOpZeroDim3(TestMKLDNNElementwiseMulOp):
+class TestOneDNNElementwiseMulOpZeroDim3(TestOneDNNElementwiseMulOp):
     def init_input_output(self):
         self.x = np.array(3.0).astype(self.dtype)
         self.y = np.array(3.0).astype(self.dtype)
@@ -144,9 +144,9 @@ def init_input_output(self):
         self.out = np.multiply(self.x, self.y)
 
     def init_scales(self):
-        self.attrs['Scale_x'] = 1.0
-        self.attrs['Scale_y'] = 1.0
-        self.attrs['Scale_out'] = 1.0
+        self.attrs['scale_x'] = 1.0
+        self.attrs['scale_y'] = 1.0
+        self.attrs['scale_out'] = 1.0
 
     def test_check_output(self):
         # TODO(wangzhongpu): support mkldnn op in dygraph mode
@@ -163,48 +163,6 @@ def test_check_grad_ingore_y(self):
         pass
 
 
-class TestInt8Scales(TestInt8):
-    def quantize(self, tensor, dt="int8"):
-        max_int = 127.0 if dt == "int8" else 255.0
-        scale = max_int / np.abs(np.amax(tensor))
-        quantized = np.round(scale * tensor).astype(dt)
-        return scale, quantized
-
-    def init_input_output(self):
-        self.x_f = np.random.random((100,)).astype("float")
-        self.y_f = np.random.random((100,)).astype("float")
-        self.out_f = np.multiply(self.x_f, self.y_f)
-
-        self.scale_x, self.x = self.quantize(self.x_f)
-        self.scale_y, self.y = self.quantize(self.y_f)
-        self.scale_o, self.out = self.quantize(self.out_f)
-
-    def init_scales(self):
-        self.attrs['Scale_x'] = self.scale_x
-        self.attrs['Scale_y'] = self.scale_y
-        self.attrs['Scale_out'] = self.scale_o
-
-    def test_check_output(self):
-        # TODO(wangzhongpu): support mkldnn op in dygraph mode
-        self.init_scales()
-        int_atol = 1  # different quantization techniques
-        self.check_output(check_dygraph=(not self.use_mkldnn), atol=int_atol)
-
-
-class TestUint8Scales(TestInt8Scales):
-    def init_input_output(self):
-        self.x_f = np.random.random((100,)).astype("float")
-        self.y_f = np.random.random((100,)).astype("float")
-        self.out_f = np.multiply(self.x_f, self.y_f)
-
-        self.scale_x, self.x = self.quantize(self.x_f, "uint8")
-        self.scale_y, self.y = self.quantize(self.y_f, "uint8")
-        self.scale_o, self.out = self.quantize(self.out_f, "uint8")
-
-    def init_dtype(self):
-        self.dtype = np.uint8
-
-
 if __name__ == '__main__':
     enable_static()
     unittest.main()
diff --git a/test/mkldnn/test_elementwise_sub_mkldnn_op.py b/test/mkldnn/test_elementwise_sub_onednn_op.py
similarity index 91%
rename from test/mkldnn/test_elementwise_sub_mkldnn_op.py
rename to test/mkldnn/test_elementwise_sub_onednn_op.py
index 2f74c0da3180a..35dac7b091ed0 100644
--- a/test/mkldnn/test_elementwise_sub_mkldnn_op.py
+++ b/test/mkldnn/test_elementwise_sub_onednn_op.py
@@ -30,7 +30,7 @@
     not (isinstance(_current_expected_place(), core.CPUPlace)),
     "GPU is not supported",
 )
-class TestMKLDNNElementwiseSubOp(OpTest):
+class TestOneDNNElementwiseSubOp(OpTest):
     def setUp(self):
         self.op_type = "elementwise_sub"
         self.init_dtype()
@@ -71,49 +71,49 @@ def test_check_output(self):
         self.check_output()
 
 
-class TestMKLDNNElementwiseSubOp2(TestMKLDNNElementwiseSubOp):
+class TestOneDNNElementwiseSubOp2(TestOneDNNElementwiseSubOp):
     def init_input_output(self):
         self.x = np.random.random((100,)).astype(self.dtype)
         self.y = np.random.random((100,)).astype(self.dtype)
         self.out = np.subtract(self.x, self.y)
 
 
-class TestMKLDNNElementwiseSubOp3(TestMKLDNNElementwiseSubOp):
+class TestOneDNNElementwiseSubOp3(TestOneDNNElementwiseSubOp):
     def init_input_output(self):
         self.x = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype)
         self.y = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype)
         self.out = np.subtract(self.x, self.y)
 
 
-class TestMKLDNNElementwiseSubOp4(TestMKLDNNElementwiseSubOp):
+class TestOneDNNElementwiseSubOp4(TestOneDNNElementwiseSubOp):
     def init_input_output(self):
         self.x = np.random.uniform(1, 2, [2, 3, 4, 32]).astype(self.dtype)
         self.y = np.random.uniform(1, 2, [4, 32]).astype(self.dtype)
         self.out = np.subtract(self.x, self.y)
 
 
-class TestMKLDNNElementwiseSubOp5(TestMKLDNNElementwiseSubOp):
+class TestOneDNNElementwiseSubOp5(TestOneDNNElementwiseSubOp):
     def init_input_output(self):
         self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype)
         self.y = np.random.uniform(1, 2, [100]).astype(self.dtype)
         self.out = np.subtract(self.x, self.y)
 
 
-class TestMKLDNNElementwiseSubOp6(TestMKLDNNElementwiseSubOp):
+class TestOneDNNElementwiseSubOp6(TestOneDNNElementwiseSubOp):
     def init_input_output(self):
         self.x = np.random.uniform(0.1, 2, [180, 1]).astype(self.dtype)
         self.y = np.random.uniform(0.1, 1, [1, 256]).astype(self.dtype)
         self.out = np.subtract(self.x, self.y)
 
 
-class TestMKLDNNElementwiseSubOp7(TestMKLDNNElementwiseSubOp):
+class TestOneDNNElementwiseSubOp7(TestOneDNNElementwiseSubOp):
     def init_input_output(self):
         self.x = np.random.uniform(0.1, 2, [1, 180]).astype(self.dtype)
         self.y = np.random.uniform(0.1, 1, [256, 1]).astype(self.dtype)
         self.out = np.subtract(self.x, self.y)
 
 
-class TestMKLDNNElementwiseSubOp_broadcast(TestMKLDNNElementwiseSubOp):
+class TestOneDNNElementwiseSubOp_broadcast(TestOneDNNElementwiseSubOp):
     def init_input_output(self):
         self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype)
         self.y = np.random.rand(10, 12).astype(self.dtype)
@@ -123,7 +123,7 @@ def init_axis(self):
         self.axis = 1
 
 
-class TestElementwiseSubOp_xsize_lessthan_ysize_sub(TestMKLDNNElementwiseSubOp):
+class TestElementwiseSubOp_xsize_lessthan_ysize_sub(TestOneDNNElementwiseSubOp):
     def init_input_output(self):
         self.x = np.random.rand(10, 12).astype(self.dtype)
         self.y = np.random.rand(2, 2, 10, 12).astype(self.dtype)
@@ -133,7 +133,7 @@ def init_axis(self):
         self.axis = 2
 
 
-class TestMKLDNNElementwiseSubOpZeroDim(TestMKLDNNElementwiseSubOp):
+class TestOneDNNElementwiseSubOpZeroDim(TestOneDNNElementwiseSubOp):
     def init_input_output(self):
         self.x = np.random.random((100,)).astype(self.dtype)
         self.y = np.array(3.0).astype(self.dtype)
@@ -149,7 +149,7 @@ def test_check_grad_ignore_y(self):
         pass
 
 
-class TestMKLDNNElementwiseSubOpZeroDim2(TestMKLDNNElementwiseSubOp):
+class TestOneDNNElementwiseSubOpZeroDim2(TestOneDNNElementwiseSubOp):
     def init_input_output(self):
         self.x = np.array(3.0).astype(self.dtype)
         self.y = np.random.random((100,)).astype(self.dtype)
@@ -165,7 +165,7 @@ def test_check_grad_ignore_y(self):
         pass
 
 
-class TestMKLDNNElementwiseSubOpZeroDim3(TestMKLDNNElementwiseSubOp):
+class TestOneDNNElementwiseSubOpZeroDim3(TestOneDNNElementwiseSubOp):
     def init_input_output(self):
         self.x = np.array(3.0).astype(self.dtype)
         self.y = np.array(3.0).astype(self.dtype)
@@ -182,7 +182,7 @@ def test_check_grad_ignore_y(self):
 
 
 @OpTestTool.skip_if_not_cpu_bf16()
-class TestBf16(TestMKLDNNElementwiseSubOp):
+class TestBf16(TestOneDNNElementwiseSubOp):
     def setUp(self):
         self.op_type = "elementwise_sub"
         self.init_dtype()
@@ -271,7 +271,7 @@ def test_check_grad_ignore_x(self):
         )
 
 
-class TestInt8(TestMKLDNNElementwiseSubOp):
+class TestInt8(TestOneDNNElementwiseSubOp):
     def init_kernel_type(self):
         self.use_mkldnn = True
         self._cpu_only = True

From d53d8fdcc24bac9636a5d7459b3e323cba41011d Mon Sep 17 00:00:00 2001
From: engineer1109 <1292846099@qq.com>
Date: Thu, 18 May 2023 15:11:25 +0800
Subject: [PATCH 402/405] remove CopyWithContext limitation (#53771)

---
 paddle/phi/kernels/funcs/strided_memcpy.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/paddle/phi/kernels/funcs/strided_memcpy.h b/paddle/phi/kernels/funcs/strided_memcpy.h
index 19a542bd2b5e9..de38e40d317e1 100644
--- a/paddle/phi/kernels/funcs/strided_memcpy.h
+++ b/paddle/phi/kernels/funcs/strided_memcpy.h
@@ -56,7 +56,8 @@ inline void CopyWithContext(const Context& ctx,
                             const Place& src_place,
                             const void* src,
                             size_t num) {
-#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) || \
+    defined(PADDLE_WITH_CUSTOM_DEVICE)
   memory_utils::Copy(dst_place, dst, src_place, src, num, ctx.stream());
 #else
   PADDLE_THROW(

From ba8494108fe62f2d2cad3d2729825e421bbbc683 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Thu, 18 May 2023 16:13:47 +0800
Subject: [PATCH 403/405] Fix qkv_transpose_out's shape and scaling of Q * K.

---
 .../operators/fused/fused_gate_attention.h    | 393 ++++++++----------
 .../fused/fused_gate_attention_op.cu          |  13 +-
 2 files changed, 187 insertions(+), 219 deletions(-)

diff --git a/paddle/fluid/operators/fused/fused_gate_attention.h b/paddle/fluid/operators/fused/fused_gate_attention.h
index 341e7fb1f8ff0..86264641b2dd8 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention.h
+++ b/paddle/fluid/operators/fused/fused_gate_attention.h
@@ -19,9 +19,9 @@ limitations under the License. */
 #endif
 
 #include "paddle/phi/kernels/arange_kernel.h"
+#include "paddle/phi/kernels/check_numerics_kernel.h"
 #include "paddle/phi/kernels/funcs/broadcast_function.h"
 #include "paddle/phi/kernels/funcs/elementwise_functor.h"
-#include "paddle/phi/kernels/funcs/range_function.h"
 #include "paddle/phi/kernels/funcs/reduce_function.h"
 #include "paddle/phi/kernels/funcs/transpose_function.cu.h"
 #include "paddle/phi/kernels/gpudnn/softmax_gpudnn.h"
@@ -29,6 +29,13 @@ limitations under the License. */
 namespace paddle {
 namespace operators {
 
+template <typename T>
+__global__ void SimleScaleKernel(int64_t numel, float scale, T* inout) {
+  CUDA_KERNEL_LOOP_TYPE(i, numel, int64_t) {
+    inout[i] = static_cast<T>(scale * static_cast<float>(inout[i]));
+  }
+}
+
 inline std::string MemoryDebugString(const phi::DenseTensor& t) {
   int device_id = platform::GetCurrentDeviceId();
   int64_t allocated =
@@ -196,11 +203,12 @@ struct GateAttentionConfig {
 
   bool CanUseFlashAttn() const {
 #ifdef PADDLE_WITH_FLASHATTN
-    if (!std::is_same<T, phi::dtype::bfloat16>::value) {
+    if (!std::is_same<T, phi::dtype::bfloat16>::value &&
+        !std::is_same<T, phi::dtype::float16>::value) {
       return false;
     }
 
-    if (merge_qkv) {
+    if (merge_qkv && batch_size == 1) {
       if (head_dim == 32 || head_dim == 64 || head_dim == 128) {
         return use_flash_attn;
       }
@@ -382,14 +390,6 @@ struct GateAttentionGradConfig : public GateAttentionConfig<T> {
   phi::DenseTensor qk_out_grad;
 };
 
-template <typename T>
-__global__ void FlashAttRange(int start, int step, int size, T* out1, T* out2) {
-  CUDA_KERNEL_LOOP(index, size) {
-    out1[index] = static_cast<T>(start + step * index);
-    out2[index] = static_cast<T>(start + step * index);
-  }
-}
-
 template <typename T>
 class FMHAGateRef {
  public:
@@ -877,9 +877,9 @@ class FlashAttnWithGating {
         platform::errors::NotFound("The input qkv_transpose_out can not be "
                                    "nullptr when merge_qkv is true."));
 
-    // 1. Dealing with qkv_out for flash_attn.
+    // 1. Transpose qkv_out for flash_attn.
     phi::DenseTensor* qkv_out = config->GetQKVOut();
-    ComputeQKVTransposeForwardForFlashAttn(*qkv_out, qkv_transpose_out);
+    ComputeQKVTransposeForward(*qkv_out, qkv_transpose_out);
     config->ClearQKVOut();
 
     // q_size == k_size
@@ -888,21 +888,23 @@ class FlashAttnWithGating {
     T* k_ptr = q_ptr + q_size;
     T* v_ptr = k_ptr + q_size;
 
-    // 2. flash_attn parameter setting.
+    // 2. Scale Q: q_ptr = alpha * q_ptr
+    ComputeScaleQ(q_size, config->head_dim, q_ptr);
+
+    // 3. flash_attn parameter setting.
     phi::DenseTensor cu_seq_q;
     phi::DenseTensor cu_seq_k;
     InitArgumentsAndSeqTensors(config, &cu_seq_q, &cu_seq_k);
 
-    // 3. Dealing with mask and bias for flash_attn.
     std::vector<int64_t> temp_mask_dim = GetCompressedDim(src_mask);
     std::vector<int64_t> temp_bias_dim = GetCompressedDim(nonbatched_bias);
 
-    // 4. construct softmax_lse
-    int softmax_lse_last_dim = ((fa_max_seqlen_q_ + 16 - 1) / 16) * 16;
-    softmax_lse->Resize({fa_batch_size_, fa_num_heads_, softmax_lse_last_dim});
-    AllocWithDebugInfo<float>(dev_ctx_, "flash_attn: softmax_lse", softmax_lse);
+    softmax_lse->Resize({fa_batch_size_, fa_num_heads_, fa_softmax_lse_dim_});
+    AllocWithDebugInfo<float>(dev_ctx_, "softmax_lse", softmax_lse);
 
     if (VLOG_IS_ON(6)) {
+      VLOG(6) << "temp_mask_dim={" << phi::make_ddim(temp_mask_dim) << "}";
+      VLOG(6) << "temp_bias_dim={" << phi::make_ddim(temp_bias_dim) << "}";
       VLOG(6) << TensorDebugString(&cu_seq_q, "cu_seq_q");
       VLOG(6) << TensorDebugString(&cu_seq_k, "cu_seq_k");
       VLOG(6) << TensorDebugString(nonbatched_bias, "nonbatched_bias");
@@ -912,75 +914,59 @@ class FlashAttnWithGating {
       VLOG(6) << TensorDebugString(fmha_out, "fmha_out");
     }
 
-    // 5. flash_attn part one, get temp worksapce size.
+    // 4. Get worksapce size and run the flash-attention kernel.
+    uint64_t workspace_size = 0;
+    phi::DenseTensor workspace;
     cudaStream_t stream = dev_ctx_.stream();
-    uint64_t workspace_size;
-    bool succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
-        static_cast<const void*>(q_ptr),
-        static_cast<const void*>(k_ptr),
-        static_cast<const void*>(v_ptr),
-        nullptr,  // set out to nullptr to calculate workspace size
-        cu_seq_q.data<int32_t>(),
-        cu_seq_k.data<int32_t>(),
-        fa_total_q_,
-        fa_total_k_,
-        fa_batch_size_,
-        fa_num_heads_,
-        fa_head_size_,
-        fa_max_seqlen_q_,
-        fa_max_seqlen_k_,
-        fa_dropout_prob_,
-        fa_softmax_scale_,
-        fa_zero_tensors_,
-        is_bf16,
-        fa_num_splits_,
-        softmax_lse->data(),
-        nullptr,
-        &workspace_size,
-        stream,
-        fa_seed_,
-        fa_offset_,
-        src_mask ? src_mask->data() : nullptr,
-        nonbatched_bias ? nonbatched_bias->data() : nullptr,
-        src_mask ? temp_mask_dim.data() : nullptr,
-        nonbatched_bias ? temp_bias_dim.data() : nullptr);
-    PADDLE_ENFORCE_EQ(
-        succ, true, phi::errors::External(phi::dynload::flash_attn_error()));
-
-    // 6. Run flash-attention kernel.
-    phi::DenseTensor workspace = CreateWorkspace(workspace_size);
-    succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
-        static_cast<const void*>(q_ptr),
-        static_cast<const void*>(k_ptr),
-        static_cast<const void*>(v_ptr),
-        static_cast<void*>(fmha_out->data()),
-        cu_seq_q.data<int32_t>(),
-        cu_seq_k.data<int32_t>(),
-        fa_total_q_,
-        fa_total_k_,
-        fa_batch_size_,
-        fa_num_heads_,
-        fa_head_size_,
-        fa_max_seqlen_q_,
-        fa_max_seqlen_k_,
-        fa_dropout_prob_,
-        fa_softmax_scale_,
-        fa_zero_tensors_,
-        is_bf16,
-        fa_num_splits_,
-        softmax_lse->data(),
-        (workspace_size > 0) ? static_cast<void*>(workspace.data()) : nullptr,
-        &workspace_size,
-        stream,
-        fa_seed_,
-        fa_offset_,
-        src_mask ? src_mask->data() : nullptr,
-        nonbatched_bias ? nonbatched_bias->data() : nullptr,
-        src_mask ? temp_mask_dim.data() : nullptr,
-        nonbatched_bias ? temp_bias_dim.data() : nullptr);
-    PADDLE_ENFORCE_EQ(
-        succ, true, phi::errors::External(phi::dynload::flash_attn_error()));
-    WaitWithDebugInfo(dev_ctx_);
+    for (bool need_calc : {false, true}) {
+      // first calling, need_calc=false, set out_ptr to nullptr to calculate
+      // workspace size second calling, need_calc=true, run flash-attention
+      // kernel.
+      void* out_ptr =
+          need_calc ? static_cast<void*>(fmha_out->data()) : nullptr;
+      void* workspace_ptr = nullptr;
+      if (need_calc) {
+        VLOG(6) << "Step 2: Call the flash-attention kernel";
+        if (workspace_size > 0) {
+          workspace = CreateWorkspace(workspace_size);
+          workspace_ptr = static_cast<void*>(workspace.data());
+        }
+      } else {
+        VLOG(6) << "Step 1: Calculate the workspace_size";
+      }
+      bool succ = phi::dynload::flash_attn_fwd_with_bias_and_mask(
+          static_cast<const void*>(q_ptr),
+          static_cast<const void*>(k_ptr),
+          static_cast<const void*>(v_ptr),
+          out_ptr,  // set out to nullptr to calculate workspace size
+          cu_seq_q.data<int32_t>(),
+          cu_seq_k.data<int32_t>(),
+          fa_total_q_,
+          fa_total_k_,
+          fa_batch_size_,
+          fa_num_heads_,
+          fa_head_size_,
+          fa_max_seqlen_q_,
+          fa_max_seqlen_k_,
+          fa_dropout_prob_,
+          fa_softmax_scale_,
+          fa_zero_tensors_,
+          is_bf16,
+          fa_num_splits_,
+          softmax_lse->data(),
+          workspace_ptr,
+          &workspace_size,
+          stream,
+          fa_seed_,
+          fa_offset_,
+          src_mask ? src_mask->data() : nullptr,
+          nonbatched_bias ? nonbatched_bias->data() : nullptr,
+          src_mask ? temp_mask_dim.data() : nullptr,
+          nonbatched_bias ? temp_bias_dim.data() : nullptr);
+      PADDLE_ENFORCE_EQ(
+          succ, true, phi::errors::External(phi::dynload::flash_attn_error()));
+      WaitWithDebugInfo(dev_ctx_);
+    }
 #else
     PADDLE_THROW(phi::errors::Unimplemented(
         "FlashAttention is unsupported, please set use_flash_attn to false."));
@@ -1012,7 +998,12 @@ class FlashAttnWithGating {
     const T* v_ptr = k_ptr + q_size;
 
     phi::DenseTensor qkv_transpose_out_grad;
-    qkv_transpose_out_grad.Resize(qkv_transpose_out->dims());
+    qkv_transpose_out_grad.Resize(phi::make_ddim({3,
+                                                  config->batch_size,
+                                                  config->seq_len_m,
+                                                  config->seq_len_r,
+                                                  config->num_heads,
+                                                  config->head_dim}));
     AllocWithDebugInfo<T>(
         dev_ctx_, "qkv_transpose_out_grad", &qkv_transpose_out_grad);
 
@@ -1021,18 +1012,16 @@ class FlashAttnWithGating {
     T* v_grad_ptr = k_grad_ptr + q_size;
     WaitWithDebugInfo(dev_ctx_);
 
-    // 2. flash_attn parameter setting.
+    // 1. flash_attn parameter setting.
     phi::DenseTensor cu_seq_q;
     phi::DenseTensor cu_seq_k;
     InitArgumentsAndSeqTensors(config, &cu_seq_q, &cu_seq_k);
     const int32_t* cu_seq_q_ptr = cu_seq_q.data<int32_t>();
     const int32_t* cu_seq_k_ptr = cu_seq_k.data<int32_t>();
 
-    // 3. Dealing with mask and bias for flash_attn.
     std::vector<int64_t> temp_mask_dim = GetCompressedDim(src_mask);
     std::vector<int64_t> temp_bias_dim = GetCompressedDim(nonbatched_bias);
 
-    // 4. construct softmax_lse
     phi::DenseTensor softmax_d;
     softmax_d.Resize(softmax_lse->dims());
     AllocWithDebugInfo<float>(dev_ctx_, "d_softmax_lse", &softmax_d);
@@ -1053,86 +1042,66 @@ class FlashAttnWithGating {
       VLOG(6) << TensorDebugString(&bias_d, "bias_d");
     }
 
-    // 5. flas_attn part one, get temp worksapce size.
-    uint64_t workspace_size;
+    // 2. Get worksapce size and run the flash-attention kernel.
+    uint64_t workspace_size = 0;
+    phi::DenseTensor workspace;
     cudaStream_t stream = dev_ctx_.stream();
-    bool succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
-        static_cast<const void*>(q_ptr),
-        static_cast<const void*>(k_ptr),
-        static_cast<const void*>(v_ptr),
-        static_cast<void*>(q_grad_ptr),
-        static_cast<void*>(k_grad_ptr),
-        static_cast<void*>(v_grad_ptr),
-        nullptr,  // set out to nullptr to calculate workspace size
-        static_cast<const void*>(fmha_out_grad->data()),
-        cu_seq_q_ptr,
-        cu_seq_k_ptr,
-        fa_total_q_,
-        fa_total_k_,
-        fa_batch_size_,
-        fa_num_heads_,
-        fa_head_size_,
-        fa_max_seqlen_q_,
-        fa_max_seqlen_k_,
-        fa_dropout_prob_,
-        fa_softmax_scale_,
-        fa_zero_tensors_,
-        is_bf16,
-        fa_num_splits_,
-        softmax_lse->data(),
-        softmax_d.data(),
-        nonbatched_bias ? bias_d.data() : nullptr,
-        nullptr,
-        &workspace_size,
-        stream,
-        fa_seed_,
-        fa_offset_,
-        src_mask ? src_mask->data() : nullptr,
-        nonbatched_bias ? nonbatched_bias->data() : nullptr,
-        src_mask ? temp_mask_dim.data() : nullptr,
-        nonbatched_bias ? temp_bias_dim.data() : nullptr);
-    PADDLE_ENFORCE_EQ(
-        succ, true, phi::errors::External(phi::dynload::flash_attn_error()));
-
-    phi::DenseTensor workspace = CreateWorkspace(workspace_size);
-    succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
-        static_cast<const void*>(q_ptr),
-        static_cast<const void*>(k_ptr),
-        static_cast<const void*>(v_ptr),
-        static_cast<void*>(q_grad_ptr),
-        static_cast<void*>(k_grad_ptr),
-        static_cast<void*>(v_grad_ptr),
-        static_cast<const void*>(fmha_out->data()),
-        static_cast<const void*>(fmha_out_grad->data()),
-        cu_seq_q_ptr,
-        cu_seq_k_ptr,
-        fa_total_q_,
-        fa_total_k_,
-        fa_batch_size_,
-        fa_num_heads_,
-        fa_head_size_,
-        fa_max_seqlen_q_,
-        fa_max_seqlen_k_,
-        fa_dropout_prob_,
-        fa_softmax_scale_,
-        fa_zero_tensors_,
-        is_bf16,
-        fa_num_splits_,
-        softmax_lse->data(),
-        softmax_d.data(),
-        nonbatched_bias ? bias_d.data() : nullptr,
-        (workspace_size > 0) ? static_cast<void*>(workspace.data()) : nullptr,
-        &workspace_size,
-        stream,
-        fa_seed_,
-        fa_offset_,
-        src_mask ? src_mask->data() : nullptr,
-        nonbatched_bias ? nonbatched_bias->data() : nullptr,
-        src_mask ? temp_mask_dim.data() : nullptr,
-        nonbatched_bias ? temp_bias_dim.data() : nullptr);
-    PADDLE_ENFORCE_EQ(
-        succ, true, phi::errors::External(phi::dynload::flash_attn_error()));
-    WaitWithDebugInfo(dev_ctx_);
+    for (bool need_calc : {false, true}) {
+      // first calling, need_calc=false, set out_ptr to nullptr to calculate
+      // workspace size second calling, need_calc=true, run flash-attention
+      // kernel.
+      const void* out_ptr =
+          need_calc ? static_cast<const void*>(fmha_out->data()) : nullptr;
+      void* workspace_ptr = nullptr;
+      if (need_calc) {
+        VLOG(6) << "Step 2: Call the flash-attention kernel";
+        if (workspace_size > 0) {
+          workspace = CreateWorkspace(workspace_size);
+          workspace_ptr = static_cast<void*>(workspace.data());
+        }
+      } else {
+        VLOG(6) << "Step 1: Calculate the workspace_size";
+      }
+
+      bool succ = phi::dynload::flash_attn_bwd_with_bias_and_mask(
+          static_cast<const void*>(q_ptr),
+          static_cast<const void*>(k_ptr),
+          static_cast<const void*>(v_ptr),
+          static_cast<void*>(q_grad_ptr),
+          static_cast<void*>(k_grad_ptr),
+          static_cast<void*>(v_grad_ptr),
+          out_ptr,  // set out to nullptr to calculate workspace size
+          static_cast<const void*>(fmha_out_grad->data()),
+          cu_seq_q_ptr,
+          cu_seq_k_ptr,
+          fa_total_q_,
+          fa_total_k_,
+          fa_batch_size_,
+          fa_num_heads_,
+          fa_head_size_,
+          fa_max_seqlen_q_,
+          fa_max_seqlen_k_,
+          fa_dropout_prob_,
+          fa_softmax_scale_,
+          fa_zero_tensors_,
+          is_bf16,
+          fa_num_splits_,
+          softmax_lse->data(),
+          softmax_d.data(),
+          nonbatched_bias ? bias_d.data() : nullptr,
+          workspace_ptr,
+          &workspace_size,
+          stream,
+          fa_seed_,
+          fa_offset_,
+          src_mask ? src_mask->data() : nullptr,
+          nonbatched_bias ? nonbatched_bias->data() : nullptr,
+          src_mask ? temp_mask_dim.data() : nullptr,
+          nonbatched_bias ? temp_bias_dim.data() : nullptr);
+      PADDLE_ENFORCE_EQ(
+          succ, true, phi::errors::External(phi::dynload::flash_attn_error()));
+      WaitWithDebugInfo(dev_ctx_);
+    }
 
     if (nonbatched_bias) {
       // compare block reduce
@@ -1150,6 +1119,10 @@ class FlashAttnWithGating {
           {0});
     }
 
+    // 3. Scale Q's grad: q_grad_ptr = alpha * q_grad_ptr
+    ComputeScaleQ(q_size, config->head_dim, q_grad_ptr);
+
+    // 4. Compute the grad of qkv_out.
     phi::DenseTensor* qkv_out_grad = config->GetQKVOutGrad();
     ComputeQKVTransposeBackward(qkv_transpose_out_grad, qkv_out_grad);
 #else
@@ -1159,29 +1132,6 @@ class FlashAttnWithGating {
   }
 
  private:
-  void AllocAndInitSeqQK(int64_t seq_batch_size,
-                         int64_t step,
-                         phi::DenseTensor* cu_seq_q,
-                         phi::DenseTensor* cu_seq_k) {
-    int64_t start = 0;
-    int64_t end_size = seq_batch_size + 1;
-    int64_t end = end_size;
-    int64_t seq_size = 0;
-    phi::funcs::GetSize<int64_t>(start, end, step, &seq_size);
-
-    cu_seq_q->Resize({end_size});
-    cu_seq_k->Resize({end_size});
-    AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_q", cu_seq_q);
-    AllocWithDebugInfo<int32_t>(dev_ctx_, "cu_seq_k", cu_seq_k);
-
-    int64_t block = std::min(seq_size, static_cast<int64_t>(256));
-    int64_t grid = (seq_size + block - 1) / block;
-    FlashAttRange<int32_t><<<grid, block, 0, dev_ctx_.stream()>>>(
-        start, step, end, cu_seq_q->data<int32_t>(), cu_seq_k->data<int32_t>());
-    VLOG(5) << "AllocAndInit cu_seq_q and cu_seq_k: start=" << start
-            << ", step=" << step << ", end=" << end;
-  }
-
   std::vector<int64_t> GetCompressedDim(const phi::DenseTensor* tensor) {
     std::vector<int64_t> compressed_dims;
     if (tensor) {
@@ -1209,11 +1159,12 @@ class FlashAttnWithGating {
     return workspace;
   }
 
-  std::pair<uint64_t, uint64_t> GenerateSeedOffsetPair(int64_t batch_size,
-                                                       int64_t num_heads) {
+  void GenerateSeedAndOffset(int64_t batch_size, int64_t num_heads) {
     auto gen = dev_ctx_.GetGenerator();
     uint64_t inc = batch_size * num_heads * 32;
-    return gen->IncrementOffset(inc);
+    auto seed_offset_pair = gen->IncrementOffset(inc);
+    fa_seed_ = seed_offset_pair.first;
+    fa_offset_ = seed_offset_pair.second;
   }
 
   void InitArgumentsAndSeqTensors(GateAttentionConfig<T>* config,
@@ -1230,16 +1181,12 @@ class FlashAttnWithGating {
 
     // 0 for an internal heuristic, which is optimal
     fa_num_splits_ = 0;
+    fa_zero_tensors_ = false;
 
-    fa_softmax_scale_ = static_cast<float>(1.0f / std::sqrt(fa_head_size_));
+    fa_softmax_lse_dim_ = ((fa_max_seqlen_q_ + 16 - 1) / 16) * 16;
+    fa_softmax_scale_ = 1.0f;
     fa_dropout_prob_ = 0.0f;
-
-    auto seed_offset_pair =
-        GenerateSeedOffsetPair(fa_batch_size_, fa_num_heads_);
-    fa_seed_ = seed_offset_pair.first;
-    fa_offset_ = seed_offset_pair.second;
-
-    fa_zero_tensors_ = false;
+    GenerateSeedAndOffset(fa_batch_size_, fa_num_heads_);
 
     phi::ArangeNullaryKernel<int32_t, phi::GPUContext>(
         dev_ctx_,
@@ -1255,23 +1202,24 @@ class FlashAttnWithGating {
         cu_seq_k);
 
     if (VLOG_IS_ON(6)) {
-      VLOG(6) << "fa_batch_size    : " << fa_batch_size_;
-      VLOG(6) << "fa_total_q       : " << fa_total_q_;
-      VLOG(6) << "fa_total_k       : " << fa_total_k_;
-      VLOG(6) << "fa_num_heads     : " << fa_num_heads_;
-      VLOG(6) << "fa_head_size     : " << fa_head_size_;
-      VLOG(6) << "fa_max_seqlen_q  : " << fa_max_seqlen_q_;
-      VLOG(6) << "fa_max_seqlen_k  : " << fa_max_seqlen_k_;
-      VLOG(6) << "fa_num_splits    : " << fa_num_splits_;
-      VLOG(6) << "fa_softmax_scale : " << fa_softmax_scale_;
-      VLOG(6) << "fa_dropout_prob  : " << fa_dropout_prob_;
+      VLOG(6) << "fa_batch_size       : " << fa_batch_size_;
+      VLOG(6) << "fa_total_q          : " << fa_total_q_;
+      VLOG(6) << "fa_total_k          : " << fa_total_k_;
+      VLOG(6) << "fa_num_heads        : " << fa_num_heads_;
+      VLOG(6) << "fa_head_size        : " << fa_head_size_;
+      VLOG(6) << "fa_max_seqlen_q     : " << fa_max_seqlen_q_;
+      VLOG(6) << "fa_max_seqlen_k     : " << fa_max_seqlen_k_;
+      VLOG(6) << "fa_num_splits       : " << fa_num_splits_;
+      VLOG(6) << "fa_softmax_lse_dim  : " << fa_softmax_lse_dim_;
+      VLOG(6) << "fa_softmax_scale    : " << fa_softmax_scale_;
+      VLOG(6) << "fa_dropout_prob     : " << fa_dropout_prob_;
     }
   }
 
   // [batch_size, seq_len_m, seq_len_r, 3, num_heads, head_dim] ->
   //         [3, batch_size, seq_len_m, seq_len_r, num_heads, head_dim]
-  void ComputeQKVTransposeForwardForFlashAttn(
-      const phi::DenseTensor& qkv_out, phi::DenseTensor* qkv_transpose_out) {
+  void ComputeQKVTransposeForward(const phi::DenseTensor& qkv_out,
+                                  phi::DenseTensor* qkv_transpose_out) {
     std::vector<int> perm = {3, 0, 1, 2, 4, 5};
     phi::funcs::TransposeGPUKernelDriver<T>(
         dev_ctx_, qkv_out, perm, qkv_transpose_out);
@@ -1287,6 +1235,18 @@ class FlashAttnWithGating {
         dev_ctx_, qkv_transpose_out_grad, perm, qkv_out_grad);
   }
 
+  void ComputeScaleQ(int64_t numel, int64_t head_dim, T* ptr) {
+    float scale = static_cast<float>(1.0f / std::sqrt(head_dim));
+    VLOG(6) << "[ComputeScaleQ] numel=" << numel << ", scale=" << scale;
+
+    auto gpu_config =
+        phi::backends::gpu::GetGpuLaunchConfig1D(dev_ctx_, numel, 1);
+    SimleScaleKernel<T><<<gpu_config.block_per_grid,
+                          gpu_config.thread_per_block,
+                          0,
+                          dev_ctx_.stream()>>>(numel, scale, ptr);
+  }
+
   const phi::GPUContext& dev_ctx_;
   bool merge_qkv_;
 
@@ -1298,6 +1258,7 @@ class FlashAttnWithGating {
   int fa_max_seqlen_q_;
   int fa_max_seqlen_k_;
   int fa_num_splits_;
+  int fa_softmax_lse_dim_;
   float fa_softmax_scale_{1.0f};
   float fa_dropout_prob_{0.0f};
   uint64_t fa_seed_{0};
diff --git a/paddle/fluid/operators/fused/fused_gate_attention_op.cu b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
index 2af6c9088f4f7..e2cdb513feada 100644
--- a/paddle/fluid/operators/fused/fused_gate_attention_op.cu
+++ b/paddle/fluid/operators/fused/fused_gate_attention_op.cu
@@ -411,6 +411,14 @@ class FusedGateAttentionOpKernel : public framework::OpKernel<T> {
       phi::DenseTensor *qkv_out = config.GetQKVOut();
       ComputeMergedQKVMatmulForward<T>(ctx, config, query, qkv_out);
 
+      if (config.CanUseFlashAttn()) {
+        qkv_transpose_out->Resize(phi::make_ddim({3,
+                                                  config.batch_size,
+                                                  config.seq_len_m,
+                                                  config.seq_len_r,
+                                                  config.num_heads,
+                                                  config.head_dim}));
+      }
       AllocWithDebugInfo<T>(dev_ctx, "qkv_transpose_out", qkv_transpose_out);
     } else {
       // 1. Separated QKV Matmul
@@ -434,7 +442,6 @@ class FusedGateAttentionOpKernel : public framework::OpKernel<T> {
                                   qkv_transpose_out,
                                   softmax_lse,
                                   fmha_out,
-                                  gate_out,
                                   &config);
     } else {
       auto *softmax_out = ctx.Output<phi::DenseTensor>("SoftmaxOut");
@@ -545,7 +552,7 @@ class FusedGateAttentionGradKernel : public framework::OpKernel<T> {
     }
 
     if (config.CanUseFlashAttn()) {
-      const auto *non_batched_bias =
+      const auto *nonbatched_bias =
           ctx.Input<phi::DenseTensor>("NonbatchedBias");
       const auto *src_mask = ctx.Input<phi::DenseTensor>("SrcMask");
       const auto *softmax_lse = ctx.Input<phi::DenseTensor>("SoftmaxLse");
@@ -553,7 +560,7 @@ class FusedGateAttentionGradKernel : public framework::OpKernel<T> {
       auto fmha_compute = FlashAttnWithGating<T>(dev_ctx, merge_qkv);
       fmha_compute.ComputeBackward(qkv_transpose_out,
                                    src_mask,
-                                   non_batched_bias,
+                                   nonbatched_bias,
                                    softmax_lse,
                                    fmha_out,
                                    &fmha_out_grad,

From c3c8579441e88cb67210ccae4be081c17be542da Mon Sep 17 00:00:00 2001
From: co63oc <co63oc@users.noreply.github.com>
Date: Thu, 18 May 2023 16:14:38 +0800
Subject: [PATCH 404/405] Add einsum tests (#53722)

---
 .../fluid/tests/unittests/test_einsum_op.py   | 64 +++++++++++++++++--
 1 file changed, 60 insertions(+), 4 deletions(-)

diff --git a/python/paddle/fluid/tests/unittests/test_einsum_op.py b/python/paddle/fluid/tests/unittests/test_einsum_op.py
index 2d6f7efca2a6a..b69bdcf075124 100644
--- a/python/paddle/fluid/tests/unittests/test_einsum_op.py
+++ b/python/paddle/fluid/tests/unittests/test_einsum_op.py
@@ -15,15 +15,18 @@
 import unittest
 
 import numpy as np
-from eager_op_test import OpTest
+from eager_op_test import OpTest, convert_float_to_uint16
 
 import paddle
+from paddle.fluid import core
 
 
 def einsum_wrapper(a, b):
     if not isinstance(a, list):
         a = [a]
-    return paddle._C_ops.einsum(a, b)
+    ret = paddle._C_ops.einsum(a, b)
+    # ret include list: [Tensor(Not initialized)], skip the list
+    return ret[0]
 
 
 class TestEinsumBinary(OpTest):
@@ -33,10 +36,14 @@ def setUp(self):
         self.python_api = einsum_wrapper
         self.python_out_sig = ['Out']
         self.disable = False
+        self.init_dtype()
         self.set_mandatory()
         self.init_input()
         np.random.seed(123)
         out = np.einsum(self.equation, *self.inputs)
+        # bfloat16 change inputs
+        if self.dtype == np.uint16:
+            self.inputs = self.bf16_inputs
         self.operands = []
         for idx, inp in enumerate(self.inputs):
             self.operands.append(("x" + str(idx), inp))
@@ -53,16 +60,28 @@ def setUp(self):
                 for i in range(len(self.operands))
             ],
         }
+        if self.dtype == np.uint16:
+            self.place = core.CUDAPlace(0)
+            self.outputs["Out"] = convert_float_to_uint16(self.outputs["Out"])
+
+    def init_dtype(self):
+        self.dtype = np.float64
 
     def init_input(self):
         self.inputs = []
+        self.bf16_inputs = []
         for t, s in zip(self.types, self.shapes):
-            self.inputs.append(np.random.random(s).astype(t))
+            input_data = np.random.random(s).astype(t)
+            self.inputs.append(input_data)
+            if self.dtype == np.uint16:
+                self.bf16_inputs.append(convert_float_to_uint16(input_data))
 
     def set_mandatory(self):
         self.shapes = [(10, 10, 20), (20, 6)]
-        self.types = [np.float64, np.float64]
+        self.types = [self.dtype, self.dtype]
         self.equation = "mij,jk->ki"
+        if self.dtype == np.uint16:
+            self.types = [self.np_dtype, self.np_dtype]
 
     def test_check_output(self):
         if not self.disable:
@@ -213,5 +232,42 @@ def set_mandatory(self):
         self.equation = "ijki,jkjk->"
 
 
+class TestEinsumFP16Op(TestEinsumBinary):
+    def init_dtype(self):
+        self.dtype = np.float16
+
+
+@unittest.skipIf(
+    not core.is_compiled_with_cuda()
+    or not core.is_bfloat16_supported(core.CUDAPlace(0)),
+    "core is not compiled with CUDA or not support bfloat16",
+)
+class TestEinsumBF16Op(TestEinsumBinary):
+    def init_dtype(self):
+        self.dtype = np.uint16
+        self.np_dtype = np.float32
+
+    # If it is a complex calculation, the difference value is large
+    def set_mandatory(self):
+        self.shapes = [(10, 3, 10)]
+        self.types = [self.np_dtype]
+        self.equation = "iji->j"
+
+    def test_check_output(self):
+        if not self.disable:
+            self.check_output_with_place(
+                self.place, no_check_set=["InnerCache", "XShape"]
+            )
+
+    def test_grad(self):
+        if not self.disable:
+            self.check_grad_with_place(
+                self.place,
+                [op[0] for op in self.operands],
+                ["Out"],
+                numeric_grad_delta=0.05,
+            )
+
+
 if __name__ == "__main__":
     unittest.main()

From 374797855e8d29eebda79fd5fd1ae31114fe8ac2 Mon Sep 17 00:00:00 2001
From: Liu Yiqun <liuyiqun01@baidu.com>
Date: Thu, 18 May 2023 18:55:13 +0800
Subject: [PATCH 405/405] Update commit of flash-attention.

---
 cmake/external/flashattn.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index 8d09f7823a4da..6895740675625 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -19,8 +19,8 @@ add_definitions(-DPADDLE_WITH_FLASHATTN)
 set(FLASHATTN_PREFIX_DIR ${THIRD_PARTY_PATH}/flashattn)
 set(FLASHATTN_SOURCE_SUBDIR csrc/flash_attn)
 set(FLASHATTN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flashattn)
-set(FLASHATTN_REPOSITORY ${GIT_URL}/Xreki/flash-attention.git)
-set(FLASHATTN_TAG reduce_lib)
+set(FLASHATTN_REPOSITORY ${GIT_URL}/PaddlePaddle/flash-attention.git)
+set(FLASHATTN_TAG 18106c1ba0ccee81b97ca947397c08a141815a47)
 
 set(FLASHATTN_INCLUDE_DIR
     "${FLASHATTN_INSTALL_DIR}/include"